raphael/minishell
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Changed (u)int\d+_t to their stdme types

Browse source
This commit is contained in:
Maix0 2024-07-03 19:03:19 +02:00
parent a7bfe526b0
commit 38bdd66f78
22 changed files with 1034 additions and 981 deletions
Download patch file Download diff file Expand all files Collapse all files

111
parser/src/api.h
View file

@ -1,9 +1,7 @@
#ifndef TREE_SITTER_API_H_ #ifndef TREE_SITTER_API_H_
#define TREE_SITTER_API_H_ #define TREE_SITTER_API_H_
#include <stdbool.h> #include "me/types.h"
#include <stdint.h>
#include <stdlib.h>
#define ERROR_STATE 0 #define ERROR_STATE 0
#define ERROR_COST_PER_RECOVERY 500 #define ERROR_COST_PER_RECOVERY 500
@ -35,9 +33,9 @@
/* Section - Types */ /* Section - Types */
/*******************/ /*******************/
typedef uint16_t TSStateId; typedef t_u16 TSStateId;
typedef uint16_t TSSymbol; typedef t_u16 TSSymbol;
typedef uint16_t TSFieldId; typedef t_u16 TSFieldId;
typedef struct TSLanguage TSLanguage; typedef struct TSLanguage TSLanguage;
typedef struct TSParser TSParser; typedef struct TSParser TSParser;
typedef struct TSTree TSTree; typedef struct TSTree TSTree;
@ -60,22 +58,22 @@ typedef enum TSSymbolType
typedef struct TSPoint typedef struct TSPoint
{ {
uint32_t row; t_u32 row;
uint32_t column; t_u32 column;
} TSPoint; } TSPoint;
typedef struct TSRange typedef struct TSRange
{ {
TSPoint start_point; TSPoint start_point;
TSPoint end_point; TSPoint end_point;
uint32_t start_byte; t_u32 start_byte;
uint32_t end_byte; t_u32 end_byte;
} TSRange; } TSRange;
typedef struct TSInput typedef struct TSInput
{ {
void *payload; void *payload;
const char *(*read)(void *payload, uint32_t byte_index, TSPoint position, uint32_t *bytes_read); const char *(*read)(void *payload, t_u32 byte_index, TSPoint position, t_u32 *bytes_read);
TSInputEncoding encoding; TSInputEncoding encoding;
} TSInput; } TSInput;
@ -93,17 +91,17 @@ typedef struct TSLogger
typedef struct TSInputEdit typedef struct TSInputEdit
{ {
uint32_t start_byte; t_u32 start_byte;
uint32_t old_end_byte; t_u32 old_end_byte;
uint32_t new_end_byte; t_u32 new_end_byte;
TSPoint start_point; TSPoint start_point;
TSPoint old_end_point; TSPoint old_end_point;
TSPoint new_end_point; TSPoint new_end_point;
} TSInputEdit; } TSInputEdit;
typedef struct TSNode typedef struct TSNode
{ {
uint32_t context[4]; t_u32 context[4];
const void *id; const void *id;
const TSTree *tree; const TSTree *tree;
} TSNode; } TSNode;
@ -112,13 +110,13 @@ typedef struct TSTreeCursor
{ {
const void *tree; const void *tree;
const void *id; const void *id;
uint32_t context[3]; t_u32 context[3];
} TSTreeCursor; } TSTreeCursor;
typedef struct TSQueryCapture typedef struct TSQueryCapture
{ {
TSNode node; TSNode node;
uint32_t index; t_u32 index;
} TSQueryCapture; } TSQueryCapture;
typedef enum TSQuantifier typedef enum TSQuantifier
@ -132,9 +130,9 @@ typedef enum TSQuantifier
typedef struct TSQueryMatch typedef struct TSQueryMatch
{ {
uint32_t id; t_u32 id;
uint16_t pattern_index; t_u16 pattern_index;
uint16_t capture_count; t_u16 capture_count;
const TSQueryCapture *captures; const TSQueryCapture *captures;
} TSQueryMatch; } TSQueryMatch;
@ -148,7 +146,7 @@ typedef enum TSQueryPredicateStepType
typedef struct TSQueryPredicateStep typedef struct TSQueryPredicateStep
{ {
TSQueryPredicateStepType type; TSQueryPredicateStepType type;
uint32_t value_id; t_u32 value_id;
} TSQueryPredicateStep; } TSQueryPredicateStep;
typedef enum TSQueryError typedef enum TSQueryError
@ -215,7 +213,7 @@ bool ts_parser_set_language(TSParser *self, const TSLanguage *language);
* will not be assigned, and this function will return `false`. On success, * will not be assigned, and this function will return `false`. On success,
* this function returns `true` * this function returns `true`
*/ */
bool ts_parser_set_included_ranges(TSParser *self, const TSRange *ranges, uint32_t count); bool ts_parser_set_included_ranges(TSParser *self, const TSRange *ranges, t_u32 count);
/** /**
* Get the ranges of text that the parser will include when parsing. * Get the ranges of text that the parser will include when parsing.
@ -224,7 +222,7 @@ bool ts_parser_set_included_ranges(TSParser *self, const TSRange *ranges, uint32
* or write to it. The length of the array will be written to the given * or write to it. The length of the array will be written to the given
* `count` pointer. * `count` pointer.
*/ */
const TSRange *ts_parser_included_ranges(const TSParser *self, uint32_t *count); const TSRange *ts_parser_included_ranges(const TSParser *self, t_u32 *count);
/** /**
* Use the parser to parse some source code and create a syntax tree. * Use the parser to parse some source code and create a syntax tree.
@ -277,7 +275,7 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input);
* above. The second two parameters indicate the location of the buffer and its * above. The second two parameters indicate the location of the buffer and its
* length in bytes. * length in bytes.
*/ */
TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, const char *string, uint32_t length); TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, const char *string, t_u32 length);
/** /**
* Use the parser to parse some source code stored in one contiguous buffer with * Use the parser to parse some source code stored in one contiguous buffer with
@ -285,8 +283,7 @@ TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, const cha
* [`ts_parser_parse_string`] method above. The final parameter indicates whether * [`ts_parser_parse_string`] method above. The final parameter indicates whether
* the text is encoded as UTF8 or UTF16. * the text is encoded as UTF8 or UTF16.
*/ */
TSTree *ts_parser_parse_string_encoding(TSParser *self, const TSTree *old_tree, const char *string, uint32_t length, TSTree *ts_parser_parse_string_encoding(TSParser *self, const TSTree *old_tree, const char *string, t_u32 length, TSInputEncoding encoding);
TSInputEncoding encoding);
/** /**
* Instruct the parser to start the next parse from the beginning. * Instruct the parser to start the next parse from the beginning.
@ -306,12 +303,12 @@ void ts_parser_reset(TSParser *self);
* If parsing takes longer than this, it will halt early, returning NULL. * If parsing takes longer than this, it will halt early, returning NULL.
* See [`ts_parser_parse`] for more information. * See [`ts_parser_parse`] for more information.
*/ */
void ts_parser_set_timeout_micros(TSParser *self, uint64_t timeout_micros); void ts_parser_set_timeout_micros(TSParser *self, t_u64 timeout_micros);
/** /**
* Get the duration in microseconds that parsing is allowed to take. * Get the duration in microseconds that parsing is allowed to take.
*/ */
uint64_t ts_parser_timeout_micros(const TSParser *self); t_u64 ts_parser_timeout_micros(const TSParser *self);
/** /**
* Set the parser's current cancellation flag pointer. * Set the parser's current cancellation flag pointer.
@ -375,7 +372,7 @@ TSNode ts_tree_root_node(const TSTree *self);
* Get the root node of the syntax tree, but with its position * Get the root node of the syntax tree, but with its position
* shifted forward by the given offset. * shifted forward by the given offset.
*/ */
TSNode ts_tree_root_node_with_offset(const TSTree *self, uint32_t offset_bytes, TSPoint offset_extent); TSNode ts_tree_root_node_with_offset(const TSTree *self, t_u32 offset_bytes, TSPoint offset_extent);
/** /**
* Get the language that was used to parse the syntax tree. * Get the language that was used to parse the syntax tree.
@ -387,7 +384,7 @@ const TSLanguage *ts_tree_language(const TSTree *self);
* *
* The returned pointer must be freed by the caller. * The returned pointer must be freed by the caller.
*/ */
TSRange *ts_tree_included_ranges(const TSTree *self, uint32_t *length); TSRange *ts_tree_included_ranges(const TSTree *self, t_u32 *length);
/** /**
* Edit the syntax tree to keep it in sync with source code that has been * Edit the syntax tree to keep it in sync with source code that has been
@ -412,7 +409,7 @@ void ts_tree_edit(TSTree *self, const TSInputEdit *edit);
* for freeing it using `free`. The length of the array will be written to the * for freeing it using `free`. The length of the array will be written to the
* given `length` pointer. * given `length` pointer.
*/ */
TSRange *ts_tree_get_changed_ranges(const TSTree *old_tree, const TSTree *new_tree, uint32_t *length); TSRange *ts_tree_get_changed_ranges(const TSTree *old_tree, const TSTree *new_tree, t_u32 *length);
/** /**
* Write a DOT graph describing the syntax tree to the given file. * Write a DOT graph describing the syntax tree to the given file.
@ -454,7 +451,7 @@ TSSymbol ts_node_grammar_symbol(TSNode self);
/** /**
* Get the node's start byte. * Get the node's start byte.
*/ */
uint32_t ts_node_start_byte(TSNode self); t_u32 ts_node_start_byte(TSNode self);
/** /**
* Get the node's start position in terms of rows and columns. * Get the node's start position in terms of rows and columns.
@ -464,7 +461,7 @@ TSPoint ts_node_start_point(TSNode self);
/** /**
* Get the node's end byte. * Get the node's end byte.
*/ */
uint32_t ts_node_end_byte(TSNode self); t_u32 ts_node_end_byte(TSNode self);
/** /**
* Get the node's end position in terms of rows and columns. * Get the node's end position in terms of rows and columns.
@ -546,43 +543,43 @@ TSNode ts_node_child_containing_descendant(TSNode self, TSNode descendant);
* Get the node's child at the given index, where zero represents the first * Get the node's child at the given index, where zero represents the first
* child. * child.
*/ */
TSNode ts_node_child(TSNode self, uint32_t child_index); TSNode ts_node_child(TSNode self, t_u32 child_index);
/** /**
* Get the field name for node's child at the given index, where zero represents * Get the field name for node's child at the given index, where zero represents
* the first child. Returns NULL, if no field is found. * the first child. Returns NULL, if no field is found.
*/ */
const char *ts_node_field_name_for_child(TSNode self, uint32_t child_index); const char *ts_node_field_name_for_child(TSNode self, t_u32 child_index);
/** /**
* Get the field name for node's child at the given index, where zero represents * Get the field name for node's child at the given index, where zero represents
* the first child. Returns NULL, if no field is found. * the first child. Returns NULL, if no field is found.
*/ */
TSFieldId ts_node_field_id_for_child(TSNode self, uint32_t child_index); TSFieldId ts_node_field_id_for_child(TSNode self, t_u32 child_index);
/** /**
* Get the node's number of children. * Get the node's number of children.
*/ */
uint32_t ts_node_child_count(TSNode self); t_u32 ts_node_child_count(TSNode self);
/** /**
* Get the node's *named* child at the given index. * Get the node's *named* child at the given index.
* *
* See also [`ts_node_is_named`]. * See also [`ts_node_is_named`].
*/ */
TSNode ts_node_named_child(TSNode self, uint32_t child_index); TSNode ts_node_named_child(TSNode self, t_u32 child_index);
/** /**
* Get the node's number of *named* children. * Get the node's number of *named* children.
* *
* See also [`ts_node_is_named`]. * See also [`ts_node_is_named`].
*/ */
uint32_t ts_node_named_child_count(TSNode self); t_u32 ts_node_named_child_count(TSNode self);
/** /**
* Get the node's child with the given field name. * Get the node's child with the given field name.
*/ */
TSNode ts_node_child_by_field_name(TSNode self, const char *name, uint32_t name_length); TSNode ts_node_child_by_field_name(TSNode self, const char *name, t_u32 name_length);
/** /**
* Get the node's child with the given numerical field id. * Get the node's child with the given numerical field id.
@ -607,30 +604,30 @@ TSNode ts_node_prev_named_sibling(TSNode self);
/** /**
* Get the node's first child that extends beyond the given byte offset. * Get the node's first child that extends beyond the given byte offset.
*/ */
TSNode ts_node_first_child_for_byte(TSNode self, uint32_t byte); TSNode ts_node_first_child_for_byte(TSNode self, t_u32 byte);
/** /**
* Get the node's first named child that extends beyond the given byte offset. * Get the node's first named child that extends beyond the given byte offset.
*/ */
TSNode ts_node_first_named_child_for_byte(TSNode self, uint32_t byte); TSNode ts_node_first_named_child_for_byte(TSNode self, t_u32 byte);
/** /**
* Get the node's number of descendants, including one for the node itself. * Get the node's number of descendants, including one for the node itself.
*/ */
uint32_t ts_node_descendant_count(TSNode self); t_u32 ts_node_descendant_count(TSNode self);
/** /**
* Get the smallest node within this node that spans the given range of bytes * Get the smallest node within this node that spans the given range of bytes
* or (row, column) positions. * or (row, column) positions.
*/ */
TSNode ts_node_descendant_for_byte_range(TSNode self, uint32_t start, uint32_t end); TSNode ts_node_descendant_for_byte_range(TSNode self, t_u32 start, t_u32 end);
TSNode ts_node_descendant_for_point_range(TSNode self, TSPoint start, TSPoint end); TSNode ts_node_descendant_for_point_range(TSNode self, TSPoint start, TSPoint end);
/** /**
* Get the smallest named node within this node that spans the given range of * Get the smallest named node within this node that spans the given range of
* bytes or (row, column) positions. * bytes or (row, column) positions.
*/ */
TSNode ts_node_named_descendant_for_byte_range(TSNode self, uint32_t start, uint32_t end); TSNode ts_node_named_descendant_for_byte_range(TSNode self, t_u32 start, t_u32 end);
TSNode ts_node_named_descendant_for_point_range(TSNode self, TSPoint start, TSPoint end); TSNode ts_node_named_descendant_for_point_range(TSNode self, TSPoint start, TSPoint end);
/** /**
@ -667,12 +664,12 @@ void ts_language_delete(const TSLanguage *self);
/** /**
* Get the number of distinct node types in the language. * Get the number of distinct node types in the language.
*/ */
uint32_t ts_language_symbol_count(const TSLanguage *self); t_u32 ts_language_symbol_count(const TSLanguage *self);
/** /**
* Get the number of valid states in this language. * Get the number of valid states in this language.
*/ */
uint32_t ts_language_state_count(const TSLanguage *self); t_u32 ts_language_state_count(const TSLanguage *self);
/** /**
* Get a node type string for the given numerical id. * Get a node type string for the given numerical id.
@ -682,12 +679,12 @@ const char *ts_language_symbol_name(const TSLanguage *self, TSSymbol symbol);
/** /**
* Get the numerical id for the given node type string. * Get the numerical id for the given node type string.
*/ */
TSSymbol ts_language_symbol_for_name(const TSLanguage *self, const char *string, uint32_t length, bool is_named); TSSymbol ts_language_symbol_for_name(const TSLanguage *self, const char *string, t_u32 length, bool is_named);
/** /**
* Get the number of distinct field names in the language. * Get the number of distinct field names in the language.
*/ */
uint32_t ts_language_field_count(const TSLanguage *self); t_u32 ts_language_field_count(const TSLanguage *self);
/** /**
* Get the field name string for the given numerical id. * Get the field name string for the given numerical id.
@ -697,7 +694,7 @@ const char *ts_language_field_name_for_id(const TSLanguage *self, TSFieldId id);
/** /**
* Get the numerical id for the given field name string. * Get the numerical id for the given field name string.
*/ */
TSFieldId ts_language_field_id_for_name(const TSLanguage *self, const char *name, uint32_t name_length); TSFieldId ts_language_field_id_for_name(const TSLanguage *self, const char *name, t_u32 name_length);
/** /**
* Check whether the given node type id belongs to named nodes, anonymous nodes, * Check whether the given node type id belongs to named nodes, anonymous nodes,
@ -714,7 +711,7 @@ TSSymbolType ts_language_symbol_type(const TSLanguage *self, TSSymbol symbol);
* *
* See also [`ts_parser_set_language`]. * See also [`ts_parser_set_language`].
*/ */
uint32_t ts_language_version(const TSLanguage *self); t_u32 ts_language_version(const TSLanguage *self);
/** /**
* Get the next parse state. Combine this with lookahead iterators to generate * Get the next parse state. Combine this with lookahead iterators to generate

35
parser/src/array.h
View file

@ -1,9 +1,9 @@
#ifndef TREE_SITTER_ARRAY_H_ #ifndef TREE_SITTER_ARRAY_H_
#define TREE_SITTER_ARRAY_H_ #define TREE_SITTER_ARRAY_H_
#include "me/types.h"
#include <assert.h> #include <assert.h>
#include <stdbool.h> #include <stdbool.h>
#include <stdint.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
@ -12,9 +12,9 @@
#define Array(T) \ #define Array(T) \
struct \ struct \
{ \ { \
T *contents; \ T *contents; \
uint32_t size; \ t_u32 size; \
uint32_t capacity; \ t_u32 capacity; \
} }
/// Initialize an array. /// Initialize an array.
@ -27,7 +27,7 @@
} }
/// Get a pointer to the element at a given `index` in the array. /// Get a pointer to the element at a given `index` in the array.
#define array_get(self, _index) (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index]) #define array_get(self, _index) (assert((t_u32)(_index) < (self)->size), &(self)->contents[_index])
/// Get a pointer to the first element in the array. /// Get a pointer to the first element in the array.
#define array_front(self) array_get(self, 0) #define array_front(self) array_get(self, 0)
@ -151,7 +151,7 @@ static inline void _array__delete(Array *self)
} }
/// This is not what you're looking for, see `array_erase`. /// This is not what you're looking for, see `array_erase`.
static inline void _array__erase(Array *self, size_t element_size, uint32_t index) static inline void _array__erase(Array *self, size_t element_size, t_u32 index)
{ {
assert(index < self->size); assert(index < self->size);
char *contents = (char *)self->contents; char *contents = (char *)self->contents;
@ -160,7 +160,7 @@ static inline void _array__erase(Array *self, size_t element_size, uint32_t inde
} }
/// This is not what you're looking for, see `array_reserve`. /// This is not what you're looking for, see `array_reserve`.
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) static inline void _array__reserve(Array *self, size_t element_size, t_u32 new_capacity)
{ {
if (new_capacity > self->capacity) if (new_capacity > self->capacity)
{ {
@ -193,12 +193,12 @@ static inline void _array__swap(Array *self, Array *other)
} }
/// This is not what you're looking for, see `array_push` or `array_grow_by`. /// This is not what you're looking for, see `array_push` or `array_grow_by`.
static inline void _array__grow(Array *self, uint32_t count, size_t element_size) static inline void _array__grow(Array *self, t_u32 count, size_t element_size)
{ {
uint32_t new_size = self->size + count; t_u32 new_size = self->size + count;
if (new_size > self->capacity) if (new_size > self->capacity)
{ {
uint32_t new_capacity = self->capacity * 2; t_u32 new_capacity = self->capacity * 2;
if (new_capacity < 8) if (new_capacity < 8)
new_capacity = 8; new_capacity = 8;
if (new_capacity < new_size) if (new_capacity < new_size)
@ -208,12 +208,11 @@ static inline void _array__grow(Array *self, uint32_t count, size_t element_size
} }
/// This is not what you're looking for, see `array_splice`. /// This is not what you're looking for, see `array_splice`.
static inline void _array__splice(Array *self, size_t element_size, uint32_t index, uint32_t old_count, uint32_t new_count, static inline void _array__splice(Array *self, size_t element_size, t_u32 index, t_u32 old_count, t_u32 new_count, const void *elements)
const void *elements)
{ {
uint32_t new_size = self->size + new_count - old_count; t_u32 new_size = self->size + new_count - old_count;
uint32_t old_end = index + old_count; t_u32 old_end = index + old_count;
uint32_t new_end = index + new_count; t_u32 new_end = index + new_count;
assert(old_end <= self->size); assert(old_end <= self->size);
_array__reserve(self, element_size, new_size); _array__reserve(self, element_size, new_size);
@ -244,14 +243,14 @@ static inline void _array__splice(Array *self, size_t element_size, uint32_t ind
{ \ { \
*(_index) = start; \ *(_index) = start; \
*(_exists) = false; \ *(_exists) = false; \
uint32_t size = (self)->size - *(_index); \ t_u32 size = (self)->size - *(_index); \
if (size == 0) \ if (size == 0) \
break; \ break; \
int comparison; \ int comparison; \
while (size > 1) \ while (size > 1) \
{ \ { \
uint32_t half_size = size / 2; \ t_u32 half_size = size / 2; \
uint32_t mid_index = *(_index) + half_size; \ t_u32 mid_index = *(_index) + half_size; \
comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \ comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
if (comparison <= 0) \ if (comparison <= 0) \
*(_index) = mid_index; \ *(_index) = mid_index; \

48
parser/src/create_language.c
View file

@ -13,33 +13,33 @@
#include "../static/headers/constants.h" #include "../static/headers/constants.h"
#include "../static/headers/symbols.h" #include "../static/headers/symbols.h"
#include "./parser.h" #include "./parser.h"
#include "me/types.h"
// bool lex_keywords_main(TSLexer *lexer, TSStateId state); // bool lex_keywords_main(TSLexer *lexer, TSStateId state);
// bool lex_normal_main(TSLexer *lexer, TSStateId state); // bool lex_normal_main(TSLexer *lexer, TSStateId state);
bool tree_sitter_sh_external_scanner_scan(void *ctx, TSLexer *lexer, const bool *ret); bool tree_sitter_sh_external_scanner_scan(void *ctx, TSLexer *lexer, const bool *ret);
void *create_external_scanner_states(void); void *create_external_scanner_states(void);
void *create_field_names(void); void *create_field_names(void);
void *create_symbols_names(void); void *create_symbols_names(void);
void *create_field_map_entries(void); void *create_field_map_entries(void);
void *create_field_map_slices(void); void *create_field_map_slices(void);
void *create_lex_modes(void); void *create_lex_modes(void);
void *create_parse_actions_entries(void); void *create_parse_actions_entries(void);
void *create_primary_state_ids(void); void *create_primary_state_ids(void);
void *create_alias_sequences(void); void *create_alias_sequences(void);
void *create_external_scanner_symbol_map(void); void *create_external_scanner_symbol_map(void);
void *create_non_terminal_alias_map(void); void *create_non_terminal_alias_map(void);
void *create_unique_symbols_map(void); void *create_unique_symbols_map(void);
void *create_symbols_metadata(void); void *create_symbols_metadata(void);
void *create_parse_table(void); void *create_parse_table(void);
void *create_small_parse_table(void); void *create_small_parse_table(void);
void *create_small_parse_table_map(void); void *create_small_parse_table_map(void);
bool ts_lex_keywords(TSLexer *lexer, TSStateId state); bool ts_lex_keywords(TSLexer *lexer, TSStateId state);
bool ts_lex_keywords(TSLexer *lexer, TSStateId state); bool ts_lex(TSLexer *lexer, TSStateId state);
t_u32 tree_sitter_sh_external_scanner_serialize(void *ctx, char *s);
uint32_t tree_sitter_sh_external_scanner_serialize(void *ctx, char *s); void tree_sitter_sh_external_scanner_deserialize(void *ctx, const char *s, t_u32 val);
void tree_sitter_sh_external_scanner_deserialize(void *ctx, const char *s, uint32_t val); void tree_sitter_sh_external_scanner_destroy(void *ctx);
void tree_sitter_sh_external_scanner_destroy(void *ctx); void *tree_sitter_sh_external_scanner_create(void);
void *tree_sitter_sh_external_scanner_create(void);
static struct ExternalScannerDefinition init_scanner(void) static struct ExternalScannerDefinition init_scanner(void)
{ {

6
parser/src/error_costs.h
View file

@ -1,6 +0,0 @@
#ifndef TREE_SITTER_ERROR_COSTS_H_
#define TREE_SITTER_ERROR_COSTS_H_
#endif

297
parser/src/language.c
View file

@ -1,173 +1,194 @@
#include "./language.h" #include "./language.h"
#include "./api.h" #include "./api.h"
#include <string.h> #include "me/types.h"
#include <assert.h> #include <assert.h>
#include <string.h>
const TSLanguage *ts_language_copy(const TSLanguage *self) { const TSLanguage *ts_language_copy(const TSLanguage *self)
return self; {
return self;
} }
void ts_language_delete(const TSLanguage *self) { void ts_language_delete(const TSLanguage *self)
(void)(self); {
(void)(self);
} }
uint32_t ts_language_symbol_count(const TSLanguage *self) { t_u32 ts_language_symbol_count(const TSLanguage *self)
return self->symbol_count + self->alias_count; {
return self->symbol_count + self->alias_count;
} }
uint32_t ts_language_state_count(const TSLanguage *self) { t_u32 ts_language_state_count(const TSLanguage *self)
return self->state_count; {
return self->state_count;
} }
uint32_t ts_language_version(const TSLanguage *self) { t_u32 ts_language_version(const TSLanguage *self)
return self->version; {
return self->version;
} }
uint32_t ts_language_field_count(const TSLanguage *self) { t_u32 ts_language_field_count(const TSLanguage *self)
return self->field_count; {
return self->field_count;
} }
void ts_language_table_entry( void ts_language_table_entry(const TSLanguage *self, TSStateId state, TSSymbol symbol, TableEntry *result)
const TSLanguage *self, {
TSStateId state, if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat)
TSSymbol symbol, {
TableEntry *result result->action_count = 0;
) { result->is_reusable = false;
if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { result->actions = NULL;
result->action_count = 0; }
result->is_reusable = false; else
result->actions = NULL; {
} else { assert(symbol < self->token_count);
assert(symbol < self->token_count); t_u32 action_index = ts_language_lookup(self, state, symbol);
uint32_t action_index = ts_language_lookup(self, state, symbol); const TSParseActionEntry *entry = &self->parse_actions[action_index];
const TSParseActionEntry *entry = &self->parse_actions[action_index]; result->action_count = entry->entry.count;
result->action_count = entry->entry.count; result->is_reusable = entry->entry.reusable;
result->is_reusable = entry->entry.reusable; result->actions = (const TSParseAction *)(entry + 1);
result->actions = (const TSParseAction *)(entry + 1); }
}
} }
TSSymbolMetadata ts_language_symbol_metadata( TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *self, TSSymbol symbol)
const TSLanguage *self, {
TSSymbol symbol if (symbol == ts_builtin_sym_error)
) { {
if (symbol == ts_builtin_sym_error) { return (TSSymbolMetadata){.visible = true, .named = true};
return (TSSymbolMetadata) {.visible = true, .named = true}; }
} else if (symbol == ts_builtin_sym_error_repeat) { else if (symbol == ts_builtin_sym_error_repeat)
return (TSSymbolMetadata) {.visible = false, .named = false}; {
} else { return (TSSymbolMetadata){.visible = false, .named = false};
return self->symbol_metadata[symbol]; }
} else
{
return self->symbol_metadata[symbol];
}
} }
TSSymbol ts_language_public_symbol( TSSymbol ts_language_public_symbol(const TSLanguage *self, TSSymbol symbol)
const TSLanguage *self, {
TSSymbol symbol if (symbol == ts_builtin_sym_error)
) { return symbol;
if (symbol == ts_builtin_sym_error) return symbol; return self->public_symbol_map[symbol];
return self->public_symbol_map[symbol];
} }
TSStateId ts_language_next_state( TSStateId ts_language_next_state(const TSLanguage *self, TSStateId state, TSSymbol symbol)
const TSLanguage *self, {
TSStateId state, if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat)
TSSymbol symbol {
) { return 0;
if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { }
return 0; else if (symbol < self->token_count)
} else if (symbol < self->token_count) { {
uint32_t count; t_u32 count;
const TSParseAction *actions = ts_language_actions(self, state, symbol, &count); const TSParseAction *actions = ts_language_actions(self, state, symbol, &count);
if (count > 0) { if (count > 0)
TSParseAction action = actions[count - 1]; {
if (action.type == TSParseActionTypeShift) { TSParseAction action = actions[count - 1];
return action.shift.extra ? state : action.shift.state; if (action.type == TSParseActionTypeShift)
} {
} return action.shift.extra ? state : action.shift.state;
return 0; }
} else { }
return ts_language_lookup(self, state, symbol); return 0;
} }
else
{
return ts_language_lookup(self, state, symbol);
}
} }
const char *ts_language_symbol_name( const char *ts_language_symbol_name(const TSLanguage *self, TSSymbol symbol)
const TSLanguage *self, {
TSSymbol symbol if (symbol == ts_builtin_sym_error)
) { {
if (symbol == ts_builtin_sym_error) { return "ERROR";
return "ERROR"; }
} else if (symbol == ts_builtin_sym_error_repeat) { else if (symbol == ts_builtin_sym_error_repeat)
return "_ERROR"; {
} else if (symbol < ts_language_symbol_count(self)) { return "_ERROR";
return self->symbol_names[symbol]; }
} else { else if (symbol < ts_language_symbol_count(self))
return NULL; {
} return self->symbol_names[symbol];
}
else
{
return NULL;
}
} }
TSSymbol ts_language_symbol_for_name( TSSymbol ts_language_symbol_for_name(const TSLanguage *self, const char *string, t_u32 length, bool is_named)
const TSLanguage *self, {
const char *string, if (!strncmp(string, "ERROR", length))
uint32_t length, return ts_builtin_sym_error;
bool is_named t_u16 count = (t_u16)ts_language_symbol_count(self);
) { for (TSSymbol i = 0; i < count; i++)
if (!strncmp(string, "ERROR", length)) return ts_builtin_sym_error; {
uint16_t count = (uint16_t)ts_language_symbol_count(self); TSSymbolMetadata metadata = ts_language_symbol_metadata(self, i);
for (TSSymbol i = 0; i < count; i++) { if ((!metadata.visible && !metadata.supertype) || metadata.named != is_named)
TSSymbolMetadata metadata = ts_language_symbol_metadata(self, i); continue;
if ((!metadata.visible && !metadata.supertype) || metadata.named != is_named) continue; const char *symbol_name = self->symbol_names[i];
const char *symbol_name = self->symbol_names[i]; if (!strncmp(symbol_name, string, length) && !symbol_name[length])
if (!strncmp(symbol_name, string, length) && !symbol_name[length]) { {
return self->public_symbol_map[i]; return self->public_symbol_map[i];
} }
} }
return 0; return 0;
} }
TSSymbolType ts_language_symbol_type( TSSymbolType ts_language_symbol_type(const TSLanguage *self, TSSymbol symbol)
const TSLanguage *self, {
TSSymbol symbol TSSymbolMetadata metadata = ts_language_symbol_metadata(self, symbol);
) { if (metadata.named && metadata.visible)
TSSymbolMetadata metadata = ts_language_symbol_metadata(self, symbol); {
if (metadata.named && metadata.visible) { return TSSymbolTypeRegular;
return TSSymbolTypeRegular; }
} else if (metadata.visible) { else if (metadata.visible)
return TSSymbolTypeAnonymous; {
} else { return TSSymbolTypeAnonymous;
return TSSymbolTypeAuxiliary; }
} else
{
return TSSymbolTypeAuxiliary;
}
} }
const char *ts_language_field_name_for_id( const char *ts_language_field_name_for_id(const TSLanguage *self, TSFieldId id)
const TSLanguage *self, {
TSFieldId id t_u32 count = ts_language_field_count(self);
) { if (count && id <= count)
uint32_t count = ts_language_field_count(self); {
if (count && id <= count) { return self->field_names[id];
return self->field_names[id]; }
} else { else
return NULL; {
} return NULL;
}
} }
TSFieldId ts_language_field_id_for_name( TSFieldId ts_language_field_id_for_name(const TSLanguage *self, const char *name, t_u32 name_length)
const TSLanguage *self, {
const char *name, t_u16 count = (t_u16)ts_language_field_count(self);
uint32_t name_length for (TSSymbol i = 1; i < count + 1; i++)
) { {
uint16_t count = (uint16_t)ts_language_field_count(self); switch (strncmp(name, self->field_names[i], name_length))
for (TSSymbol i = 1; i < count + 1; i++) { {
switch (strncmp(name, self->field_names[i], name_length)) { case 0:
case 0: if (self->field_names[i][name_length] == 0)
if (self->field_names[i][name_length] == 0) return i; return i;
break; break;
case -1: case -1:
return 0; return 0;
default: default:
break; break;
} }
} }
return 0; return 0;
} }
// TSLookaheadIterator *ts_lookahead_iterator_new(const TSLanguage *self, TSStateId state) { // TSLookaheadIterator *ts_lookahead_iterator_new(const TSLanguage *self, TSStateId state) {

26
parser/src/language.h
View file

@ -2,6 +2,7 @@
#define TREE_SITTER_LANGUAGE_H_ #define TREE_SITTER_LANGUAGE_H_
#include "./parser.h" #include "./parser.h"
#include "me/types.h"
#define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1) #define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1)
@ -11,7 +12,7 @@
typedef struct TableEntry typedef struct TableEntry
{ {
const TSParseAction *actions; const TSParseAction *actions;
uint32_t action_count; t_u32 action_count;
bool is_reusable; bool is_reusable;
} TableEntry; } TableEntry;
@ -28,7 +29,7 @@ static inline bool ts_language_is_symbol_external(const TSLanguage *self, TSSymb
return 0 < symbol && symbol < self->external_token_count + 1; return 0 < symbol && symbol < self->external_token_count + 1;
} }
static inline const TSParseAction *ts_language_actions(const TSLanguage *self, TSStateId state, TSSymbol symbol, uint32_t *count) static inline const TSParseAction *ts_language_actions(const TSLanguage *self, TSStateId state, TSSymbol symbol, t_u32 *count)
{ {
TableEntry entry; TableEntry entry;
ts_language_table_entry(self, state, symbol, &entry); ts_language_table_entry(self, state, symbol, &entry);
@ -50,17 +51,17 @@ static inline bool ts_language_has_reduce_action(const TSLanguage *self, TSState
// For 'large' parse states, this is a direct lookup. For 'small' parse // For 'large' parse states, this is a direct lookup. For 'small' parse
// states, this requires searching through the symbol groups to find // states, this requires searching through the symbol groups to find
// the given symbol. // the given symbol.
static inline uint16_t ts_language_lookup(const TSLanguage *self, TSStateId state, TSSymbol symbol) static inline t_u16 ts_language_lookup(const TSLanguage *self, TSStateId state, TSSymbol symbol)
{ {
if (state >= self->large_state_count) if (state >= self->large_state_count)
{ {
uint32_t index = self->small_parse_table_map[state - self->large_state_count]; t_u32 index = self->small_parse_table_map[state - self->large_state_count];
const uint16_t *data = &self->small_parse_table[index]; const t_u16 *data = &self->small_parse_table[index];
uint16_t group_count = *(data++); t_u16 group_count = *(data++);
for (unsigned i = 0; i < group_count; i++) for (unsigned i = 0; i < group_count; i++)
{ {
uint16_t section_value = *(data++); t_u16 section_value = *(data++);
uint16_t symbol_count = *(data++); t_u16 symbol_count = *(data++);
for (unsigned j = 0; j < symbol_count; j++) for (unsigned j = 0; j < symbol_count; j++)
{ {
if (*(data++) == symbol) if (*(data++) == symbol)
@ -92,17 +93,17 @@ static inline const bool *ts_language_enabled_external_tokens(const TSLanguage *
} }
} }
static inline const TSSymbol *ts_language_alias_sequence(const TSLanguage *self, uint32_t production_id) static inline const TSSymbol *ts_language_alias_sequence(const TSLanguage *self, t_u32 production_id)
{ {
return production_id ? &self->alias_sequences[production_id * self->max_alias_sequence_length] : NULL; return production_id ? &self->alias_sequences[production_id * self->max_alias_sequence_length] : NULL;
} }
static inline TSSymbol ts_language_alias_at(const TSLanguage *self, uint32_t production_id, uint32_t child_index) static inline TSSymbol ts_language_alias_at(const TSLanguage *self, t_u32 production_id, t_u32 child_index)
{ {
return production_id ? self->alias_sequences[production_id * self->max_alias_sequence_length + child_index] : 0; return production_id ? self->alias_sequences[production_id * self->max_alias_sequence_length + child_index] : 0;
} }
static inline void ts_language_field_map(const TSLanguage *self, uint32_t production_id, const TSFieldMapEntry **start, static inline void ts_language_field_map(const TSLanguage *self, t_u32 production_id, const TSFieldMapEntry **start,
const TSFieldMapEntry **end) const TSFieldMapEntry **end)
{ {
if (self->field_count == 0) if (self->field_count == 0)
@ -129,7 +130,7 @@ static inline void ts_language_aliases_for_symbol(const TSLanguage *self, TSSymb
TSSymbol symbol = self->alias_map[idx++]; TSSymbol symbol = self->alias_map[idx++];
if (symbol == 0 || symbol > original_symbol) if (symbol == 0 || symbol > original_symbol)
break; break;
uint16_t count = self->alias_map[idx++]; t_u16 count = self->alias_map[idx++];
if (symbol == original_symbol) if (symbol == original_symbol)
{ {
*start = &self->alias_map[idx]; *start = &self->alias_map[idx];
@ -140,5 +141,4 @@ static inline void ts_language_aliases_for_symbol(const TSLanguage *self, TSSymb
} }
} }
#endif // TREE_SITTER_LANGUAGE_H_ #endif // TREE_SITTER_LANGUAGE_H_

66
parser/src/length.h
View file

@ -1,51 +1,61 @@
#ifndef TREE_SITTER_LENGTH_H_ #ifndef TREE_SITTER_LENGTH_H_
#define TREE_SITTER_LENGTH_H_ #define TREE_SITTER_LENGTH_H_
#include <stdbool.h>
#include "./point.h"
#include "./api.h" #include "./api.h"
#include "./point.h"
#include "me/types.h"
typedef struct Length { typedef struct Length
uint32_t bytes; {
TSPoint extent; t_u32 bytes;
TSPoint extent;
} Length; } Length;
static const Length LENGTH_UNDEFINED = {0, {0, 1}}; static const Length LENGTH_UNDEFINED = {0, {0, 1}};
static const Length LENGTH_MAX = {UINT32_MAX, {UINT32_MAX, UINT32_MAX}}; static const Length LENGTH_MAX = {UINT32_MAX, {UINT32_MAX, UINT32_MAX}};
static inline bool length_is_undefined(Length length) { static inline bool length_is_undefined(Length length)
return length.bytes == 0 && length.extent.column != 0; {
return length.bytes == 0 && length.extent.column != 0;
} }
static inline Length length_min(Length len1, Length len2) { static inline Length length_min(Length len1, Length len2)
return (len1.bytes < len2.bytes) ? len1 : len2; {
return (len1.bytes < len2.bytes) ? len1 : len2;
} }
static inline Length length_add(Length len1, Length len2) { static inline Length length_add(Length len1, Length len2)
Length result; {
result.bytes = len1.bytes + len2.bytes; Length result;
result.extent = point_add(len1.extent, len2.extent); result.bytes = len1.bytes + len2.bytes;
return result; result.extent = point_add(len1.extent, len2.extent);
return result;
} }
static inline Length length_sub(Length len1, Length len2) { static inline Length length_sub(Length len1, Length len2)
Length result; {
result.bytes = len1.bytes - len2.bytes; Length result;
result.extent = point_sub(len1.extent, len2.extent); result.bytes = len1.bytes - len2.bytes;
return result; result.extent = point_sub(len1.extent, len2.extent);
return result;
} }
static inline Length length_zero(void) { static inline Length length_zero(void)
Length result = {0, {0, 0}}; {
return result; Length result = {0, {0, 0}};
return result;
} }
static inline Length length_saturating_sub(Length len1, Length len2) { static inline Length length_saturating_sub(Length len1, Length len2)
if (len1.bytes > len2.bytes) { {
return length_sub(len1, len2); if (len1.bytes > len2.bytes)
} else { {
return length_zero(); return length_sub(len1, len2);
} }
else
{
return length_zero();
}
} }
#endif #endif

675
parser/src/lexer.c
View file

@ -1,419 +1,442 @@
#include <stdio.h>
#include "./lexer.h" #include "./lexer.h"
#include "./subtree.h"
#include "./length.h" #include "./length.h"
#include "./unicode.h" #include "./unicode.h"
#include "me/mem/mem.h"
#include "me/types.h"
#include <string.h>
#define LOG(message, character) \ #define LOG(...)
if (self->logger.log) { \
snprintf( \
self->debug_buffer, \
TREE_SITTER_SERIALIZATION_BUFFER_SIZE, \
32 <= character && character < 127 ? \
message " character:'%c'" : \
message " character:%d", \
character \
); \
self->logger.log( \
self->logger.payload, \
TSLogTypeLex, \
self->debug_buffer \
); \
}
static const int32_t BYTE_ORDER_MARK = 0xFEFF; static const t_i32 BYTE_ORDER_MARK = 0xFEFF;
static const TSRange DEFAULT_RANGE = { static const TSRange DEFAULT_RANGE = {.start_point =
.start_point = { {
.row = 0, .row = 0,
.column = 0, .column = 0,
}, },
.end_point = { .end_point =
.row = UINT32_MAX, {
.column = UINT32_MAX, .row = UINT32_MAX,
}, .column = UINT32_MAX,
.start_byte = 0, },
.end_byte = UINT32_MAX .start_byte = 0,
}; .end_byte = UINT32_MAX};
// Check if the lexer has reached EOF. This state is stored // Check if the lexer has reached EOF. This state is stored
// by setting the lexer's `current_included_range_index` such that // by setting the lexer's `current_included_range_index` such that
// it has consumed all of its available ranges. // it has consumed all of its available ranges.
static bool ts_lexer__eof(const TSLexer *_self) { static bool ts_lexer__eof(const TSLexer *_self)
Lexer *self = (Lexer *)_self; {
return self->current_included_range_index == self->included_range_count; Lexer *self = (Lexer *)_self;
return self->current_included_range_index == self->included_range_count;
} }
// Clear the currently stored chunk of source code, because the lexer's // Clear the currently stored chunk of source code, because the lexer's
// position has changed. // position has changed.
static void ts_lexer__clear_chunk(Lexer *self) { static void ts_lexer__clear_chunk(Lexer *self)
self->chunk = NULL; {
self->chunk_size = 0; self->chunk = NULL;
self->chunk_start = 0; self->chunk_size = 0;
self->chunk_start = 0;
} }
// Call the lexer's input callback to obtain a new chunk of source code // Call the lexer's input callback to obtain a new chunk of source code
// for the current position. // for the current position.
static void ts_lexer__get_chunk(Lexer *self) { static void ts_lexer__get_chunk(Lexer *self)
self->chunk_start = self->current_position.bytes; {
self->chunk = self->input.read( self->chunk_start = self->current_position.bytes;
self->input.payload, self->chunk = self->input.read(self->input.payload, self->current_position.bytes, self->current_position.extent, &self->chunk_size);
self->current_position.bytes, if (!self->chunk_size)
self->current_position.extent, {
&self->chunk_size self->current_included_range_index = self->included_range_count;
); self->chunk = NULL;
if (!self->chunk_size) { }
self->current_included_range_index = self->included_range_count;
self->chunk = NULL;
}
} }
// Decode the next unicode character in the current chunk of source code. // Decode the next unicode character in the current chunk of source code.
// This assumes that the lexer has already retrieved a chunk of source // This assumes that the lexer has already retrieved a chunk of source
// code that spans the current position. // code that spans the current position.
static void ts_lexer__get_lookahead(Lexer *self) { static void ts_lexer__get_lookahead(Lexer *self)
uint32_t position_in_chunk = self->current_position.bytes - self->chunk_start; {
uint32_t size = self->chunk_size - position_in_chunk; t_u32 position_in_chunk = self->current_position.bytes - self->chunk_start;
t_u32 size = self->chunk_size - position_in_chunk;
if (size == 0) { if (size == 0)
self->lookahead_size = 1; {
self->data.lookahead = '\0'; self->lookahead_size = 1;
return; self->data.lookahead = '\0';
} return;
}
const uint8_t *chunk = (const uint8_t *)self->chunk + position_in_chunk; const t_u8 *chunk = (const t_u8 *)self->chunk + position_in_chunk;
UnicodeDecodeFunction decode = self->input.encoding == TSInputEncodingUTF8 UnicodeDecodeFunction decode = self->input.encoding == TSInputEncodingUTF8 ? ts_decode_utf8 : ts_decode_utf16;
? ts_decode_utf8
: ts_decode_utf16;
self->lookahead_size = decode(chunk, size, &self->data.lookahead); self->lookahead_size = decode(chunk, size, &self->data.lookahead);
// If this chunk ended in the middle of a multi-byte character, // If this chunk ended in the middle of a multi-byte character,
// try again with a fresh chunk. // try again with a fresh chunk.
if (self->data.lookahead == TS_DECODE_ERROR && size < 4) { if (self->data.lookahead == TS_DECODE_ERROR && size < 4)
ts_lexer__get_chunk(self); {
chunk = (const uint8_t *)self->chunk; ts_lexer__get_chunk(self);
size = self->chunk_size; chunk = (const t_u8 *)self->chunk;
self->lookahead_size = decode(chunk, size, &self->data.lookahead); size = self->chunk_size;
} self->lookahead_size = decode(chunk, size, &self->data.lookahead);
}
if (self->data.lookahead == TS_DECODE_ERROR) { if (self->data.lookahead == TS_DECODE_ERROR)
self->lookahead_size = 1; {
} self->lookahead_size = 1;
}
} }
static void ts_lexer_goto(Lexer *self, Length position) { static void ts_lexer_goto(Lexer *self, Length position)
self->current_position = position; {
self->current_position = position;
// Move to the first valid position at or after the given position. // Move to the first valid position at or after the given position.
bool found_included_range = false; bool found_included_range = false;
for (unsigned i = 0; i < self->included_range_count; i++) { for (unsigned i = 0; i < self->included_range_count; i++)
TSRange *included_range = &self->included_ranges[i]; {
if ( TSRange *included_range = &self->included_ranges[i];
included_range->end_byte > self->current_position.bytes && if (included_range->end_byte > self->current_position.bytes && included_range->end_byte > included_range->start_byte)
included_range->end_byte > included_range->start_byte {
) { if (included_range->start_byte >= self->current_position.bytes)
if (included_range->start_byte >= self->current_position.bytes) { {
self->current_position = (Length) { self->current_position = (Length){
.bytes = included_range->start_byte, .bytes = included_range->start_byte,
.extent = included_range->start_point, .extent = included_range->start_point,
}; };
} }
self->current_included_range_index = i; self->current_included_range_index = i;
found_included_range = true; found_included_range = true;
break; break;
} }
} }
if (found_included_range) { if (found_included_range)
// If the current position is outside of the current chunk of text, {
// then clear out the current chunk of text. // If the current position is outside of the current chunk of text,
if (self->chunk && ( // then clear out the current chunk of text.
self->current_position.bytes < self->chunk_start || if (self->chunk &&
self->current_position.bytes >= self->chunk_start + self->chunk_size (self->current_position.bytes < self->chunk_start || self->current_position.bytes >= self->chunk_start + self->chunk_size))
)) { {
ts_lexer__clear_chunk(self); ts_lexer__clear_chunk(self);
} }
self->lookahead_size = 0; self->lookahead_size = 0;
self->data.lookahead = '\0'; self->data.lookahead = '\0';
} }
// If the given position is beyond any of included ranges, move to the EOF // If the given position is beyond any of included ranges, move to the EOF
// state - past the end of the included ranges. // state - past the end of the included ranges.
else { else
self->current_included_range_index = self->included_range_count; {
TSRange *last_included_range = &self->included_ranges[self->included_range_count - 1]; self->current_included_range_index = self->included_range_count;
self->current_position = (Length) { TSRange *last_included_range = &self->included_ranges[self->included_range_count - 1];
.bytes = last_included_range->end_byte, self->current_position = (Length){
.extent = last_included_range->end_point, .bytes = last_included_range->end_byte,
}; .extent = last_included_range->end_point,
ts_lexer__clear_chunk(self); };
self->lookahead_size = 1; ts_lexer__clear_chunk(self);
self->data.lookahead = '\0'; self->lookahead_size = 1;
} self->data.lookahead = '\0';
}
} }
// Intended to be called only from functions that control logging. // Intended to be called only from functions that control logging.
static void ts_lexer__do_advance(Lexer *self, bool skip) { static void ts_lexer__do_advance(Lexer *self, bool skip)
if (self->lookahead_size) { {
self->current_position.bytes += self->lookahead_size; if (self->lookahead_size)
if (self->data.lookahead == '\n') { {
self->current_position.extent.row++; self->current_position.bytes += self->lookahead_size;
self->current_position.extent.column = 0; if (self->data.lookahead == '\n')
} else { {
self->current_position.extent.column += self->lookahead_size; self->current_position.extent.row++;
} self->current_position.extent.column = 0;
} }
else
{
self->current_position.extent.column += self->lookahead_size;
}
}
const TSRange *current_range = &self->included_ranges[self->current_included_range_index]; const TSRange *current_range = &self->included_ranges[self->current_included_range_index];
while ( while (self->current_position.bytes >= current_range->end_byte || current_range->end_byte == current_range->start_byte)
self->current_position.bytes >= current_range->end_byte || {
current_range->end_byte == current_range->start_byte if (self->current_included_range_index < self->included_range_count)
) { {
if (self->current_included_range_index < self->included_range_count) { self->current_included_range_index++;
self->current_included_range_index++; }
} if (self->current_included_range_index < self->included_range_count)
if (self->current_included_range_index < self->included_range_count) { {
current_range++; current_range++;
self->current_position = (Length) { self->current_position = (Length){
current_range->start_byte, current_range->start_byte,
current_range->start_point, current_range->start_point,
}; };
} else { }
current_range = NULL; else
break; {
} current_range = NULL;
} break;
}
}
if (skip) self->token_start_position = self->current_position; if (skip)
self->token_start_position = self->current_position;
if (current_range) { if (current_range)
if ( {
self->current_position.bytes < self->chunk_start || if (self->current_position.bytes < self->chunk_start || self->current_position.bytes >= self->chunk_start + self->chunk_size)
self->current_position.bytes >= self->chunk_start + self->chunk_size {
) { ts_lexer__get_chunk(self);
ts_lexer__get_chunk(self); }
} ts_lexer__get_lookahead(self);
ts_lexer__get_lookahead(self); }
} else { else
ts_lexer__clear_chunk(self); {
self->data.lookahead = '\0'; ts_lexer__clear_chunk(self);
self->lookahead_size = 1; self->data.lookahead = '\0';
} self->lookahead_size = 1;
}
} }
// Advance to the next character in the source code, retrieving a new // Advance to the next character in the source code, retrieving a new
// chunk of source code if needed. // chunk of source code if needed.
static void ts_lexer__advance(TSLexer *_self, bool skip) { static void ts_lexer__advance(TSLexer *_self, bool skip)
Lexer *self = (Lexer *)_self; {
if (!self->chunk) return; Lexer *self = (Lexer *)_self;
if (!self->chunk)
return;
if (skip) { if (skip)
LOG("skip", self->data.lookahead) {
} else { LOG("skip", self->data.lookahead)
LOG("consume", self->data.lookahead) }
} else
{
LOG("consume", self->data.lookahead)
}
ts_lexer__do_advance(self, skip); ts_lexer__do_advance(self, skip);
} }
// Mark that a token match has completed. This can be called multiple // Mark that a token match has completed. This can be called multiple
// times if a longer match is found later. // times if a longer match is found later.
static void ts_lexer__mark_end(TSLexer *_self) { static void ts_lexer__mark_end(TSLexer *_self)
Lexer *self = (Lexer *)_self; {
if (!ts_lexer__eof(&self->data)) { Lexer *self = (Lexer *)_self;
// If the lexer is right at the beginning of included range, if (!ts_lexer__eof(&self->data))
// then the token should be considered to end at the *end* of the {
// previous included range, rather than here. // If the lexer is right at the beginning of included range,
TSRange *current_included_range = &self->included_ranges[ // then the token should be considered to end at the *end* of the
self->current_included_range_index // previous included range, rather than here.
]; TSRange *current_included_range = &self->included_ranges[self->current_included_range_index];
if ( if (self->current_included_range_index > 0 && self->current_position.bytes == current_included_range->start_byte)
self->current_included_range_index > 0 && {
self->current_position.bytes == current_included_range->start_byte TSRange *previous_included_range = current_included_range - 1;
) { self->token_end_position = (Length){
TSRange *previous_included_range = current_included_range - 1; previous_included_range->end_byte,
self->token_end_position = (Length) { previous_included_range->end_point,
previous_included_range->end_byte, };
previous_included_range->end_point, return;
}; }
return; }
} self->token_end_position = self->current_position;
}
self->token_end_position = self->current_position;
} }
static uint32_t ts_lexer__get_column(TSLexer *_self) { static t_u32 ts_lexer__get_column(TSLexer *_self)
Lexer *self = (Lexer *)_self; {
Lexer *self = (Lexer *)_self;
uint32_t goal_byte = self->current_position.bytes; t_u32 goal_byte = self->current_position.bytes;
self->did_get_column = true; self->did_get_column = true;
self->current_position.bytes -= self->current_position.extent.column; self->current_position.bytes -= self->current_position.extent.column;
self->current_position.extent.column = 0; self->current_position.extent.column = 0;
if (self->current_position.bytes < self->chunk_start) { if (self->current_position.bytes < self->chunk_start)
ts_lexer__get_chunk(self); {
} ts_lexer__get_chunk(self);
}
uint32_t result = 0; t_u32 result = 0;
if (!ts_lexer__eof(_self)) { if (!ts_lexer__eof(_self))
ts_lexer__get_lookahead(self); {
while (self->current_position.bytes < goal_byte && self->chunk) { ts_lexer__get_lookahead(self);
result++; while (self->current_position.bytes < goal_byte && self->chunk)
ts_lexer__do_advance(self, false); {
if (ts_lexer__eof(_self)) break; result++;
} ts_lexer__do_advance(self, false);
} if (ts_lexer__eof(_self))
break;
}
}
return result; return result;
} }
// Is the lexer at a boundary between two disjoint included ranges of // Is the lexer at a boundary between two disjoint included ranges of
// source code? This is exposed as an API because some languages' external // source code? This is exposed as an API because some languages' external
// scanners need to perform custom actions at these boundaries. // scanners need to perform custom actions at these boundaries.
static bool ts_lexer__is_at_included_range_start(const TSLexer *_self) { static bool ts_lexer__is_at_included_range_start(const TSLexer *_self)
const Lexer *self = (const Lexer *)_self; {
if (self->current_included_range_index < self->included_range_count) { const Lexer *self = (const Lexer *)_self;
TSRange *current_range = &self->included_ranges[self->current_included_range_index]; if (self->current_included_range_index < self->included_range_count)
return self->current_position.bytes == current_range->start_byte; {
} else { TSRange *current_range = &self->included_ranges[self->current_included_range_index];
return false; return self->current_position.bytes == current_range->start_byte;
} }
else
{
return false;
}
} }
void ts_lexer_init(Lexer *self) { void ts_lexer_init(Lexer *self)
*self = (Lexer) { {
.data = { *self = (Lexer){
// The lexer's methods are stored as struct fields so that generated .data =
// parsers can call them without needing to be linked against this {
// library. // The lexer's methods are stored as struct fields so that generated
.advance = ts_lexer__advance, // parsers can call them without needing to be linked against this
.mark_end = ts_lexer__mark_end, // library.
.get_column = ts_lexer__get_column, .advance = ts_lexer__advance,
.is_at_included_range_start = ts_lexer__is_at_included_range_start, .mark_end = ts_lexer__mark_end,
.eof = ts_lexer__eof, .get_column = ts_lexer__get_column,
.lookahead = 0, .is_at_included_range_start = ts_lexer__is_at_included_range_start,
.result_symbol = 0, .eof = ts_lexer__eof,
}, .lookahead = 0,
.chunk = NULL, .result_symbol = 0,
.chunk_size = 0, },
.chunk_start = 0, .chunk = NULL,
.current_position = {0, {0, 0}}, .chunk_size = 0,
.logger = { .chunk_start = 0,
.payload = NULL, .current_position = {0, {0, 0}},
.log = NULL .logger = {.payload = NULL, .log = NULL},
}, .included_ranges = NULL,
.included_ranges = NULL, .included_range_count = 0,
.included_range_count = 0, .current_included_range_index = 0,
.current_included_range_index = 0, };
}; ts_lexer_set_included_ranges(self, NULL, 0);
ts_lexer_set_included_ranges(self, NULL, 0);
} }
void ts_lexer_delete(Lexer *self) { void ts_lexer_delete(Lexer *self)
mem_free(self->included_ranges); {
mem_free(self->included_ranges);
} }
void ts_lexer_set_input(Lexer *self, TSInput input) { void ts_lexer_set_input(Lexer *self, TSInput input)
self->input = input; {
ts_lexer__clear_chunk(self); self->input = input;
ts_lexer_goto(self, self->current_position); ts_lexer__clear_chunk(self);
ts_lexer_goto(self, self->current_position);
} }
// Move the lexer to the given position. This doesn't do any work // Move the lexer to the given position. This doesn't do any work
// if the parser is already at the given position. // if the parser is already at the given position.
void ts_lexer_reset(Lexer *self, Length position) { void ts_lexer_reset(Lexer *self, Length position)
if (position.bytes != self->current_position.bytes) { {
ts_lexer_goto(self, position); if (position.bytes != self->current_position.bytes)
} {
ts_lexer_goto(self, position);
}
} }
void ts_lexer_start(Lexer *self) { void ts_lexer_start(Lexer *self)
self->token_start_position = self->current_position; {
self->token_end_position = LENGTH_UNDEFINED; self->token_start_position = self->current_position;
self->data.result_symbol = 0; self->token_end_position = LENGTH_UNDEFINED;
self->did_get_column = false; self->data.result_symbol = 0;
if (!ts_lexer__eof(&self->data)) { self->did_get_column = false;
if (!self->chunk_size) ts_lexer__get_chunk(self); if (!ts_lexer__eof(&self->data))
if (!self->lookahead_size) ts_lexer__get_lookahead(self); {
if ( if (!self->chunk_size)
self->current_position.bytes == 0 && ts_lexer__get_chunk(self);
self->data.lookahead == BYTE_ORDER_MARK if (!self->lookahead_size)
) ts_lexer__advance(&self->data, true); ts_lexer__get_lookahead(self);
} if (self->current_position.bytes == 0 && self->data.lookahead == BYTE_ORDER_MARK)
ts_lexer__advance(&self->data, true);
}
} }
void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte) { void ts_lexer_finish(Lexer *self, t_u32 *lookahead_end_byte)
if (length_is_undefined(self->token_end_position)) { {
ts_lexer__mark_end(&self->data); if (length_is_undefined(self->token_end_position))
} {
ts_lexer__mark_end(&self->data);
}
// If the token ended at an included range boundary, then its end position // If the token ended at an included range boundary, then its end position
// will have been reset to the end of the preceding range. Reset the start // will have been reset to the end of the preceding range. Reset the start
// position to match. // position to match.
if (self->token_end_position.bytes < self->token_start_position.bytes) { if (self->token_end_position.bytes < self->token_start_position.bytes)
self->token_start_position = self->token_end_position; {
} self->token_start_position = self->token_end_position;
}
uint32_t current_lookahead_end_byte = self->current_position.bytes + 1; t_u32 current_lookahead_end_byte = self->current_position.bytes + 1;
// In order to determine that a byte sequence is invalid UTF8 or UTF16, // In order to determine that a byte sequence is invalid UTF8 or UTF16,
// the character decoding algorithm may have looked at the following byte. // the character decoding algorithm may have looked at the following byte.
// Therefore, the next byte *after* the current (invalid) character // Therefore, the next byte *after* the current (invalid) character
// affects the interpretation of the current character. // affects the interpretation of the current character.
if (self->data.lookahead == TS_DECODE_ERROR) { if (self->data.lookahead == TS_DECODE_ERROR)
current_lookahead_end_byte += 4; // the maximum number of bytes read to identify an invalid code point {
} current_lookahead_end_byte += 4; // the maximum number of bytes read to identify an invalid code point
}
if (current_lookahead_end_byte > *lookahead_end_byte) { if (current_lookahead_end_byte > *lookahead_end_byte)
*lookahead_end_byte = current_lookahead_end_byte; {
} *lookahead_end_byte = current_lookahead_end_byte;
}
} }
void ts_lexer_advance_to_end(Lexer *self) { void ts_lexer_advance_to_end(Lexer *self)
while (self->chunk) { {
ts_lexer__advance(&self->data, false); while (self->chunk)
} {
ts_lexer__advance(&self->data, false);
}
} }
void ts_lexer_mark_end(Lexer *self) { void ts_lexer_mark_end(Lexer *self)
ts_lexer__mark_end(&self->data); {
ts_lexer__mark_end(&self->data);
} }
bool ts_lexer_set_included_ranges( bool ts_lexer_set_included_ranges(Lexer *self, const TSRange *ranges, t_u32 count)
Lexer *self, {
const TSRange *ranges, if (count == 0 || !ranges)
uint32_t count {
) { ranges = &DEFAULT_RANGE;
if (count == 0 || !ranges) { count = 1;
ranges = &DEFAULT_RANGE; }
count = 1; else
} else { {
uint32_t previous_byte = 0; t_u32 previous_byte = 0;
for (unsigned i = 0; i < count; i++) { for (unsigned i = 0; i < count; i++)
const TSRange *range = &ranges[i]; {
if ( const TSRange *range = &ranges[i];
range->start_byte < previous_byte || if (range->start_byte < previous_byte || range->end_byte < range->start_byte)
range->end_byte < range->start_byte return false;
) return false; previous_byte = range->end_byte;
previous_byte = range->end_byte; }
} }
}
size_t size = count * sizeof(TSRange); size_t size = count * sizeof(TSRange);
self->included_ranges = mem_realloc(self->included_ranges, size); self->included_ranges = mem_realloc(self->included_ranges, size);
memcpy(self->included_ranges, ranges, size); memcpy(self->included_ranges, ranges, size);
self->included_range_count = count; self->included_range_count = count;
ts_lexer_goto(self, self->current_position); ts_lexer_goto(self, self->current_position);
return true; return true;
} }
TSRange *ts_lexer_included_ranges(const Lexer *self, uint32_t *count) { TSRange *ts_lexer_included_ranges(const Lexer *self, t_u32 *count)
*count = self->included_range_count; {
return self->included_ranges; *count = self->included_range_count;
return self->included_ranges;
} }
#undef LOG #undef LOG

61
parser/src/lexer.h
View file

@ -1,43 +1,42 @@
#ifndef TREE_SITTER_LEXER_H_ #ifndef TREE_SITTER_LEXER_H_
#define TREE_SITTER_LEXER_H_ #define TREE_SITTER_LEXER_H_
#include "./api.h"
#include "./length.h" #include "./length.h"
#include "./subtree.h"
#include "api.h"
#include "./parser.h" #include "./parser.h"
#include "me/types.h"
typedef struct Lexer{ typedef struct Lexer
TSLexer data; {
Length current_position; TSLexer data;
Length token_start_position; Length current_position;
Length token_end_position; Length token_start_position;
Length token_end_position;
TSRange *included_ranges; TSRange *included_ranges;
const char *chunk; const char *chunk;
TSInput input; TSInput input;
TSLogger logger; TSLogger logger;
uint32_t included_range_count; t_u32 included_range_count;
uint32_t current_included_range_index; t_u32 current_included_range_index;
uint32_t chunk_start; t_u32 chunk_start;
uint32_t chunk_size; t_u32 chunk_size;
uint32_t lookahead_size; t_u32 lookahead_size;
bool did_get_column; bool did_get_column;
char debug_buffer[TREE_SITTER_SERIALIZATION_BUFFER_SIZE]; char debug_buffer[TREE_SITTER_SERIALIZATION_BUFFER_SIZE];
} Lexer; } Lexer;
void ts_lexer_init(Lexer *); void ts_lexer_init(Lexer *);
void ts_lexer_delete(Lexer *); void ts_lexer_delete(Lexer *);
void ts_lexer_set_input(Lexer *, TSInput); void ts_lexer_set_input(Lexer *, TSInput);
void ts_lexer_reset(Lexer *, Length); void ts_lexer_reset(Lexer *, Length);
void ts_lexer_start(Lexer *); void ts_lexer_start(Lexer *);
void ts_lexer_finish(Lexer *, uint32_t *); void ts_lexer_finish(Lexer *, t_u32 *);
void ts_lexer_advance_to_end(Lexer *); void ts_lexer_advance_to_end(Lexer *);
void ts_lexer_mark_end(Lexer *); void ts_lexer_mark_end(Lexer *);
bool ts_lexer_set_included_ranges(Lexer *self, const TSRange *ranges, uint32_t count); bool ts_lexer_set_included_ranges(Lexer *self, const TSRange *ranges, t_u32 count);
TSRange *ts_lexer_included_ranges(const Lexer *self, uint32_t *count); TSRange *ts_lexer_included_ranges(const Lexer *self, t_u32 *count);
#endif // TREE_SITTER_LEXER_H_
#endif // TREE_SITTER_LEXER_H_

84
parser/src/node.c
View file

@ -1,16 +1,16 @@
#include "./api.h"
#include "./language.h" #include "./language.h"
#include "./subtree.h" #include "./subtree.h"
#include "./tree.h" #include "./tree.h"
#include "api.h" #include "me/types.h"
#include <stdbool.h>
typedef struct NodeChildIterator typedef struct NodeChildIterator
{ {
Subtree parent; Subtree parent;
const TSTree *tree; const TSTree *tree;
Length position; Length position;
uint32_t child_index; t_u32 child_index;
uint32_t structural_child_index; t_u32 structural_child_index;
const TSSymbol *alias_sequence; const TSSymbol *alias_sequence;
} NodeChildIterator; } NodeChildIterator;
@ -32,7 +32,7 @@ static inline TSNode ts_node__null(void)
// TSNode - accessors // TSNode - accessors
uint32_t ts_node_start_byte(TSNode self) t_u32 ts_node_start_byte(TSNode self)
{ {
return self.context[0]; return self.context[0];
} }
@ -42,7 +42,7 @@ TSPoint ts_node_start_point(TSNode self)
return (TSPoint){self.context[1], self.context[2]}; return (TSPoint){self.context[1], self.context[2]};
} }
static inline uint32_t ts_node__alias(const TSNode *self) static inline t_u32 ts_node__alias(const TSNode *self)
{ {
return self->context[3]; return self->context[3];
} }
@ -124,7 +124,7 @@ static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous)
} }
} }
static inline uint32_t ts_node__relevant_child_count(TSNode self, bool include_anonymous) static inline t_u32 ts_node__relevant_child_count(TSNode self, bool include_anonymous)
{ {
Subtree tree = ts_node__subtree(self); Subtree tree = ts_node__subtree(self);
if (ts_subtree_child_count(tree) > 0) if (ts_subtree_child_count(tree) > 0)
@ -144,7 +144,7 @@ static inline uint32_t ts_node__relevant_child_count(TSNode self, bool include_a
} }
} }
static inline TSNode ts_node__child(TSNode self, uint32_t child_index, bool include_anonymous) static inline TSNode ts_node__child(TSNode self, t_u32 child_index, bool include_anonymous)
{ {
TSNode result = self; TSNode result = self;
bool did_descend = true; bool did_descend = true;
@ -154,7 +154,7 @@ static inline TSNode ts_node__child(TSNode self, uint32_t child_index, bool incl
did_descend = false; did_descend = false;
TSNode child; TSNode child;
uint32_t index = 0; t_u32 index = 0;
NodeChildIterator iterator = ts_node_iterate_children(&result); NodeChildIterator iterator = ts_node_iterate_children(&result);
while (ts_node_child_iterator_next(&iterator, &child)) while (ts_node_child_iterator_next(&iterator, &child))
{ {
@ -168,8 +168,8 @@ static inline TSNode ts_node__child(TSNode self, uint32_t child_index, bool incl
} }
else else
{ {
uint32_t grandchild_index = child_index - index; t_u32 grandchild_index = child_index - index;
uint32_t grandchild_count = ts_node__relevant_child_count(child, include_anonymous); t_u32 grandchild_count = ts_node__relevant_child_count(child, include_anonymous);
if (grandchild_index < grandchild_count) if (grandchild_index < grandchild_count)
{ {
did_descend = true; did_descend = true;
@ -202,9 +202,9 @@ static bool ts_subtree_has_trailing_empty_descendant(Subtree self, Subtree other
static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous)
{ {
Subtree self_subtree = ts_node__subtree(self); Subtree self_subtree = ts_node__subtree(self);
bool self_is_empty = ts_subtree_total_bytes(self_subtree) == 0; bool self_is_empty = ts_subtree_total_bytes(self_subtree) == 0;
uint32_t target_end_byte = ts_node_end_byte(self); t_u32 target_end_byte = ts_node_end_byte(self);
TSNode node = ts_node_parent(self); TSNode node = ts_node_parent(self);
TSNode earlier_node = ts_node__null(); TSNode earlier_node = ts_node__null();
@ -281,7 +281,7 @@ static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous)
static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous)
{ {
uint32_t target_end_byte = ts_node_end_byte(self); t_u32 target_end_byte = ts_node_end_byte(self);
TSNode node = ts_node_parent(self); TSNode node = ts_node_parent(self);
TSNode later_node = ts_node__null(); TSNode later_node = ts_node__null();
@ -350,7 +350,7 @@ static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous)
return ts_node__null(); return ts_node__null();
} }
static inline TSNode ts_node__first_child_for_byte(TSNode self, uint32_t goal, bool include_anonymous) static inline TSNode ts_node__first_child_for_byte(TSNode self, t_u32 goal, bool include_anonymous)
{ {
TSNode node = self; TSNode node = self;
bool did_descend = true; bool did_descend = true;
@ -382,7 +382,7 @@ static inline TSNode ts_node__first_child_for_byte(TSNode self, uint32_t goal, b
return ts_node__null(); return ts_node__null();
} }
static inline TSNode ts_node__descendant_for_byte_range(TSNode self, uint32_t range_start, uint32_t range_end, bool include_anonymous) static inline TSNode ts_node__descendant_for_byte_range(TSNode self, t_u32 range_start, t_u32 range_end, bool include_anonymous)
{ {
TSNode node = self; TSNode node = self;
TSNode last_visible_node = self; TSNode last_visible_node = self;
@ -396,7 +396,7 @@ static inline TSNode ts_node__descendant_for_byte_range(TSNode self, uint32_t ra
NodeChildIterator iterator = ts_node_iterate_children(&node); NodeChildIterator iterator = ts_node_iterate_children(&node);
while (ts_node_child_iterator_next(&iterator, &child)) while (ts_node_child_iterator_next(&iterator, &child))
{ {
uint32_t node_end = iterator.position.bytes; t_u32 node_end = iterator.position.bytes;
// The end of this node must extend far enough forward to touch // The end of this node must extend far enough forward to touch
// the end of the range and exceed the start of the range. // the end of the range and exceed the start of the range.
@ -466,7 +466,7 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint ra
// TSNode - public // TSNode - public
uint32_t ts_node_end_byte(TSNode self) t_u32 ts_node_end_byte(TSNode self)
{ {
return ts_node_start_byte(self) + ts_subtree_size(ts_node__subtree(self)).bytes; return ts_node_start_byte(self) + ts_subtree_size(ts_node__subtree(self)).bytes;
} }
@ -557,7 +557,7 @@ bool ts_node_is_error(TSNode self)
return symbol == ts_builtin_sym_error; return symbol == ts_builtin_sym_error;
} }
uint32_t ts_node_descendant_count(TSNode self) t_u32 ts_node_descendant_count(TSNode self)
{ {
return ts_subtree_visible_descendant_count(ts_node__subtree(self)) + 1; return ts_subtree_visible_descendant_count(ts_node__subtree(self)) + 1;
} }
@ -570,12 +570,12 @@ TSStateId ts_node_parse_state(TSNode self)
TSStateId ts_node_next_parse_state(TSNode self) TSStateId ts_node_next_parse_state(TSNode self)
{ {
const TSLanguage *language = self.tree->language; const TSLanguage *language = self.tree->language;
uint16_t state = ts_node_parse_state(self); t_u16 state = ts_node_parse_state(self);
if (state == TS_TREE_STATE_NONE) if (state == TS_TREE_STATE_NONE)
{ {
return TS_TREE_STATE_NONE; return TS_TREE_STATE_NONE;
} }
uint16_t symbol = ts_node_grammar_symbol(self); t_u16 symbol = ts_node_grammar_symbol(self);
return ts_language_next_state(language, state, symbol); return ts_language_next_state(language, state, symbol);
} }
@ -598,8 +598,8 @@ TSNode ts_node_parent(TSNode self)
TSNode ts_node_child_containing_descendant(TSNode self, TSNode subnode) TSNode ts_node_child_containing_descendant(TSNode self, TSNode subnode)
{ {
uint32_t start_byte = ts_node_start_byte(subnode); t_u32 start_byte = ts_node_start_byte(subnode);
uint32_t end_byte = ts_node_end_byte(subnode); t_u32 end_byte = ts_node_end_byte(subnode);
do do
{ {
@ -616,12 +616,12 @@ TSNode ts_node_child_containing_descendant(TSNode self, TSNode subnode)
return self; return self;
} }
TSNode ts_node_child(TSNode self, uint32_t child_index) TSNode ts_node_child(TSNode self, t_u32 child_index)
{ {
return ts_node__child(self, child_index, true); return ts_node__child(self, child_index, true);
} }
TSNode ts_node_named_child(TSNode self, uint32_t child_index) TSNode ts_node_named_child(TSNode self, t_u32 child_index)
{ {
return ts_node__child(self, child_index, false); return ts_node__child(self, child_index, false);
} }
@ -658,7 +658,7 @@ recur:
{ {
if (!ts_subtree_extra(ts_node__subtree(child))) if (!ts_subtree_extra(ts_node__subtree(child)))
{ {
uint32_t index = iterator.structural_child_index - 1; t_u32 index = iterator.structural_child_index - 1;
if (index < field_map->child_index) if (index < field_map->child_index)
continue; continue;
@ -712,7 +712,7 @@ recur:
return ts_node__null(); return ts_node__null();
} }
static inline const char *ts_node__field_name_from_language(TSNode self, uint32_t structural_child_index) static inline const char *ts_node__field_name_from_language(TSNode self, t_u32 structural_child_index)
{ {
const TSFieldMapEntry *field_map, *field_map_end; const TSFieldMapEntry *field_map, *field_map_end;
ts_language_field_map(self.tree->language, ts_node__subtree(self).ptr->production_id, &field_map, &field_map_end); ts_language_field_map(self.tree->language, ts_node__subtree(self).ptr->production_id, &field_map, &field_map_end);
@ -726,7 +726,7 @@ static inline const char *ts_node__field_name_from_language(TSNode self, uint32_
return NULL; return NULL;
} }
const char *ts_node_field_name_for_child(TSNode self, uint32_t child_index) const char *ts_node_field_name_for_child(TSNode self, t_u32 child_index)
{ {
TSNode result = self; TSNode result = self;
bool did_descend = true; bool did_descend = true;
@ -737,7 +737,7 @@ const char *ts_node_field_name_for_child(TSNode self, uint32_t child_index)
did_descend = false; did_descend = false;
TSNode child; TSNode child;
uint32_t index = 0; t_u32 index = 0;
NodeChildIterator iterator = ts_node_iterate_children(&result); NodeChildIterator iterator = ts_node_iterate_children(&result);
while (ts_node_child_iterator_next(&iterator, &child)) while (ts_node_child_iterator_next(&iterator, &child))
{ {
@ -758,8 +758,8 @@ const char *ts_node_field_name_for_child(TSNode self, uint32_t child_index)
} }
else else
{ {
uint32_t grandchild_index = child_index - index; t_u32 grandchild_index = child_index - index;
uint32_t grandchild_count = ts_node__relevant_child_count(child, true); t_u32 grandchild_count = ts_node__relevant_child_count(child, true);
if (grandchild_index < grandchild_count) if (grandchild_index < grandchild_count)
{ {
const char *field_name = ts_node__field_name_from_language(result, iterator.structural_child_index - 1); const char *field_name = ts_node__field_name_from_language(result, iterator.structural_child_index - 1);
@ -779,13 +779,13 @@ const char *ts_node_field_name_for_child(TSNode self, uint32_t child_index)
return NULL; return NULL;
} }
TSNode ts_node_child_by_field_name(TSNode self, const char *name, uint32_t name_length) TSNode ts_node_child_by_field_name(TSNode self, const char *name, t_u32 name_length)
{ {
TSFieldId field_id = ts_language_field_id_for_name(self.tree->language, name, name_length); TSFieldId field_id = ts_language_field_id_for_name(self.tree->language, name, name_length);
return ts_node_child_by_field_id(self, field_id); return ts_node_child_by_field_id(self, field_id);
} }
uint32_t ts_node_child_count(TSNode self) t_u32 ts_node_child_count(TSNode self)
{ {
Subtree tree = ts_node__subtree(self); Subtree tree = ts_node__subtree(self);
if (ts_subtree_child_count(tree) > 0) if (ts_subtree_child_count(tree) > 0)
@ -798,7 +798,7 @@ uint32_t ts_node_child_count(TSNode self)
} }
} }
uint32_t ts_node_named_child_count(TSNode self) t_u32 ts_node_named_child_count(TSNode self)
{ {
Subtree tree = ts_node__subtree(self); Subtree tree = ts_node__subtree(self);
if (ts_subtree_child_count(tree) > 0) if (ts_subtree_child_count(tree) > 0)
@ -831,22 +831,22 @@ TSNode ts_node_prev_named_sibling(TSNode self)
return ts_node__prev_sibling(self, false); return ts_node__prev_sibling(self, false);
} }
TSNode ts_node_first_child_for_byte(TSNode self, uint32_t byte) TSNode ts_node_first_child_for_byte(TSNode self, t_u32 byte)
{ {
return ts_node__first_child_for_byte(self, byte, true); return ts_node__first_child_for_byte(self, byte, true);
} }
TSNode ts_node_first_named_child_for_byte(TSNode self, uint32_t byte) TSNode ts_node_first_named_child_for_byte(TSNode self, t_u32 byte)
{ {
return ts_node__first_child_for_byte(self, byte, false); return ts_node__first_child_for_byte(self, byte, false);
} }
TSNode ts_node_descendant_for_byte_range(TSNode self, uint32_t start, uint32_t end) TSNode ts_node_descendant_for_byte_range(TSNode self, t_u32 start, t_u32 end)
{ {
return ts_node__descendant_for_byte_range(self, start, end, true); return ts_node__descendant_for_byte_range(self, start, end, true);
} }
TSNode ts_node_named_descendant_for_byte_range(TSNode self, uint32_t start, uint32_t end) TSNode ts_node_named_descendant_for_byte_range(TSNode self, t_u32 start, t_u32 end)
{ {
return ts_node__descendant_for_byte_range(self, start, end, false); return ts_node__descendant_for_byte_range(self, start, end, false);
} }
@ -863,8 +863,8 @@ TSNode ts_node_named_descendant_for_point_range(TSNode self, TSPoint start, TSPo
void ts_node_edit(TSNode *self, const TSInputEdit *edit) void ts_node_edit(TSNode *self, const TSInputEdit *edit)
{ {
uint32_t start_byte = ts_node_start_byte(*self); t_u32 start_byte = ts_node_start_byte(*self);
TSPoint start_point = ts_node_start_point(*self); TSPoint start_point = ts_node_start_point(*self);
if (start_byte >= edit->old_end_byte) if (start_byte >= edit->old_end_byte)
{ {
@ -882,7 +882,7 @@ void ts_node_edit(TSNode *self, const TSInputEdit *edit)
self->context[2] = start_point.column; self->context[2] = start_point.column;
} }
TSSymbol ts_node_field_id_for_child(TSNode self, uint32_t child_index) TSSymbol ts_node_field_id_for_child(TSNode self, t_u32 child_index)
{ {
const char *name = ts_node_field_name_for_child(self, child_index); const char *name = ts_node_field_name_for_child(self, child_index);
if (name == NULL) if (name == NULL)

115
parser/src/parser.c
View file

@ -10,11 +10,9 @@
#include "./subtree.h" #include "./subtree.h"
#include "./tree.h" #include "./tree.h"
#include "me/mem/mem.h" #include "me/mem/mem.h"
#include "me/types.h"
#include <assert.h> #include <assert.h>
#include <limits.h>
#include <stdbool.h>
#include <stdio.h> #include <stdio.h>
#include <time.h>
#define LOG(...) #define LOG(...)
#define LOG_LOOKAHEAD(...) #define LOG_LOOKAHEAD(...)
@ -33,9 +31,9 @@ static const unsigned OP_COUNT_PER_TIMEOUT_CHECK = 100;
typedef struct TokenCache typedef struct TokenCache
{ {
Subtree token; Subtree token;
Subtree last_external_token; Subtree last_external_token;
uint32_t byte_index; t_u32 byte_index;
} TokenCache; } TokenCache;
struct TSParser struct TSParser
@ -79,12 +77,12 @@ typedef enum ErrorComparison
typedef struct TSStringInput typedef struct TSStringInput
{ {
const char *string; const char *string;
uint32_t length; t_u32 length;
} TSStringInput; } TSStringInput;
// StringInput // StringInput
static const char *ts_string_input_read(void *_self, uint32_t byte, TSPoint point, uint32_t *length) static const char *ts_string_input_read(void *_self, t_u32 byte, TSPoint point, t_u32 *length)
{ {
(void)point; (void)point;
TSStringInput *self = (TSStringInput *)_self; TSStringInput *self = (TSStringInput *)_self;
@ -135,13 +133,13 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi
did_break_down = true; did_break_down = true;
pending = false; pending = false;
for (uint32_t i = 0; i < pop.size; i++) for (t_u32 i = 0; i < pop.size; i++)
{ {
StackSlice slice = pop.contents[i]; StackSlice slice = pop.contents[i];
TSStateId state = ts_stack_state(self->stack, slice.version); TSStateId state = ts_stack_state(self->stack, slice.version);
Subtree parent = *array_front(&slice.subtrees); Subtree parent = *array_front(&slice.subtrees);
for (uint32_t j = 0, n = ts_subtree_child_count(parent); j < n; j++) for (t_u32 j = 0, n = ts_subtree_child_count(parent); j < n; j++)
{ {
Subtree child = ts_subtree_children(parent)[j]; Subtree child = ts_subtree_children(parent)[j];
pending = ts_subtree_child_count(child) > 0; pending = ts_subtree_child_count(child) > 0;
@ -159,7 +157,7 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion versi
ts_stack_push(self->stack, slice.version, child, pending, state); ts_stack_push(self->stack, slice.version, child, pending, state);
} }
for (uint32_t j = 1; j < slice.subtrees.size; j++) for (t_u32 j = 1; j < slice.subtrees.size; j++)
{ {
Subtree tree = slice.subtrees.contents[j]; Subtree tree = slice.subtrees.contents[j];
ts_stack_push(self->stack, slice.version, tree, false, state); ts_stack_push(self->stack, slice.version, tree, false, state);
@ -318,7 +316,7 @@ static void ts_parser__external_scanner_destroy(TSParser *self)
static unsigned ts_parser__external_scanner_serialize(TSParser *self) static unsigned ts_parser__external_scanner_serialize(TSParser *self)
{ {
uint32_t length = self->language->external_scanner.serialize(self->external_scanner_payload, self->lexer.debug_buffer); t_u32 length = self->language->external_scanner.serialize(self->external_scanner_payload, self->lexer.debug_buffer);
assert(length <= TREE_SITTER_SERIALIZATION_BUFFER_SIZE); assert(length <= TREE_SITTER_SERIALIZATION_BUFFER_SIZE);
return length; return length;
} }
@ -326,7 +324,7 @@ static unsigned ts_parser__external_scanner_serialize(TSParser *self)
static void ts_parser__external_scanner_deserialize(TSParser *self, Subtree external_token) static void ts_parser__external_scanner_deserialize(TSParser *self, Subtree external_token)
{ {
const char *data = NULL; const char *data = NULL;
uint32_t length = 0; t_u32 length = 0;
if (external_token.ptr) if (external_token.ptr)
{ {
data = ts_external_scanner_state_data(&external_token.ptr->external_scanner_state); data = ts_external_scanner_state_data(&external_token.ptr->external_scanner_state);
@ -353,7 +351,7 @@ static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, Sub
// NULL, which indicates that the parser should look for a reduce action // NULL, which indicates that the parser should look for a reduce action
// at symbol `0`. Avoid reusing tokens in this situation to ensure that // at symbol `0`. Avoid reusing tokens in this situation to ensure that
// the same thing happens when incrementally reparsing. // the same thing happens when incrementally reparsing.
if (current_lex_mode.lex_state == (uint16_t)(-1)) if (current_lex_mode.lex_state == (t_u16)(-1))
return false; return false;
// If the token was created in a state with the same set of lookaheads, it is reusable. // If the token was created in a state with the same set of lookaheads, it is reusable.
@ -373,7 +371,7 @@ static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, Sub
static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state) static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state)
{ {
TSLexMode lex_mode = self->language->lex_modes[parse_state]; TSLexMode lex_mode = self->language->lex_modes[parse_state];
if (lex_mode.lex_state == (uint16_t)-1) if (lex_mode.lex_state == (t_u16)-1)
{ {
LOG("no_lookahead_after_non_terminal_extra"); LOG("no_lookahead_after_non_terminal_extra");
return NULL_SUBTREE; return NULL_SUBTREE;
@ -382,16 +380,16 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId pa
const Length start_position = ts_stack_position(self->stack, version); const Length start_position = ts_stack_position(self->stack, version);
const Subtree external_token = ts_stack_last_external_token(self->stack, version); const Subtree external_token = ts_stack_last_external_token(self->stack, version);
bool found_external_token = false; bool found_external_token = false;
bool error_mode = parse_state == ERROR_STATE; bool error_mode = parse_state == ERROR_STATE;
bool skipped_error = false; bool skipped_error = false;
bool called_get_column = false; bool called_get_column = false;
int32_t first_error_character = 0; t_i32 first_error_character = 0;
Length error_start_position = length_zero(); Length error_start_position = length_zero();
Length error_end_position = length_zero(); Length error_end_position = length_zero();
uint32_t lookahead_end_byte = 0; t_u32 lookahead_end_byte = 0;
uint32_t external_scanner_state_len = 0; t_u32 external_scanner_state_len = 0;
bool external_scanner_state_changed = false; bool external_scanner_state_changed = false;
ts_lexer_reset(&self->lexer, start_position); ts_lexer_reset(&self->lexer, start_position);
for (;;) for (;;)
@ -486,9 +484,9 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId pa
Subtree result; Subtree result;
if (skipped_error) if (skipped_error)
{ {
Length padding = length_sub(error_start_position, start_position); Length padding = length_sub(error_start_position, start_position);
Length size = length_sub(error_end_position, error_start_position); Length size = length_sub(error_end_position, error_start_position);
uint32_t lookahead_bytes = lookahead_end_byte - error_end_position.bytes; t_u32 lookahead_bytes = lookahead_end_byte - error_end_position.bytes;
result = ts_subtree_new_error(&self->tree_pool, first_error_character, padding, size, lookahead_bytes, parse_state, self->language); result = ts_subtree_new_error(&self->tree_pool, first_error_character, padding, size, lookahead_bytes, parse_state, self->language);
} }
else else
@ -497,7 +495,7 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId pa
TSSymbol symbol = self->lexer.data.result_symbol; TSSymbol symbol = self->lexer.data.result_symbol;
Length padding = length_sub(self->lexer.token_start_position, start_position); Length padding = length_sub(self->lexer.token_start_position, start_position);
Length size = length_sub(self->lexer.token_end_position, self->lexer.token_start_position); Length size = length_sub(self->lexer.token_end_position, self->lexer.token_start_position);
uint32_t lookahead_bytes = lookahead_end_byte - self->lexer.token_end_position.bytes; t_u32 lookahead_bytes = lookahead_end_byte - self->lexer.token_end_position.bytes;
if (found_external_token) if (found_external_token)
{ {
@ -505,7 +503,7 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId pa
} }
else if (symbol == self->language->keyword_capture_token && symbol != 0) else if (symbol == self->language->keyword_capture_token && symbol != 0)
{ {
uint32_t end_byte = self->lexer.token_end_position.bytes; t_u32 end_byte = self->lexer.token_end_position.bytes;
ts_lexer_reset(&self->lexer, self->lexer.token_start_position); ts_lexer_reset(&self->lexer, self->lexer.token_start_position);
ts_lexer_start(&self->lexer); ts_lexer_start(&self->lexer);
@ -550,7 +548,7 @@ static Subtree ts_parser__get_cached_token(TSParser *self, TSStateId state, size
return NULL_SUBTREE; return NULL_SUBTREE;
} }
static void ts_parser__set_cached_token(TSParser *self, uint32_t byte_index, Subtree last_external_token, Subtree token) static void ts_parser__set_cached_token(TSParser *self, t_u32 byte_index, Subtree last_external_token, Subtree token)
{ {
TokenCache *cache = &self->token_cache; TokenCache *cache = &self->token_cache;
if (token.ptr) if (token.ptr)
@ -655,10 +653,10 @@ static void ts_parser__shift(TSParser *self, StackVersion version, TSStateId sta
} }
} }
static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSymbol symbol, uint32_t count, int dynamic_precedence, static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSymbol symbol, t_u32 count, int dynamic_precedence,
uint16_t production_id, bool is_fragile, bool end_of_non_terminal_extra) t_u16 production_id, bool is_fragile, bool end_of_non_terminal_extra)
{ {
uint32_t initial_version_count = ts_stack_version_count(self->stack); t_u32 initial_version_count = ts_stack_version_count(self->stack);
// Pop the given number of nodes from the given version of the parse stack. // Pop the given number of nodes from the given version of the parse stack.
// If stack versions have previously merged, then there may be more than one // If stack versions have previously merged, then there may be more than one
@ -666,8 +664,8 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSy
// contain the popped children, and push it onto the stack in place of the // contain the popped children, and push it onto the stack in place of the
// children. // children.
StackSliceArray pop = ts_stack_pop_count(self->stack, version, count); StackSliceArray pop = ts_stack_pop_count(self->stack, version, count);
uint32_t removed_version_count = 0; t_u32 removed_version_count = 0;
for (uint32_t i = 0; i < pop.size; i++) for (t_u32 i = 0; i < pop.size; i++)
{ {
StackSlice slice = pop.contents[i]; StackSlice slice = pop.contents[i];
StackVersion slice_version = slice.version - removed_version_count; StackVersion slice_version = slice.version - removed_version_count;
@ -749,7 +747,7 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSy
// Push the parent node onto the stack, along with any extra tokens that // Push the parent node onto the stack, along with any extra tokens that
// were previously on top of the stack. // were previously on top of the stack.
ts_stack_push(self->stack, slice_version, ts_subtree_from_mut(parent), false, next_state); ts_stack_push(self->stack, slice_version, ts_subtree_from_mut(parent), false, next_state);
for (uint32_t j = 0; j < self->trailing_extras.size; j++) for (t_u32 j = 0; j < self->trailing_extras.size; j++)
{ {
ts_stack_push(self->stack, slice_version, self->trailing_extras.contents[j], false, next_state); ts_stack_push(self->stack, slice_version, self->trailing_extras.contents[j], false, next_state);
} }
@ -776,20 +774,20 @@ static void ts_parser__accept(TSParser *self, StackVersion version, Subtree look
ts_stack_push(self->stack, version, lookahead, false, 1); ts_stack_push(self->stack, version, lookahead, false, 1);
StackSliceArray pop = ts_stack_pop_all(self->stack, version); StackSliceArray pop = ts_stack_pop_all(self->stack, version);
for (uint32_t i = 0; i < pop.size; i++) for (t_u32 i = 0; i < pop.size; i++)
{ {
SubtreeArray trees = pop.contents[i].subtrees; SubtreeArray trees = pop.contents[i].subtrees;
Subtree root = NULL_SUBTREE; Subtree root = NULL_SUBTREE;
for (uint32_t j = trees.size - 1; j + 1 > 0; j--) for (t_u32 j = trees.size - 1; j + 1 > 0; j--)
{ {
Subtree tree = trees.contents[j]; Subtree tree = trees.contents[j];
if (!ts_subtree_extra(tree)) if (!ts_subtree_extra(tree))
{ {
assert(!tree.data.is_inline); assert(!tree.data.is_inline);
uint32_t child_count = ts_subtree_child_count(tree); t_u32 child_count = ts_subtree_child_count(tree);
const Subtree *children = ts_subtree_children(tree); const Subtree *children = ts_subtree_children(tree);
for (uint32_t k = 0; k < child_count; k++) for (t_u32 k = 0; k < child_count; k++)
{ {
ts_subtree_retain(children[k]); ts_subtree_retain(children[k]);
} }
@ -827,13 +825,13 @@ static void ts_parser__accept(TSParser *self, StackVersion version, Subtree look
static bool ts_parser__do_all_potential_reductions(TSParser *self, StackVersion starting_version, TSSymbol lookahead_symbol) static bool ts_parser__do_all_potential_reductions(TSParser *self, StackVersion starting_version, TSSymbol lookahead_symbol)
{ {
uint32_t initial_version_count = ts_stack_version_count(self->stack); t_u32 initial_version_count = ts_stack_version_count(self->stack);
bool can_shift_lookahead_symbol = false; bool can_shift_lookahead_symbol = false;
StackVersion version = starting_version; StackVersion version = starting_version;
for (unsigned i = 0; true; i++) for (unsigned i = 0; true; i++)
{ {
uint32_t version_count = ts_stack_version_count(self->stack); t_u32 version_count = ts_stack_version_count(self->stack);
if (version >= version_count) if (version >= version_count)
break; break;
@ -869,7 +867,7 @@ static bool ts_parser__do_all_potential_reductions(TSParser *self, StackVersion
{ {
TableEntry entry; TableEntry entry;
ts_language_table_entry(self->language, state, symbol, &entry); ts_language_table_entry(self->language, state, symbol, &entry);
for (uint32_t j = 0; j < entry.action_count; j++) for (t_u32 j = 0; j < entry.action_count; j++)
{ {
TSParseAction action = entry.actions[j]; TSParseAction action = entry.actions[j];
switch (action.type) switch (action.type)
@ -895,7 +893,7 @@ static bool ts_parser__do_all_potential_reductions(TSParser *self, StackVersion
} }
StackVersion reduction_version = STACK_VERSION_NONE; StackVersion reduction_version = STACK_VERSION_NONE;
for (uint32_t j = 0; j < self->reduce_actions.size; j++) for (t_u32 j = 0; j < self->reduce_actions.size; j++)
{ {
ReduceAction action = self->reduce_actions.contents[j]; ReduceAction action = self->reduce_actions.contents[j];
@ -958,8 +956,8 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un
if (error_trees.size > 0) if (error_trees.size > 0)
{ {
assert(error_trees.size == 1); assert(error_trees.size == 1);
Subtree error_tree = error_trees.contents[0]; Subtree error_tree = error_trees.contents[0];
uint32_t error_child_count = ts_subtree_child_count(error_tree); t_u32 error_child_count = ts_subtree_child_count(error_tree);
if (error_child_count > 0) if (error_child_count > 0)
{ {
array_splice(&slice.subtrees, 0, 0, error_child_count, ts_subtree_children(error_tree)); array_splice(&slice.subtrees, 0, 0, error_child_count, ts_subtree_children(error_tree));
@ -1172,14 +1170,14 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree loo
static void ts_parser__handle_error(TSParser *self, StackVersion version, Subtree lookahead) static void ts_parser__handle_error(TSParser *self, StackVersion version, Subtree lookahead)
{ {
uint32_t previous_version_count = ts_stack_version_count(self->stack); t_u32 previous_version_count = ts_stack_version_count(self->stack);
// Perform any reductions that can happen in this state, regardless of the lookahead. After // Perform any reductions that can happen in this state, regardless of the lookahead. After
// skipping one or more invalid tokens, the parser might find a token that would have allowed // skipping one or more invalid tokens, the parser might find a token that would have allowed
// a reduction to take place. // a reduction to take place.
ts_parser__do_all_potential_reductions(self, version, 0); ts_parser__do_all_potential_reductions(self, version, 0);
uint32_t version_count = ts_stack_version_count(self->stack); t_u32 version_count = ts_stack_version_count(self->stack);
Length position = ts_stack_position(self->stack, version); Length position = ts_stack_position(self->stack, version);
// Push a discontinuity onto the stack. Merge all of the stack versions that // Push a discontinuity onto the stack. Merge all of the stack versions that
// were created in the previous step. // were created in the previous step.
@ -1189,7 +1187,7 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version, Subtre
if (!did_insert_missing_token) if (!did_insert_missing_token)
{ {
TSStateId state = ts_stack_state(self->stack, v); TSStateId state = ts_stack_state(self->stack, v);
for (TSSymbol missing_symbol = 1; missing_symbol < (uint16_t)self->language->token_count; missing_symbol++) for (TSSymbol missing_symbol = 1; missing_symbol < (t_u16)self->language->token_count; missing_symbol++)
{ {
TSStateId state_after_missing_symbol = ts_language_next_state(self->language, state, missing_symbol); TSStateId state_after_missing_symbol = ts_language_next_state(self->language, state, missing_symbol);
if (state_after_missing_symbol == 0 || state_after_missing_symbol == state) if (state_after_missing_symbol == 0 || state_after_missing_symbol == state)
@ -1204,8 +1202,8 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version, Subtre
// must be assigned to position it within the next included range. // must be assigned to position it within the next included range.
ts_lexer_reset(&self->lexer, position); ts_lexer_reset(&self->lexer, position);
ts_lexer_mark_end(&self->lexer); ts_lexer_mark_end(&self->lexer);
Length padding = length_sub(self->lexer.token_end_position, position); Length padding = length_sub(self->lexer.token_end_position, position);
uint32_t lookahead_bytes = ts_subtree_total_bytes(lookahead) + ts_subtree_lookahead_bytes(lookahead); t_u32 lookahead_bytes = ts_subtree_total_bytes(lookahead) + ts_subtree_lookahead_bytes(lookahead);
StackVersion version_with_missing_tree = ts_stack_copy_version(self->stack, v); StackVersion version_with_missing_tree = ts_stack_copy_version(self->stack, v);
Subtree missing_tree = Subtree missing_tree =
@ -1250,7 +1248,7 @@ static bool ts_parser__advance(TSParser *self, StackVersion version, bool allow_
{ {
(void)(allow_node_reuse); (void)(allow_node_reuse);
TSStateId state = ts_stack_state(self->stack, version); TSStateId state = ts_stack_state(self->stack, version);
uint32_t position = ts_stack_position(self->stack, version).bytes; t_u32 position = ts_stack_position(self->stack, version).bytes;
Subtree last_external_token = ts_stack_last_external_token(self->stack, version); Subtree last_external_token = ts_stack_last_external_token(self->stack, version);
Subtree lookahead = NULL_SUBTREE; Subtree lookahead = NULL_SUBTREE;
@ -1310,7 +1308,7 @@ static bool ts_parser__advance(TSParser *self, StackVersion version, bool allow_
// version, whereas SHIFT actions update the existing stack version // version, whereas SHIFT actions update the existing stack version
// and terminate this loop. // and terminate this loop.
StackVersion last_reduction_version = STACK_VERSION_NONE; StackVersion last_reduction_version = STACK_VERSION_NONE;
for (uint32_t i = 0; i < table_entry.action_count; i++) for (t_u32 i = 0; i < table_entry.action_count; i++)
{ {
TSParseAction action = table_entry.actions[i]; TSParseAction action = table_entry.actions[i];
@ -1694,7 +1692,7 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input)
self->operation_count = 0; self->operation_count = 0;
uint32_t position = 0, last_position = 0, version_count = 0; t_u32 position = 0, last_position = 0, version_count = 0;
do do
{ {
for (StackVersion version = 0; version_count = ts_stack_version_count(self->stack), version < version_count; version++) for (StackVersion version = 0; version_count = ts_stack_version_count(self->stack), version < version_count; version++)
@ -1753,13 +1751,12 @@ exit:
return result; return result;
} }
TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, const char *string, uint32_t length) TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, const char *string, t_u32 length)
{ {
return ts_parser_parse_string_encoding(self, old_tree, string, length, TSInputEncodingUTF8); return ts_parser_parse_string_encoding(self, old_tree, string, length, TSInputEncodingUTF8);
} }
TSTree *ts_parser_parse_string_encoding(TSParser *self, const TSTree *old_tree, const char *string, uint32_t length, TSTree *ts_parser_parse_string_encoding(TSParser *self, const TSTree *old_tree, const char *string, t_u32 length, TSInputEncoding encoding)
TSInputEncoding encoding)
{ {
TSStringInput input = {string, length}; TSStringInput input = {string, length};
return ts_parser_parse(self, old_tree, return ts_parser_parse(self, old_tree,

90
parser/src/parser.h
View file

@ -1,32 +1,30 @@
#ifndef TREE_SITTER_PARSER_H_ #ifndef TREE_SITTER_PARSER_H_
#define TREE_SITTER_PARSER_H_ #define TREE_SITTER_PARSER_H_
#include <stdbool.h> #include "me/types.h"
#include <stdint.h>
#include <stdlib.h>
#define ts_builtin_sym_error ((TSSymbol)-1) #define ts_builtin_sym_error ((TSSymbol)-1)
#define ts_builtin_sym_end 0 #define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
#ifndef TREE_SITTER_API_H_ #ifndef TREE_SITTER_API_H_
typedef uint16_t TSStateId; typedef t_u16 TSStateId;
typedef uint16_t TSSymbol; typedef t_u16 TSSymbol;
typedef uint16_t TSFieldId; typedef t_u16 TSFieldId;
typedef struct TSLanguage TSLanguage; typedef struct TSLanguage TSLanguage;
#endif #endif
typedef struct TSFieldMapEntry typedef struct TSFieldMapEntry
{ {
TSFieldId field_id; TSFieldId field_id;
uint8_t child_index; t_u8 child_index;
bool inherited; bool inherited;
} TSFieldMapEntry; } TSFieldMapEntry;
typedef struct TSFieldMapSlice typedef struct TSFieldMapSlice
{ {
uint16_t index; t_u16 index;
uint16_t length; t_u16 length;
} TSFieldMapSlice; } TSFieldMapSlice;
typedef struct TSSymbolMetadata typedef struct TSSymbolMetadata
@ -40,11 +38,11 @@ typedef struct TSLexer TSLexer;
struct TSLexer struct TSLexer
{ {
int32_t lookahead; t_i32 lookahead;
TSSymbol result_symbol; TSSymbol result_symbol;
void (*advance)(TSLexer *, bool); void (*advance)(TSLexer *, bool);
void (*mark_end)(TSLexer *); void (*mark_end)(TSLexer *);
uint32_t (*get_column)(TSLexer *); t_u32 (*get_column)(TSLexer *);
bool (*is_at_included_range_start)(const TSLexer *); bool (*is_at_included_range_start)(const TSLexer *);
bool (*eof)(const TSLexer *); bool (*eof)(const TSLexer *);
}; };
@ -60,58 +58,58 @@ typedef enum TSParseActionType
typedef union TSParseAction { typedef union TSParseAction {
struct TSParseActionShift struct TSParseActionShift
{ {
uint8_t type; t_u8 type;
TSStateId state; TSStateId state;
bool extra; bool extra;
bool repetition; bool repetition;
} shift; } shift;
struct TSParseActionReduce struct TSParseActionReduce
{ {
uint8_t type; t_u8 type;
uint8_t child_count; t_u8 child_count;
TSSymbol symbol; TSSymbol symbol;
int16_t dynamic_precedence; t_i16 dynamic_precedence;
uint16_t production_id; t_u16 production_id;
} reduce; } reduce;
uint8_t type; t_u8 type;
} TSParseAction; } TSParseAction;
typedef struct TSLexMode typedef struct TSLexMode
{ {
uint16_t lex_state; t_u16 lex_state;
uint16_t external_lex_state; t_u16 external_lex_state;
} TSLexMode; } TSLexMode;
typedef union TSParseActionEntry { typedef union TSParseActionEntry {
TSParseAction action; TSParseAction action;
struct TSParseActionEntryInner struct TSParseActionEntryInner
{ {
uint8_t count; t_u8 count;
bool reusable; bool reusable;
} entry; } entry;
} TSParseActionEntry; } TSParseActionEntry;
typedef struct TSCharacterRange typedef struct TSCharacterRange
{ {
int32_t start; t_i32 start;
int32_t end; t_i32 end;
} TSCharacterRange; } TSCharacterRange;
struct TSLanguage struct TSLanguage
{ {
uint32_t version; t_u32 version;
uint32_t symbol_count; t_u32 symbol_count;
uint32_t alias_count; t_u32 alias_count;
uint32_t token_count; t_u32 token_count;
uint32_t external_token_count; t_u32 external_token_count;
uint32_t state_count; t_u32 state_count;
uint32_t large_state_count; t_u32 large_state_count;
uint32_t production_id_count; t_u32 production_id_count;
uint32_t field_count; t_u32 field_count;
uint16_t max_alias_sequence_length; t_u16 max_alias_sequence_length;
const uint16_t *parse_table; const t_u16 *parse_table;
const uint16_t *small_parse_table; const t_u16 *small_parse_table;
const uint32_t *small_parse_table_map; const t_u32 *small_parse_table_map;
const TSParseActionEntry *parse_actions; const TSParseActionEntry *parse_actions;
const char *const *symbol_names; const char *const *symbol_names;
const char *const *field_names; const char *const *field_names;
@ -119,7 +117,7 @@ struct TSLanguage
const TSFieldMapEntry *field_map_entries; const TSFieldMapEntry *field_map_entries;
const TSSymbolMetadata *symbol_metadata; const TSSymbolMetadata *symbol_metadata;
const TSSymbol *public_symbol_map; const TSSymbol *public_symbol_map;
const uint16_t *alias_map; const t_u16 *alias_map;
const TSSymbol *alias_sequences; const TSSymbol *alias_sequences;
const TSLexMode *lex_modes; const TSLexMode *lex_modes;
bool (*lex_fn)(TSLexer *, TSStateId); bool (*lex_fn)(TSLexer *, TSStateId);
@ -138,14 +136,14 @@ struct TSLanguage
const TSStateId *primary_state_ids; const TSStateId *primary_state_ids;
}; };
static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) static inline bool set_contains(TSCharacterRange *ranges, t_u32 len, t_i32 lookahead)
{ {
uint32_t index = 0; t_u32 index = 0;
uint32_t size = len - index; t_u32 size = len - index;
while (size > 1) while (size > 1)
{ {
uint32_t half_size = size / 2; t_u32 half_size = size / 2;
uint32_t mid_index = index + half_size; t_u32 mid_index = index + half_size;
TSCharacterRange *range = &ranges[mid_index]; TSCharacterRange *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end) if (lookahead >= range->start && lookahead <= range->end)
{ {
@ -171,8 +169,8 @@ static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t
bool result = false; \ bool result = false; \
bool skip = false; \ bool skip = false; \
UNUSED \ UNUSED \
bool eof = false; \ bool eof = false; \
int32_t lookahead; \ t_i32 lookahead; \
goto start; \ goto start; \
next_state: \ next_state: \
lexer->advance(lexer, skip); \ lexer->advance(lexer, skip); \
@ -188,8 +186,8 @@ start:
#define ADVANCE_MAP(...) \ #define ADVANCE_MAP(...) \
{ \ { \
static const uint16_t map[] = {__VA_ARGS__}; \ static const t_u16 map[] = {__VA_ARGS__}; \
for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) \ for (t_u32 i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) \
{ \ { \
if (map[i] == lookahead) \ if (map[i] == lookahead) \
{ \ { \

81
parser/src/point.h
View file

@ -2,61 +2,72 @@
#define TREE_SITTER_POINT_H_ #define TREE_SITTER_POINT_H_
#include "./api.h" #include "./api.h"
#include "me/types.h"
#define POINT_ZERO ((TSPoint) {0, 0}) #define POINT_ZERO ((TSPoint){0, 0})
#define POINT_MAX ((TSPoint) {UINT32_MAX, UINT32_MAX}) #define POINT_MAX ((TSPoint){UINT32_MAX, UINT32_MAX})
static inline TSPoint point__new(unsigned row, unsigned column) { static inline TSPoint point__new(unsigned row, unsigned column)
TSPoint result = {row, column}; {
return result; TSPoint result = {row, column};
return result;
} }
static inline TSPoint point_add(TSPoint a, TSPoint b) { static inline TSPoint point_add(TSPoint a, TSPoint b)
if (b.row > 0) {
return point__new(a.row + b.row, b.column); if (b.row > 0)
else return point__new(a.row + b.row, b.column);
return point__new(a.row, a.column + b.column); else
return point__new(a.row, a.column + b.column);
} }
static inline TSPoint point_sub(TSPoint a, TSPoint b) { static inline TSPoint point_sub(TSPoint a, TSPoint b)
if (a.row > b.row) {
return point__new(a.row - b.row, a.column); if (a.row > b.row)
else return point__new(a.row - b.row, a.column);
return point__new(0, a.column - b.column); else
return point__new(0, a.column - b.column);
} }
static inline bool point_lte(TSPoint a, TSPoint b) { static inline bool point_lte(TSPoint a, TSPoint b)
return (a.row < b.row) || (a.row == b.row && a.column <= b.column); {
return (a.row < b.row) || (a.row == b.row && a.column <= b.column);
} }
static inline bool point_lt(TSPoint a, TSPoint b) { static inline bool point_lt(TSPoint a, TSPoint b)
return (a.row < b.row) || (a.row == b.row && a.column < b.column); {
return (a.row < b.row) || (a.row == b.row && a.column < b.column);
} }
static inline bool point_gt(TSPoint a, TSPoint b) { static inline bool point_gt(TSPoint a, TSPoint b)
return (a.row > b.row) || (a.row == b.row && a.column > b.column); {
return (a.row > b.row) || (a.row == b.row && a.column > b.column);
} }
static inline bool point_gte(TSPoint a, TSPoint b) { static inline bool point_gte(TSPoint a, TSPoint b)
return (a.row > b.row) || (a.row == b.row && a.column >= b.column); {
return (a.row > b.row) || (a.row == b.row && a.column >= b.column);
} }
static inline bool point_eq(TSPoint a, TSPoint b) { static inline bool point_eq(TSPoint a, TSPoint b)
return a.row == b.row && a.column == b.column; {
return a.row == b.row && a.column == b.column;
} }
static inline TSPoint point_min(TSPoint a, TSPoint b) { static inline TSPoint point_min(TSPoint a, TSPoint b)
if (a.row < b.row || (a.row == b.row && a.column < b.column)) {
return a; if (a.row < b.row || (a.row == b.row && a.column < b.column))
else return a;
return b; else
return b;
} }
static inline TSPoint point_max(TSPoint a, TSPoint b) { static inline TSPoint point_max(TSPoint a, TSPoint b)
if (a.row > b.row || (a.row == b.row && a.column > b.column)) {
return a; if (a.row > b.row || (a.row == b.row && a.column > b.column))
else return a;
return b; else
return b;
} }
#endif #endif

35
parser/src/reduce_action.h
View file

@ -1,28 +1,29 @@
#ifndef TREE_SITTER_REDUCE_ACTION_H_ #ifndef TREE_SITTER_REDUCE_ACTION_H_
#define TREE_SITTER_REDUCE_ACTION_H_ #define TREE_SITTER_REDUCE_ACTION_H_
#include "./array.h"
#include "./api.h" #include "./api.h"
#include "./array.h"
#include "me/types.h"
typedef struct ReduceAction { typedef struct ReduceAction
uint32_t count; {
TSSymbol symbol; t_u32 count;
int dynamic_precedence; TSSymbol symbol;
unsigned short production_id; int dynamic_precedence;
unsigned short production_id;
} ReduceAction; } ReduceAction;
typedef Array(ReduceAction) ReduceActionSet; typedef Array(ReduceAction) ReduceActionSet;
static inline void ts_reduce_action_set_add(ReduceActionSet *self, static inline void ts_reduce_action_set_add(ReduceActionSet *self, ReduceAction new_action)
ReduceAction new_action) { {
for (uint32_t i = 0; i < self->size; i++) { for (t_u32 i = 0; i < self->size; i++)
ReduceAction action = self->contents[i]; {
if (action.symbol == new_action.symbol && action.count == new_action.count) ReduceAction action = self->contents[i];
return; if (action.symbol == new_action.symbol && action.count == new_action.count)
} return;
array_push(self, new_action); }
array_push(self, new_action);
} }
#endif // TREE_SITTER_REDUCE_ACTION_H_
#endif // TREE_SITTER_REDUCE_ACTION_H_

63
parser/src/scanner.c
View file

@ -1,8 +1,7 @@
#include "array.h" #include "array.h"
#include "me/types.h"
#include "parser.h" #include "parser.h"
#include <assert.h> #include <assert.h>
#include <ctype.h>
#include <string.h> #include <string.h>
#include <wctype.h> #include <wctype.h>
@ -52,9 +51,9 @@ typedef struct Heredoc
typedef struct Scanner typedef struct Scanner
{ {
uint8_t last_glob_paren_depth; t_u8 last_glob_paren_depth;
bool ext_was_in_double_quote; bool ext_was_in_double_quote;
bool ext_saw_outside_quote; bool ext_saw_outside_quote;
Array(Heredoc) heredocs; Array(Heredoc) heredocs;
} Scanner; } Scanner;
@ -92,7 +91,7 @@ static inline void reset_heredoc(Heredoc *heredoc)
static inline void reset(Scanner *scanner) static inline void reset(Scanner *scanner)
{ {
for (uint32_t i = 0; i < scanner->heredocs.size; i++) for (t_u32 i = 0; i < scanner->heredocs.size; i++)
{ {
reset_heredoc(array_get(&scanner->heredocs, i)); reset_heredoc(array_get(&scanner->heredocs, i));
} }
@ -100,14 +99,14 @@ static inline void reset(Scanner *scanner)
static unsigned serialize(Scanner *scanner, char *buffer) static unsigned serialize(Scanner *scanner, char *buffer)
{ {
uint32_t size = 0; t_u32 size = 0;
buffer[size++] = (char)scanner->last_glob_paren_depth; buffer[size++] = (char)scanner->last_glob_paren_depth;
buffer[size++] = (char)scanner->ext_was_in_double_quote; buffer[size++] = (char)scanner->ext_was_in_double_quote;
buffer[size++] = (char)scanner->ext_saw_outside_quote; buffer[size++] = (char)scanner->ext_saw_outside_quote;
buffer[size++] = (char)scanner->heredocs.size; buffer[size++] = (char)scanner->heredocs.size;
for (uint32_t i = 0; i < scanner->heredocs.size; i++) for (t_u32 i = 0; i < scanner->heredocs.size; i++)
{ {
Heredoc *heredoc = array_get(&scanner->heredocs, i); Heredoc *heredoc = array_get(&scanner->heredocs, i);
if (heredoc->delimiter.size + 3 + size >= TREE_SITTER_SERIALIZATION_BUFFER_SIZE) if (heredoc->delimiter.size + 3 + size >= TREE_SITTER_SERIALIZATION_BUFFER_SIZE)
@ -119,8 +118,8 @@ static unsigned serialize(Scanner *scanner, char *buffer)
buffer[size++] = (char)heredoc->started; buffer[size++] = (char)heredoc->started;
buffer[size++] = (char)heredoc->allows_indent; buffer[size++] = (char)heredoc->allows_indent;
memcpy(&buffer[size], &heredoc->delimiter.size, sizeof(uint32_t)); memcpy(&buffer[size], &heredoc->delimiter.size, sizeof(t_u32));
size += sizeof(uint32_t); size += sizeof(t_u32);
if (heredoc->delimiter.size > 0) if (heredoc->delimiter.size > 0)
{ {
memcpy(&buffer[size], heredoc->delimiter.contents, heredoc->delimiter.size); memcpy(&buffer[size], heredoc->delimiter.contents, heredoc->delimiter.size);
@ -138,12 +137,12 @@ static void deserialize(Scanner *scanner, const char *buffer, unsigned length)
} }
else else
{ {
uint32_t size = 0; t_u32 size = 0;
scanner->last_glob_paren_depth = buffer[size++]; scanner->last_glob_paren_depth = buffer[size++];
scanner->ext_was_in_double_quote = buffer[size++]; scanner->ext_was_in_double_quote = buffer[size++];
scanner->ext_saw_outside_quote = buffer[size++]; scanner->ext_saw_outside_quote = buffer[size++];
uint32_t heredoc_count = (unsigned char)buffer[size++]; t_u32 heredoc_count = (unsigned char)buffer[size++];
for (uint32_t i = 0; i < heredoc_count; i++) for (t_u32 i = 0; i < heredoc_count; i++)
{ {
Heredoc *heredoc = NULL; Heredoc *heredoc = NULL;
if (i < scanner->heredocs.size) if (i < scanner->heredocs.size)
@ -161,8 +160,8 @@ static void deserialize(Scanner *scanner, const char *buffer, unsigned length)
heredoc->started = buffer[size++]; heredoc->started = buffer[size++];
heredoc->allows_indent = buffer[size++]; heredoc->allows_indent = buffer[size++];
memcpy(&heredoc->delimiter.size, &buffer[size], sizeof(uint32_t)); memcpy(&heredoc->delimiter.size, &buffer[size], sizeof(t_u32));
size += sizeof(uint32_t); size += sizeof(t_u32);
array_reserve(&heredoc->delimiter, heredoc->delimiter.size); array_reserve(&heredoc->delimiter, heredoc->delimiter.size);
if (heredoc->delimiter.size > 0) if (heredoc->delimiter.size > 0)
@ -184,8 +183,8 @@ static void deserialize(Scanner *scanner, const char *buffer, unsigned length)
*/ */
static bool advance_word(TSLexer *lexer, String *unquoted_word) static bool advance_word(TSLexer *lexer, String *unquoted_word)
{ {
bool empty = true; bool empty = true;
int32_t quote = 0; t_i32 quote = 0;
if (lexer->lookahead == '\'' || lexer->lookahead == '"') if (lexer->lookahead == '\'' || lexer->lookahead == '"')
{ {
@ -254,10 +253,10 @@ static bool scan_heredoc_end_identifier(Heredoc *heredoc, TSLexer *lexer)
reset_string(&heredoc->current_leading_word); reset_string(&heredoc->current_leading_word);
// Scan the first 'n' characters on this line, to see if they match the // Scan the first 'n' characters on this line, to see if they match the
// heredoc delimiter // heredoc delimiter
int32_t size = 0; t_i32 size = 0;
if (heredoc->delimiter.size > 0) if (heredoc->delimiter.size > 0)
{ {
while (lexer->lookahead != '\0' && lexer->lookahead != '\n' && (int32_t)*array_get(&heredoc->delimiter, size) == lexer->lookahead && while (lexer->lookahead != '\0' && lexer->lookahead != '\n' && (t_i32)*array_get(&heredoc->delimiter, size) == lexer->lookahead &&
heredoc->current_leading_word.size < heredoc->delimiter.size) heredoc->current_leading_word.size < heredoc->delimiter.size)
{ {
array_push(&heredoc->current_leading_word, lexer->lookahead); array_push(&heredoc->current_leading_word, lexer->lookahead);
@ -710,14 +709,14 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols)
{ {
typedef struct typedef struct
{ {
bool done; bool done;
bool advanced_once; bool advanced_once;
bool found_non_alnumdollarunderdash; bool found_non_alnumdollarunderdash;
bool last_was_escape; bool last_was_escape;
bool in_single_quote; bool in_single_quote;
uint32_t paren_depth; t_u32 paren_depth;
uint32_t bracket_depth; t_u32 bracket_depth;
uint32_t brace_depth; t_u32 brace_depth;
} State; } State;
if (lexer->lookahead == '$') if (lexer->lookahead == '$')
@ -952,11 +951,11 @@ extglob_pattern:
typedef struct typedef struct
{ {
bool done; bool done;
bool saw_non_alphadot; bool saw_non_alphadot;
uint32_t paren_depth; t_u32 paren_depth;
uint32_t bracket_depth; t_u32 bracket_depth;
uint32_t brace_depth; t_u32 brace_depth;
} State; } State;
State state = {false, was_non_alpha, scanner->last_glob_paren_depth, 0, 0}; State state = {false, was_non_alpha, scanner->last_glob_paren_depth, 0, 0};

34
parser/src/stack.c
View file

@ -5,8 +5,8 @@
#include "./length.h" #include "./length.h"
#include "./subtree.h" #include "./subtree.h"
#include "me/mem/mem.h" #include "me/mem/mem.h"
#include "me/types.h"
#include <assert.h> #include <assert.h>
#include <inttypes.h>
#include <stdio.h> #include <stdio.h>
#define MAX_LINK_COUNT 8 #define MAX_LINK_COUNT 8
@ -28,7 +28,7 @@ struct StackNode
Length position; Length position;
StackLink links[MAX_LINK_COUNT]; StackLink links[MAX_LINK_COUNT];
short unsigned int link_count; short unsigned int link_count;
uint32_t ref_count; t_u32 ref_count;
unsigned error_cost; unsigned error_cost;
unsigned node_count; unsigned node_count;
int dynamic_precedence; int dynamic_precedence;
@ -38,7 +38,7 @@ typedef struct StackIterator
{ {
StackNode *node; StackNode *node;
SubtreeArray subtrees; SubtreeArray subtrees;
uint32_t subtree_count; t_u32 subtree_count;
bool is_pending; bool is_pending;
} StackIterator; } StackIterator;
@ -132,9 +132,9 @@ recur:
/// Get the number of nodes in the subtree, for the purpose of measuring /// Get the number of nodes in the subtree, for the purpose of measuring
/// how much progress has been made by a given version of the stack. /// how much progress has been made by a given version of the stack.
static uint32_t stack__subtree_node_count(Subtree subtree) static t_u32 stack__subtree_node_count(Subtree subtree)
{ {
uint32_t count = ts_subtree_visible_descendant_count(subtree); t_u32 count = ts_subtree_visible_descendant_count(subtree);
if (ts_subtree_visible(subtree)) if (ts_subtree_visible(subtree))
count++; count++;
@ -237,7 +237,7 @@ static void stack_node_add_link(StackNode *self, StackLink link, SubtreePool *su
{ {
stack_node_add_link(existing_link->node, link.node->links[j], subtree_pool); stack_node_add_link(existing_link->node, link.node->links[j], subtree_pool);
} }
int32_t dynamic_precedence = link.node->dynamic_precedence; t_i32 dynamic_precedence = link.node->dynamic_precedence;
if (link.subtree.ptr) if (link.subtree.ptr)
{ {
dynamic_precedence += ts_subtree_dynamic_precedence(link.subtree); dynamic_precedence += ts_subtree_dynamic_precedence(link.subtree);
@ -311,7 +311,7 @@ static StackVersion ts_stack__add_version(Stack *self, StackVersion original_ver
static void ts_stack__add_slice(Stack *self, StackVersion original_version, StackNode *node, SubtreeArray *subtrees) static void ts_stack__add_slice(Stack *self, StackVersion original_version, StackNode *node, SubtreeArray *subtrees)
{ {
for (uint32_t i = self->slices.size - 1; i + 1 > 0; i--) for (t_u32 i = self->slices.size - 1; i + 1 > 0; i--)
{ {
StackVersion version = self->slices.contents[i].version; StackVersion version = self->slices.contents[i].version;
if (self->heads.contents[version].node == node) if (self->heads.contents[version].node == node)
@ -344,14 +344,14 @@ static StackSliceArray stack__iter(Stack *self, StackVersion version, StackCallb
if (goal_subtree_count >= 0) if (goal_subtree_count >= 0)
{ {
include_subtrees = true; include_subtrees = true;
array_reserve(&new_iterator.subtrees, (uint32_t)ts_subtree_alloc_size(goal_subtree_count) / sizeof(Subtree)); array_reserve(&new_iterator.subtrees, (t_u32)ts_subtree_alloc_size(goal_subtree_count) / sizeof(Subtree));
} }
array_push(&self->iterators, new_iterator); array_push(&self->iterators, new_iterator);
while (self->iterators.size > 0) while (self->iterators.size > 0)
{ {
for (uint32_t i = 0, size = self->iterators.size; i < size; i++) for (t_u32 i = 0, size = self->iterators.size; i < size; i++)
{ {
StackIterator *iterator = &self->iterators.contents[i]; StackIterator *iterator = &self->iterators.contents[i];
StackNode *node = iterator->node; StackNode *node = iterator->node;
@ -382,7 +382,7 @@ static StackSliceArray stack__iter(Stack *self, StackVersion version, StackCallb
continue; continue;
} }
for (uint32_t j = 1; j <= node->link_count; j++) for (t_u32 j = 1; j <= node->link_count; j++)
{ {
StackIterator *next_iterator; StackIterator *next_iterator;
StackLink link; StackLink link;
@ -459,14 +459,14 @@ void ts_stack_delete(Stack *self)
if (self->iterators.contents) if (self->iterators.contents)
array_delete(&self->iterators); array_delete(&self->iterators);
stack_node_release(self->base_node, &self->node_pool, self->subtree_pool); stack_node_release(self->base_node, &self->node_pool, self->subtree_pool);
for (uint32_t i = 0; i < self->heads.size; i++) for (t_u32 i = 0; i < self->heads.size; i++)
{ {
stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool); stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool);
} }
array_clear(&self->heads); array_clear(&self->heads);
if (self->node_pool.contents) if (self->node_pool.contents)
{ {
for (uint32_t i = 0; i < self->node_pool.size; i++) for (t_u32 i = 0; i < self->node_pool.size; i++)
mem_free(self->node_pool.contents[i]); mem_free(self->node_pool.contents[i]);
array_delete(&self->node_pool); array_delete(&self->node_pool);
} }
@ -474,7 +474,7 @@ void ts_stack_delete(Stack *self)
mem_free(self); mem_free(self);
} }
uint32_t ts_stack_version_count(const Stack *self) t_u32 ts_stack_version_count(const Stack *self)
{ {
return self->heads.size; return self->heads.size;
} }
@ -547,7 +547,7 @@ StackAction pop_count_callback(void *payload, const StackIterator *iterator)
} }
} }
StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t count) StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, t_u32 count)
{ {
return stack__iter(self, version, pop_count_callback, &count, (int)count); return stack__iter(self, version, pop_count_callback, &count, (int)count);
} }
@ -729,7 +729,7 @@ void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2)
if (v1 == v2) if (v1 == v2)
return; return;
assert(v2 < v1); assert(v2 < v1);
assert((uint32_t)v1 < self->heads.size); assert((t_u32)v1 < self->heads.size);
StackHead *source_head = &self->heads.contents[v1]; StackHead *source_head = &self->heads.contents[v1];
StackHead *target_head = &self->heads.contents[v2]; StackHead *target_head = &self->heads.contents[v2];
if (target_head->summary && !source_head->summary) if (target_head->summary && !source_head->summary)
@ -767,7 +767,7 @@ bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2)
return false; return false;
StackHead *head1 = &self->heads.contents[version1]; StackHead *head1 = &self->heads.contents[version1];
StackHead *head2 = &self->heads.contents[version2]; StackHead *head2 = &self->heads.contents[version2];
for (uint32_t i = 0; i < head2->node->link_count; i++) for (t_u32 i = 0; i < head2->node->link_count; i++)
{ {
stack_node_add_link(head1->node, head2->node->links[i], self->subtree_pool); stack_node_add_link(head1->node, head2->node->links[i], self->subtree_pool);
} }
@ -829,7 +829,7 @@ Subtree ts_stack_resume(Stack *self, StackVersion version)
void ts_stack_clear(Stack *self) void ts_stack_clear(Stack *self)
{ {
stack_node_retain(self->base_node); stack_node_retain(self->base_node);
for (uint32_t i = 0; i < self->heads.size; i++) for (t_u32 i = 0; i < self->heads.size; i++)
{ {
stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool); stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool);
} }

9
parser/src/stack.h
View file

@ -1,8 +1,9 @@
#ifndef TREE_SITTER_PARSE_STACK_H_ #ifndef TREE_SITTER_PARSE_STACK_H_
#define TREE_SITTER_PARSE_STACK_H_ #define TREE_SITTER_PARSE_STACK_H_
#include "me/types.h"
#include "./array.h" #include "./array.h"
// #include "./error_costs.h"
#include "./subtree.h" #include "./subtree.h"
typedef struct Stack Stack; typedef struct Stack Stack;
@ -26,7 +27,7 @@ typedef struct StackSummaryEntry
typedef Array(StackSummaryEntry) StackSummary; typedef Array(StackSummaryEntry) StackSummary;
typedef void (*StackIterateCallback)(void *, TSStateId, uint32_t); typedef void (*StackIterateCallback)(void *, TSStateId, t_u32);
// Create a stack. // Create a stack.
Stack *ts_stack_new(SubtreePool *); Stack *ts_stack_new(SubtreePool *);
@ -35,7 +36,7 @@ Stack *ts_stack_new(SubtreePool *);
void ts_stack_delete(Stack *); void ts_stack_delete(Stack *);
// Get the stack's current number of versions. // Get the stack's current number of versions.
uint32_t ts_stack_version_count(const Stack *); t_u32 ts_stack_version_count(const Stack *);
// Get the state at the top of the given version of the stack. If the stack is // Get the state at the top of the given version of the stack. If the stack is
// empty, this returns the initial state, 0. // empty, this returns the initial state, 0.
@ -62,7 +63,7 @@ void ts_stack_push(Stack *, StackVersion, Subtree, bool, TSStateId);
// versions which had previously been merged. It returns an array that // versions which had previously been merged. It returns an array that
// specifies the index of each revealed version and the trees that were // specifies the index of each revealed version and the trees that were
// removed from that version. // removed from that version.
StackSliceArray ts_stack_pop_count(Stack *, StackVersion, uint32_t count); StackSliceArray ts_stack_pop_count(Stack *, StackVersion, t_u32 count);
// Remove an error at the top of the given version of the stack. // Remove an error at the top of the given version of the stack.
SubtreeArray ts_stack_pop_error(Stack *, StackVersion); SubtreeArray ts_stack_pop_error(Stack *, StackVersion);

70
parser/src/subtree.c
View file

@ -1,13 +1,11 @@
#include <assert.h> #include <assert.h>
#include <ctype.h> #include <ctype.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
#include <stddef.h> #include <stddef.h>
#include "me/types.h"
#include "./array.h" #include "./array.h"
#include "./error_costs.h"
#include "./language.h" #include "./language.h"
#include "./length.h" #include "./length.h"
#include "./subtree.h" #include "./subtree.h"
@ -86,7 +84,7 @@ void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest)
{ {
dest->contents = mem_alloc_array(self.capacity, sizeof(Subtree)); dest->contents = mem_alloc_array(self.capacity, sizeof(Subtree));
memcpy(dest->contents, self.contents, self.size * sizeof(Subtree)); memcpy(dest->contents, self.contents, self.size * sizeof(Subtree));
for (uint32_t i = 0; i < self.size; i++) for (t_u32 i = 0; i < self.size; i++)
{ {
ts_subtree_retain(dest->contents[i]); ts_subtree_retain(dest->contents[i]);
} }
@ -95,7 +93,7 @@ void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest)
void ts_subtree_array_clear(SubtreePool *pool, SubtreeArray *self) void ts_subtree_array_clear(SubtreePool *pool, SubtreeArray *self)
{ {
for (uint32_t i = 0; i < self->size; i++) for (t_u32 i = 0; i < self->size; i++)
{ {
ts_subtree_release(pool, self->contents[i]); ts_subtree_release(pool, self->contents[i]);
} }
@ -129,7 +127,7 @@ void ts_subtree_array_remove_trailing_extras(SubtreeArray *self, SubtreeArray *d
void ts_subtree_array_reverse(SubtreeArray *self) void ts_subtree_array_reverse(SubtreeArray *self)
{ {
for (uint32_t i = 0, limit = self->size / 2; i < limit; i++) for (t_u32 i = 0, limit = self->size / 2; i < limit; i++)
{ {
size_t reverse_index = self->size - 1 - i; size_t reverse_index = self->size - 1 - i;
Subtree swap = self->contents[i]; Subtree swap = self->contents[i];
@ -140,7 +138,7 @@ void ts_subtree_array_reverse(SubtreeArray *self)
// SubtreePool // SubtreePool
SubtreePool ts_subtree_pool_new(uint32_t capacity) SubtreePool ts_subtree_pool_new(t_u32 capacity)
{ {
SubtreePool self = {array_new(), array_new()}; SubtreePool self = {array_new(), array_new()};
array_reserve(&self.free_trees, capacity); array_reserve(&self.free_trees, capacity);
@ -187,13 +185,13 @@ static void ts_subtree_pool_free(SubtreePool *self, SubtreeHeapData *tree)
// Subtree // Subtree
static inline bool ts_subtree_can_inline(Length padding, Length size, uint32_t lookahead_bytes) static inline bool ts_subtree_can_inline(Length padding, Length size, t_u32 lookahead_bytes)
{ {
return padding.bytes < TS_MAX_INLINE_TREE_LENGTH && padding.extent.row < 16 && padding.extent.column < TS_MAX_INLINE_TREE_LENGTH && return padding.bytes < TS_MAX_INLINE_TREE_LENGTH && padding.extent.row < 16 && padding.extent.column < TS_MAX_INLINE_TREE_LENGTH &&
size.extent.row == 0 && size.extent.column < TS_MAX_INLINE_TREE_LENGTH && lookahead_bytes < 16; size.extent.row == 0 && size.extent.column < TS_MAX_INLINE_TREE_LENGTH && lookahead_bytes < 16;
} }
Subtree ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, uint32_t lookahead_bytes, Subtree ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, t_u32 lookahead_bytes,
TSStateId parse_state, bool has_external_tokens, bool depends_on_column, bool is_keyword, TSStateId parse_state, bool has_external_tokens, bool depends_on_column, bool is_keyword,
const TSLanguage *language) const TSLanguage *language)
{ {
@ -266,7 +264,7 @@ void ts_subtree_set_symbol(MutableSubtree *self, TSSymbol symbol, const TSLangua
} }
} }
Subtree ts_subtree_new_error(SubtreePool *pool, int32_t lookahead_char, Length padding, Length size, uint32_t bytes_scanned, Subtree ts_subtree_new_error(SubtreePool *pool, t_i32 lookahead_char, Length padding, Length size, t_u32 bytes_scanned,
TSStateId parse_state, const TSLanguage *language) TSStateId parse_state, const TSLanguage *language)
{ {
Subtree result = Subtree result =
@ -288,7 +286,7 @@ MutableSubtree ts_subtree_clone(Subtree self)
SubtreeHeapData *result = (SubtreeHeapData *)&new_children[self.ptr->child_count]; SubtreeHeapData *result = (SubtreeHeapData *)&new_children[self.ptr->child_count];
if (self.ptr->child_count > 0) if (self.ptr->child_count > 0)
{ {
for (uint32_t i = 0; i < self.ptr->child_count; i++) for (t_u32 i = 0; i < self.ptr->child_count; i++)
{ {
ts_subtree_retain(new_children[i]); ts_subtree_retain(new_children[i]);
} }
@ -384,7 +382,7 @@ void ts_subtree_balance(Subtree self, SubtreePool *pool, const TSLanguage *langu
} }
} }
for (uint32_t i = 0; i < tree.ptr->child_count; i++) for (t_u32 i = 0; i < tree.ptr->child_count; i++)
{ {
Subtree child = ts_subtree_children(tree)[i]; Subtree child = ts_subtree_children(tree)[i];
if (ts_subtree_child_count(child) > 0 && child.ptr->ref_count == 1) if (ts_subtree_child_count(child) > 0 && child.ptr->ref_count == 1)
@ -410,12 +408,12 @@ void ts_subtree_summarize_children(MutableSubtree self, const TSLanguage *langua
self.ptr->has_external_scanner_state_change = false; self.ptr->has_external_scanner_state_change = false;
self.ptr->dynamic_precedence = 0; self.ptr->dynamic_precedence = 0;
uint32_t structural_index = 0; t_u32 structural_index = 0;
const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self.ptr->production_id); const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self.ptr->production_id);
uint32_t lookahead_end_byte = 0; t_u32 lookahead_end_byte = 0;
const Subtree *children = ts_subtree_children(self); const Subtree *children = ts_subtree_children(self);
for (uint32_t i = 0; i < self.ptr->child_count; i++) for (t_u32 i = 0; i < self.ptr->child_count; i++)
{ {
Subtree child = children[i]; Subtree child = children[i];
@ -439,7 +437,7 @@ void ts_subtree_summarize_children(MutableSubtree self, const TSLanguage *langua
self.ptr->size = length_add(self.ptr->size, ts_subtree_total_size(child)); self.ptr->size = length_add(self.ptr->size, ts_subtree_total_size(child));
} }
uint32_t child_lookahead_end_byte = self.ptr->padding.bytes + self.ptr->size.bytes + ts_subtree_lookahead_bytes(child); t_u32 child_lookahead_end_byte = self.ptr->padding.bytes + self.ptr->size.bytes + ts_subtree_lookahead_bytes(child);
if (child_lookahead_end_byte > lookahead_end_byte) if (child_lookahead_end_byte > lookahead_end_byte)
{ {
lookahead_end_byte = child_lookahead_end_byte; lookahead_end_byte = child_lookahead_end_byte;
@ -450,7 +448,7 @@ void ts_subtree_summarize_children(MutableSubtree self, const TSLanguage *langua
self.ptr->error_cost += ts_subtree_error_cost(child); self.ptr->error_cost += ts_subtree_error_cost(child);
} }
uint32_t grandchild_count = ts_subtree_child_count(child); t_u32 grandchild_count = ts_subtree_child_count(child);
if (self.ptr->symbol == ts_builtin_sym_error || self.ptr->symbol == ts_builtin_sym_error_repeat) if (self.ptr->symbol == ts_builtin_sym_error || self.ptr->symbol == ts_builtin_sym_error_repeat)
{ {
if (!ts_subtree_extra(child) && !(ts_subtree_is_error(child) && grandchild_count == 0)) if (!ts_subtree_extra(child) && !(ts_subtree_is_error(child) && grandchild_count == 0))
@ -552,7 +550,7 @@ MutableSubtree ts_subtree_new_node(TSSymbol symbol, SubtreeArray *children, unsi
if (children->capacity * sizeof(Subtree) < new_byte_size) if (children->capacity * sizeof(Subtree) < new_byte_size)
{ {
children->contents = mem_realloc(children->contents, new_byte_size); children->contents = mem_realloc(children->contents, new_byte_size);
children->capacity = (uint32_t)(new_byte_size / sizeof(Subtree)); children->capacity = (t_u32)(new_byte_size / sizeof(Subtree));
} }
SubtreeHeapData *data = (SubtreeHeapData *)&children->contents[children->size]; SubtreeHeapData *data = (SubtreeHeapData *)&children->contents[children->size];
@ -591,7 +589,7 @@ Subtree ts_subtree_new_error_node(SubtreeArray *children, bool extra, const TSLa
// //
// This node is treated as 'extra'. Its children are prevented from having // This node is treated as 'extra'. Its children are prevented from having
// having any effect on the parse state. // having any effect on the parse state.
Subtree ts_subtree_new_missing_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, uint32_t lookahead_bytes, Subtree ts_subtree_new_missing_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, t_u32 lookahead_bytes,
const TSLanguage *language) const TSLanguage *language)
{ {
Subtree result = ts_subtree_new_leaf(pool, symbol, padding, length_zero(), lookahead_bytes, 0, false, false, false, language); Subtree result = ts_subtree_new_leaf(pool, symbol, padding, length_zero(), lookahead_bytes, 0, false, false, false, language);
@ -611,7 +609,7 @@ void ts_subtree_retain(Subtree self)
if (self.data.is_inline) if (self.data.is_inline)
return; return;
assert(self.ptr->ref_count > 0); assert(self.ptr->ref_count > 0);
(*(uint32_t *)(&self.ptr->ref_count))++; (*(t_u32 *)(&self.ptr->ref_count))++;
assert(self.ptr->ref_count != 0); assert(self.ptr->ref_count != 0);
} }
@ -622,7 +620,7 @@ void ts_subtree_release(SubtreePool *pool, Subtree self)
array_clear(&pool->tree_stack); array_clear(&pool->tree_stack);
assert(self.ptr->ref_count > 0); assert(self.ptr->ref_count > 0);
if (--(*(uint32_t *)(&self.ptr->ref_count)) == 0) if (--(*(t_u32 *)(&self.ptr->ref_count)) == 0)
{ {
array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(self)); array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(self));
} }
@ -633,13 +631,13 @@ void ts_subtree_release(SubtreePool *pool, Subtree self)
if (tree.ptr->child_count > 0) if (tree.ptr->child_count > 0)
{ {
Subtree *children = ts_subtree_children(tree); Subtree *children = ts_subtree_children(tree);
for (uint32_t i = 0; i < tree.ptr->child_count; i++) for (t_u32 i = 0; i < tree.ptr->child_count; i++)
{ {
Subtree child = children[i]; Subtree child = children[i];
if (child.data.is_inline) if (child.data.is_inline)
continue; continue;
assert(child.ptr->ref_count > 0); assert(child.ptr->ref_count > 0);
if (--(*(uint32_t *)(&child.ptr->ref_count)) == 0) if (--(*(t_u32 *)(&child.ptr->ref_count)) == 0)
{ {
array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(child)); array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(child));
} }
@ -682,7 +680,7 @@ int ts_subtree_compare(Subtree left, Subtree right, SubtreePool *pool)
return result; return result;
} }
for (uint32_t i = ts_subtree_child_count(left); i > 0; i--) for (t_u32 i = ts_subtree_child_count(left); i > 0; i--)
{ {
Subtree left_child = ts_subtree_children(left)[i - 1]; Subtree left_child = ts_subtree_children(left)[i - 1];
Subtree right_child = ts_subtree_children(right)[i - 1]; Subtree right_child = ts_subtree_children(right)[i - 1];
@ -736,8 +734,8 @@ Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit, SubtreePool
Length size = ts_subtree_size(*entry.tree); Length size = ts_subtree_size(*entry.tree);
Length padding = ts_subtree_padding(*entry.tree); Length padding = ts_subtree_padding(*entry.tree);
Length total_size = length_add(padding, size); Length total_size = length_add(padding, size);
uint32_t lookahead_bytes = ts_subtree_lookahead_bytes(*entry.tree); t_u32 lookahead_bytes = ts_subtree_lookahead_bytes(*entry.tree);
uint32_t end_byte = total_size.bytes + lookahead_bytes; t_u32 end_byte = total_size.bytes + lookahead_bytes;
if (edit.start.bytes > end_byte || (is_noop && edit.start.bytes == end_byte)) if (edit.start.bytes > end_byte || (is_noop && edit.start.bytes == end_byte))
continue; continue;
@ -814,7 +812,7 @@ Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit, SubtreePool
*entry.tree = ts_subtree_from_mut(result); *entry.tree = ts_subtree_from_mut(result);
Length child_left, child_right = length_zero(); Length child_left, child_right = length_zero();
for (uint32_t i = 0, n = ts_subtree_child_count(*entry.tree); i < n; i++) for (t_u32 i = 0, n = ts_subtree_child_count(*entry.tree); i < n; i++)
{ {
Subtree *child = &ts_subtree_children(*entry.tree)[i]; Subtree *child = &ts_subtree_children(*entry.tree)[i];
Length child_size = ts_subtree_total_size(*child); Length child_size = ts_subtree_total_size(*child);
@ -874,7 +872,7 @@ Subtree ts_subtree_last_external_token(Subtree tree)
return NULL_SUBTREE; return NULL_SUBTREE;
while (tree.ptr->child_count > 0) while (tree.ptr->child_count > 0)
{ {
for (uint32_t i = tree.ptr->child_count - 1; i + 1 > 0; i--) for (t_u32 i = tree.ptr->child_count - 1; i + 1 > 0; i--)
{ {
Subtree child = ts_subtree_children(tree)[i]; Subtree child = ts_subtree_children(tree)[i];
if (ts_subtree_has_external_tokens(child)) if (ts_subtree_has_external_tokens(child))
@ -887,7 +885,7 @@ Subtree ts_subtree_last_external_token(Subtree tree)
return tree; return tree;
} }
static size_t ts_subtree__write_char_to_string(char *str, size_t n, int32_t chr) static size_t ts_subtree__write_char_to_string(char *str, size_t n, t_i32 chr)
{ {
if (chr == -1) if (chr == -1)
return snprintf(str, n, "INVALID"); return snprintf(str, n, "INVALID");
@ -981,8 +979,8 @@ static size_t ts_subtree__write_to_string(Subtree self, char *string, size_t lim
const TSFieldMapEntry *field_map, *field_map_end; const TSFieldMapEntry *field_map, *field_map_end;
ts_language_field_map(language, self.ptr->production_id, &field_map, &field_map_end); ts_language_field_map(language, self.ptr->production_id, &field_map, &field_map_end);
uint32_t structural_child_index = 0; t_u32 structural_child_index = 0;
for (uint32_t i = 0; i < self.ptr->child_count; i++) for (t_u32 i = 0; i < self.ptr->child_count; i++)
{ {
Subtree child = ts_subtree_children(self)[i]; Subtree child = ts_subtree_children(self)[i];
if (ts_subtree_extra(child)) if (ts_subtree_extra(child))
@ -1029,11 +1027,11 @@ char *ts_subtree_string(Subtree self, TSSymbol alias_symbol, bool alias_is_named
/* /*
void ts_subtree__print_dot_graph(const Subtree *self, uint32_t start_offset, const TSLanguage *language, TSSymbol alias_symbol, FILE *f) void ts_subtree__print_dot_graph(const Subtree *self, t_u32 start_offset, const TSLanguage *language, TSSymbol alias_symbol, FILE *f)
{ {
TSSymbol subtree_symbol = ts_subtree_symbol(*self); TSSymbol subtree_symbol = ts_subtree_symbol(*self);
TSSymbol symbol = alias_symbol ? alias_symbol : subtree_symbol; TSSymbol symbol = alias_symbol ? alias_symbol : subtree_symbol;
uint32_t end_offset = start_offset + ts_subtree_total_bytes(*self); t_u32 end_offset = start_offset + ts_subtree_total_bytes(*self);
fprintf(f, "tree_%p [label=\"", (void *)self); fprintf(f, "tree_%p [label=\"", (void *)self);
ts_language_write_symbol_as_dot_string(language, f, symbol); ts_language_write_symbol_as_dot_string(language, f, symbol);
fprintf(f, "\""); fprintf(f, "\"");
@ -1064,9 +1062,9 @@ void ts_subtree__print_dot_graph(const Subtree *self, uint32_t start_offset, con
fprintf(f, "\"]\n"); fprintf(f, "\"]\n");
uint32_t child_start_offset = start_offset; t_u32 child_start_offset = start_offset;
uint32_t child_info_offset = language->max_alias_sequence_length * ts_subtree_production_id(*self); t_u32 child_info_offset = language->max_alias_sequence_length * ts_subtree_production_id(*self);
for (uint32_t i = 0, n = ts_subtree_child_count(*self); i < n; i++) for (t_u32 i = 0, n = ts_subtree_child_count(*self); i < n; i++)
{ {
const Subtree *child = &ts_subtree_children(*self)[i]; const Subtree *child = &ts_subtree_children(*self)[i];
TSSymbol subtree_alias_symbol = 0; TSSymbol subtree_alias_symbol = 0;

81
parser/src/subtree.h
View file

@ -1,8 +1,9 @@
#ifndef TREE_SITTER_SUBTREE_H_ #ifndef TREE_SITTER_SUBTREE_H_
#define TREE_SITTER_SUBTREE_H_ #define TREE_SITTER_SUBTREE_H_
#include "me/types.h"
#include "./array.h" #include "./array.h"
#include "./error_costs.h"
#include "./length.h" #include "./length.h"
#include "./parser.h" #include "./parser.h"
#include "api.h" #include "api.h"
@ -31,7 +32,7 @@ typedef struct ExternalScannerState
char *long_data; char *long_data;
char short_data[24]; char short_data[24];
}; };
uint32_t length; t_u32 length;
} ExternalScannerState; } ExternalScannerState;
// A compact representation of a subtree. // A compact representation of a subtree.
@ -56,19 +57,19 @@ typedef struct SubtreeInlineData SubtreeInlineData;
bool is_keyword : 1; bool is_keyword : 1;
#define SUBTREE_SIZE \ #define SUBTREE_SIZE \
uint8_t padding_columns; \ t_u8 padding_columns; \
uint8_t padding_rows : 4; \ t_u8 padding_rows : 4; \
uint8_t lookahead_bytes : 4; \ t_u8 lookahead_bytes : 4; \
uint8_t padding_bytes; \ t_u8 padding_bytes; \
uint8_t size_bytes; t_u8 size_bytes;
#if TS_BIG_ENDIAN #if TS_BIG_ENDIAN
# if TS_PTR_SIZE == 32 # if TS_PTR_SIZE == 32
struct SubtreeInlineData struct SubtreeInlineData
{ {
uint16_t parse_state; t_u16 parse_state;
uint8_t symbol; t_u8 symbol;
SUBTREE_BITS SUBTREE_BITS
bool unused : 1; bool unused : 1;
bool is_inline : 1; bool is_inline : 1;
@ -80,8 +81,8 @@ struct SubtreeInlineData
struct SubtreeInlineData struct SubtreeInlineData
{ {
SUBTREE_SIZE SUBTREE_SIZE
uint16_t parse_state; t_u16 parse_state;
uint8_t symbol; t_u8 symbol;
SUBTREE_BITS SUBTREE_BITS
bool unused : 1; bool unused : 1;
bool is_inline : 1; bool is_inline : 1;
@ -94,8 +95,8 @@ struct SubtreeInlineData
{ {
bool is_inline : 1; bool is_inline : 1;
SUBTREE_BITS SUBTREE_BITS
uint8_t symbol; t_u8 symbol;
uint16_t parse_state; t_u16 parse_state;
SUBTREE_SIZE SUBTREE_SIZE
}; };
@ -111,12 +112,12 @@ struct SubtreeInlineData
// the inline representation. // the inline representation.
typedef struct SubtreeHeapData typedef struct SubtreeHeapData
{ {
volatile uint32_t ref_count; volatile t_u32 ref_count;
Length padding; Length padding;
Length size; Length size;
uint32_t lookahead_bytes; t_u32 lookahead_bytes;
uint32_t error_cost; t_u32 error_cost;
uint32_t child_count; t_u32 child_count;
TSSymbol symbol; TSSymbol symbol;
TSStateId parse_state; TSStateId parse_state;
@ -136,12 +137,12 @@ typedef struct SubtreeHeapData
// Non-terminal subtrees (`child_count > 0`) // Non-terminal subtrees (`child_count > 0`)
struct struct
{ {
uint32_t visible_child_count; t_u32 visible_child_count;
uint32_t named_child_count; t_u32 named_child_count;
uint32_t visible_descendant_count; t_u32 visible_descendant_count;
int32_t dynamic_precedence; t_i32 dynamic_precedence;
uint16_t repeat_depth; t_u16 repeat_depth;
uint16_t production_id; t_u16 production_id;
struct struct
{ {
TSSymbol symbol; TSSymbol symbol;
@ -153,7 +154,7 @@ typedef struct SubtreeHeapData
ExternalScannerState external_scanner_state; ExternalScannerState external_scanner_state;
// Error terminal subtrees (`child_count == 0 && symbol == ts_builtin_sym_error`) // Error terminal subtrees (`child_count == 0 && symbol == ts_builtin_sym_error`)
int32_t lookahead_char; t_i32 lookahead_char;
}; };
} SubtreeHeapData; } SubtreeHeapData;
@ -189,20 +190,20 @@ void ts_subtree_array_delete(SubtreePool *, SubtreeArray *);
void ts_subtree_array_remove_trailing_extras(SubtreeArray *, SubtreeArray *); void ts_subtree_array_remove_trailing_extras(SubtreeArray *, SubtreeArray *);
void ts_subtree_array_reverse(SubtreeArray *); void ts_subtree_array_reverse(SubtreeArray *);
SubtreePool ts_subtree_pool_new(uint32_t capacity); SubtreePool ts_subtree_pool_new(t_u32 capacity);
void ts_subtree_pool_delete(SubtreePool *); void ts_subtree_pool_delete(SubtreePool *);
Subtree ts_subtree_new_leaf(SubtreePool *, TSSymbol, Length, Length, uint32_t, TSStateId, bool, bool, bool, const TSLanguage *); Subtree ts_subtree_new_leaf(SubtreePool *, TSSymbol, Length, Length, t_u32, TSStateId, bool, bool, bool, const TSLanguage *);
Subtree ts_subtree_new_error(SubtreePool *, int32_t, Length, Length, uint32_t, TSStateId, const TSLanguage *); Subtree ts_subtree_new_error(SubtreePool *, t_i32, Length, Length, t_u32, TSStateId, const TSLanguage *);
MutableSubtree ts_subtree_new_node(TSSymbol, SubtreeArray *, unsigned, const TSLanguage *); MutableSubtree ts_subtree_new_node(TSSymbol, SubtreeArray *, unsigned, const TSLanguage *);
Subtree ts_subtree_new_error_node(SubtreeArray *, bool, const TSLanguage *); Subtree ts_subtree_new_error_node(SubtreeArray *, bool, const TSLanguage *);
Subtree ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, Length, uint32_t, const TSLanguage *); Subtree ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, Length, t_u32, const TSLanguage *);
MutableSubtree ts_subtree_make_mut(SubtreePool *, Subtree); MutableSubtree ts_subtree_make_mut(SubtreePool *, Subtree);
void ts_subtree_retain(Subtree); void ts_subtree_retain(Subtree);
void ts_subtree_release(SubtreePool *, Subtree); void ts_subtree_release(SubtreePool *, Subtree);
int ts_subtree_compare(Subtree, Subtree, SubtreePool *); int ts_subtree_compare(Subtree, Subtree, SubtreePool *);
void ts_subtree_set_symbol(MutableSubtree *, TSSymbol, const TSLanguage *); void ts_subtree_set_symbol(MutableSubtree *, TSSymbol, const TSLanguage *);
void ts_subtree_summarize(MutableSubtree, const Subtree *, uint32_t, const TSLanguage *); void ts_subtree_summarize(MutableSubtree, const Subtree *, t_u32, const TSLanguage *);
void ts_subtree_summarize_children(MutableSubtree, const TSLanguage *); void ts_subtree_summarize_children(MutableSubtree, const TSLanguage *);
void ts_subtree_balance(Subtree, SubtreePool *, const TSLanguage *); void ts_subtree_balance(Subtree, SubtreePool *, const TSLanguage *);
Subtree ts_subtree_edit(Subtree, const TSInputEdit *edit, SubtreePool *); Subtree ts_subtree_edit(Subtree, const TSInputEdit *edit, SubtreePool *);
@ -246,7 +247,7 @@ static inline TSStateId ts_subtree_parse_state(Subtree self)
{ {
return SUBTREE_GET(self, parse_state); return SUBTREE_GET(self, parse_state);
} }
static inline uint32_t ts_subtree_lookahead_bytes(Subtree self) static inline t_u32 ts_subtree_lookahead_bytes(Subtree self)
{ {
return SUBTREE_GET(self, lookahead_bytes); return SUBTREE_GET(self, lookahead_bytes);
} }
@ -255,7 +256,7 @@ static inline uint32_t ts_subtree_lookahead_bytes(Subtree self)
// Get the size needed to store a heap-allocated subtree with the given // Get the size needed to store a heap-allocated subtree with the given
// number of children. // number of children.
static inline size_t ts_subtree_alloc_size(uint32_t child_count) static inline size_t ts_subtree_alloc_size(t_u32 child_count)
{ {
return child_count * sizeof(Subtree) + sizeof(SubtreeHeapData); return child_count * sizeof(Subtree) + sizeof(SubtreeHeapData);
} }
@ -325,32 +326,32 @@ static inline Length ts_subtree_total_size(Subtree self)
return length_add(ts_subtree_padding(self), ts_subtree_size(self)); return length_add(ts_subtree_padding(self), ts_subtree_size(self));
} }
static inline uint32_t ts_subtree_total_bytes(Subtree self) static inline t_u32 ts_subtree_total_bytes(Subtree self)
{ {
return ts_subtree_total_size(self).bytes; return ts_subtree_total_size(self).bytes;
} }
static inline uint32_t ts_subtree_child_count(Subtree self) static inline t_u32 ts_subtree_child_count(Subtree self)
{ {
return self.data.is_inline ? 0 : self.ptr->child_count; return self.data.is_inline ? 0 : self.ptr->child_count;
} }
static inline uint32_t ts_subtree_repeat_depth(Subtree self) static inline t_u32 ts_subtree_repeat_depth(Subtree self)
{ {
return self.data.is_inline ? 0 : self.ptr->repeat_depth; return self.data.is_inline ? 0 : self.ptr->repeat_depth;
} }
static inline uint32_t ts_subtree_is_repetition(Subtree self) static inline t_u32 ts_subtree_is_repetition(Subtree self)
{ {
return self.data.is_inline ? 0 : !self.ptr->named && !self.ptr->visible && self.ptr->child_count != 0; return self.data.is_inline ? 0 : !self.ptr->named && !self.ptr->visible && self.ptr->child_count != 0;
} }
static inline uint32_t ts_subtree_visible_descendant_count(Subtree self) static inline t_u32 ts_subtree_visible_descendant_count(Subtree self)
{ {
return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->visible_descendant_count; return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->visible_descendant_count;
} }
static inline uint32_t ts_subtree_visible_child_count(Subtree self) static inline t_u32 ts_subtree_visible_child_count(Subtree self)
{ {
if (ts_subtree_child_count(self) > 0) if (ts_subtree_child_count(self) > 0)
{ {
@ -362,7 +363,7 @@ static inline uint32_t ts_subtree_visible_child_count(Subtree self)
} }
} }
static inline uint32_t ts_subtree_error_cost(Subtree self) static inline t_u32 ts_subtree_error_cost(Subtree self)
{ {
if (ts_subtree_missing(self)) if (ts_subtree_missing(self))
{ {
@ -374,12 +375,12 @@ static inline uint32_t ts_subtree_error_cost(Subtree self)
} }
} }
static inline int32_t ts_subtree_dynamic_precedence(Subtree self) static inline t_i32 ts_subtree_dynamic_precedence(Subtree self)
{ {
return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->dynamic_precedence; return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->dynamic_precedence;
} }
static inline uint16_t ts_subtree_production_id(Subtree self) static inline t_u16 ts_subtree_production_id(Subtree self)
{ {
if (ts_subtree_child_count(self) > 0) if (ts_subtree_child_count(self) > 0)
{ {

6
parser/src/tree.c
View file

@ -1,5 +1,7 @@
#define _POSIX_C_SOURCE 200112L #define _POSIX_C_SOURCE 200112L
#include "me/types.h"
#include "./tree.h" #include "./tree.h"
#include "./array.h" #include "./array.h"
#include "./length.h" #include "./length.h"
@ -41,7 +43,7 @@ TSNode ts_tree_root_node(const TSTree *self)
return ts_node_new(self, &self->root, ts_subtree_padding(self->root), 0); return ts_node_new(self, &self->root, ts_subtree_padding(self->root), 0);
} }
TSNode ts_tree_root_node_with_offset(const TSTree *self, uint32_t offset_bytes, TSPoint offset_extent) TSNode ts_tree_root_node_with_offset(const TSTree *self, t_u32 offset_bytes, TSPoint offset_extent)
{ {
Length offset = {offset_bytes, offset_extent}; Length offset = {offset_bytes, offset_extent};
return ts_node_new(self, &self->root, length_add(offset, ts_subtree_padding(self->root)), 0); return ts_node_new(self, &self->root, length_add(offset, ts_subtree_padding(self->root)), 0);
@ -97,7 +99,7 @@ void ts_tree_edit(TSTree *self, const TSInputEdit *edit)
ts_subtree_pool_delete(&pool); ts_subtree_pool_delete(&pool);
} }
TSRange *ts_tree_included_ranges(const TSTree *self, uint32_t *length) TSRange *ts_tree_included_ranges(const TSTree *self, t_u32 *length)
{ {
*length = self->included_range_count; *length = self->included_range_count;
TSRange *ranges = mem_alloc_array(self->included_range_count, sizeof(TSRange)); TSRange *ranges = mem_alloc_array(self->included_range_count, sizeof(TSRange));

2
parser/src/tree.h
View file

@ -1,6 +1,8 @@
#ifndef TREE_SITTER_TREE_H_ #ifndef TREE_SITTER_TREE_H_
#define TREE_SITTER_TREE_H_ #define TREE_SITTER_TREE_H_
#include "me/types.h"
#include "./subtree.h" #include "./subtree.h"
typedef struct ParentCacheEntry typedef struct ParentCacheEntry

20
parser/src/unicode.h
View file

@ -1,35 +1,35 @@
#ifndef TREE_SITTER_UNICODE_H_ #ifndef TREE_SITTER_UNICODE_H_
#define TREE_SITTER_UNICODE_H_ #define TREE_SITTER_UNICODE_H_
#include <stdint.h> #include "me/types.h"
static const int32_t TS_DECODE_ERROR = -1; static const t_i32 TS_DECODE_ERROR = -1;
// These functions read one unicode code point from the given string, // These functions read one unicode code point from the given string,
// returning the number of bytes consumed. // returning the number of bytes consumed.
typedef uint32_t (*UnicodeDecodeFunction)(const uint8_t *string, uint32_t length, int32_t *code_point); typedef t_u32 (*UnicodeDecodeFunction)(const t_u8 *string, t_u32 length, t_i32 *code_point);
static inline uint32_t ts_decode_ascii(const uint8_t *string, uint32_t length, int32_t *code_point) static inline t_u32 ts_decode_ascii(const t_u8 *string, t_u32 length, t_i32 *code_point)
{ {
(void)(length); (void)(length);
*code_point = 0; *code_point = 0;
*(uint8_t *)code_point = *string; *(t_u8 *)code_point = *string;
return (1); return (1);
} }
static inline uint32_t ts_decode_utf8(const uint8_t *string, uint32_t length, int32_t *code_point) static inline t_u32 ts_decode_utf8(const t_u8 *string, t_u32 length, t_i32 *code_point)
{ {
return (ts_decode_ascii(string, length, code_point)); return (ts_decode_ascii(string, length, code_point));
// uint32_t i = 0; // t_u32 i = 0;
// U8_NEXT(string, i, length, *code_point); // U8_NEXT(string, i, length, *code_point);
// return i; // return i;
} }
static inline uint32_t ts_decode_utf16(const uint8_t *string, uint32_t length, int32_t *code_point) static inline t_u32 ts_decode_utf16(const t_u8 *string, t_u32 length, t_i32 *code_point)
{ {
return (ts_decode_ascii(string, length, code_point)); return (ts_decode_ascii(string, length, code_point));
// uint32_t i = 0; // t_u32 i = 0;
// U16_NEXT(((uint16_t *)string), i, length, *code_point); // U16_NEXT(((t_u16 *)string), i, length, *code_point);
// return i * 2; // return i * 2;
} }