diff --git a/parser/src/api.h b/parser/src/api.h index 3dc444a7..dad717ae 100644 --- a/parser/src/api.h +++ b/parser/src/api.h @@ -73,7 +73,7 @@ typedef struct TSRange typedef struct TSInput { void *payload; - const char *(*read)(void *payload, t_u32 byte_index, TSPoint position, t_u32 *bytes_read); + const t_u8 *(*read)(void *payload, t_u32 byte_index, TSPoint position, t_u32 *bytes_read); TSInputEncoding encoding; } TSInput; @@ -86,7 +86,7 @@ typedef enum TSLogType typedef struct TSLogger { void *payload; - void (*log)(void *payload, TSLogType log_type, const char *buffer); + void (*log)(void *payload, TSLogType log_type, const t_u8 *buffer); } TSLogger; typedef struct TSInputEdit @@ -275,7 +275,7 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input); * above. The second two parameters indicate the location of the buffer and its * length in bytes. */ -TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, const char *string, t_u32 length); +TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, t_const_str string, t_u32 length); /** * Use the parser to parse some source code stored in one contiguous buffer with @@ -283,7 +283,7 @@ TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, const cha * [`ts_parser_parse_string`] method above. The final parameter indicates whether * the text is encoded as UTF8 or UTF16. */ -TSTree *ts_parser_parse_string_encoding(TSParser *self, const TSTree *old_tree, const char *string, t_u32 length, TSInputEncoding encoding); +TSTree *ts_parser_parse_string_encoding(TSParser *self, const TSTree *old_tree, t_const_str string, t_u32 length, TSInputEncoding encoding); /** * Instruct the parser to start the next parse from the beginning. @@ -423,7 +423,7 @@ void ts_tree_print_dot_graph(const TSTree *self, int file_descriptor); /** * Get the node's type as a null-terminated string. */ -const char *ts_node_type(TSNode self); +t_const_str ts_node_type(TSNode self); /** * Get the node's type as a numerical id. @@ -439,7 +439,7 @@ const TSLanguage *ts_node_language(TSNode self); * Get the node's type as it appears in the grammar ignoring aliases as a * null-terminated string. */ -const char *ts_node_grammar_type(TSNode self); +t_const_str ts_node_grammar_type(TSNode self); /** * Get the node's type as a numerical id as it appears in the grammar ignoring @@ -549,7 +549,7 @@ TSNode ts_node_child(TSNode self, t_u32 child_index); * Get the field name for node's child at the given index, where zero represents * the first child. Returns NULL, if no field is found. */ -const char *ts_node_field_name_for_child(TSNode self, t_u32 child_index); +t_const_str ts_node_field_name_for_child(TSNode self, t_u32 child_index); /** * Get the field name for node's child at the given index, where zero represents @@ -579,7 +579,7 @@ t_u32 ts_node_named_child_count(TSNode self); /** * Get the node's child with the given field name. */ -TSNode ts_node_child_by_field_name(TSNode self, const char *name, t_u32 name_length); +TSNode ts_node_child_by_field_name(TSNode self, t_const_str name, t_u32 name_length); /** * Get the node's child with the given numerical field id. @@ -674,12 +674,12 @@ t_u32 ts_language_state_count(const TSLanguage *self); /** * Get a node type string for the given numerical id. */ -const char *ts_language_symbol_name(const TSLanguage *self, TSSymbol symbol); +t_const_str ts_language_symbol_name(const TSLanguage *self, TSSymbol symbol); /** * Get the numerical id for the given node type string. */ -TSSymbol ts_language_symbol_for_name(const TSLanguage *self, const char *string, t_u32 length, bool is_named); +TSSymbol ts_language_symbol_for_name(const TSLanguage *self, t_const_str string, t_u32 length, bool is_named); /** * Get the number of distinct field names in the language. @@ -689,12 +689,12 @@ t_u32 ts_language_field_count(const TSLanguage *self); /** * Get the field name string for the given numerical id. */ -const char *ts_language_field_name_for_id(const TSLanguage *self, TSFieldId id); +t_const_str ts_language_field_name_for_id(const TSLanguage *self, TSFieldId id); /** * Get the numerical id for the given field name string. */ -TSFieldId ts_language_field_id_for_name(const TSLanguage *self, const char *name, t_u32 name_length); +TSFieldId ts_language_field_id_for_name(const TSLanguage *self, t_const_str name, t_u32 name_length); /** * Check whether the given node type id belongs to named nodes, anonymous nodes, diff --git a/parser/src/array.h b/parser/src/array.h index 3682d542..f8e3fcb1 100644 --- a/parser/src/array.h +++ b/parser/src/array.h @@ -115,7 +115,7 @@ #define array_insert_sorted_with(self, compare, value) \ do \ { \ - unsigned _index, _exists; \ + t_u32 _index, _exists; \ array_search_sorted_with(self, compare, &(value), &_index, &_exists); \ if (!_exists) \ array_insert(self, _index, value); \ @@ -128,7 +128,7 @@ #define array_insert_sorted_by(self, field, value) \ do \ { \ - unsigned _index, _exists; \ + t_u32 _index, _exists; \ array_search_sorted_by(self, field, (value)field, &_index, &_exists); \ if (!_exists) \ array_insert(self, _index, value); \ diff --git a/parser/src/create_language.c b/parser/src/create_language.c index e08b4613..28ef230e 100644 --- a/parser/src/create_language.c +++ b/parser/src/create_language.c @@ -36,8 +36,8 @@ void *create_small_parse_table(void); void *create_small_parse_table_map(void); bool ts_lex_keywords(TSLexer *lexer, TSStateId state); bool ts_lex(TSLexer *lexer, TSStateId state); -t_u32 tree_sitter_sh_external_scanner_serialize(void *ctx, char *s); -void tree_sitter_sh_external_scanner_deserialize(void *ctx, const char *s, t_u32 val); +t_u32 tree_sitter_sh_external_scanner_serialize(void *ctx, t_u8 *state); +void tree_sitter_sh_external_scanner_deserialize(void *ctx, const t_u8 *state, t_u32 val); void tree_sitter_sh_external_scanner_destroy(void *ctx); void *tree_sitter_sh_external_scanner_create(void); diff --git a/parser/src/language.c b/parser/src/language.c index 074735ab..494c9f6b 100644 --- a/parser/src/language.c +++ b/parser/src/language.c @@ -102,7 +102,7 @@ TSStateId ts_language_next_state(const TSLanguage *self, TSStateId state, TSSymb } } -const char *ts_language_symbol_name(const TSLanguage *self, TSSymbol symbol) +t_const_str ts_language_symbol_name(const TSLanguage *self, TSSymbol symbol) { if (symbol == ts_builtin_sym_error) { @@ -122,7 +122,7 @@ const char *ts_language_symbol_name(const TSLanguage *self, TSSymbol symbol) } } -TSSymbol ts_language_symbol_for_name(const TSLanguage *self, const char *string, t_u32 length, bool is_named) +TSSymbol ts_language_symbol_for_name(const TSLanguage *self, t_const_str string, t_u32 length, bool is_named) { if (!strncmp(string, "ERROR", length)) return ts_builtin_sym_error; @@ -132,7 +132,7 @@ TSSymbol ts_language_symbol_for_name(const TSLanguage *self, const char *string, TSSymbolMetadata metadata = ts_language_symbol_metadata(self, i); if ((!metadata.visible && !metadata.supertype) || metadata.named != is_named) continue; - const char *symbol_name = self->symbol_names[i]; + t_const_str symbol_name = self->symbol_names[i]; if (!strncmp(symbol_name, string, length) && !symbol_name[length]) { return self->public_symbol_map[i]; @@ -158,7 +158,7 @@ TSSymbolType ts_language_symbol_type(const TSLanguage *self, TSSymbol symbol) } } -const char *ts_language_field_name_for_id(const TSLanguage *self, TSFieldId id) +t_const_str ts_language_field_name_for_id(const TSLanguage *self, TSFieldId id) { t_u32 count = ts_language_field_count(self); if (count && id <= count) @@ -171,7 +171,7 @@ const char *ts_language_field_name_for_id(const TSLanguage *self, TSFieldId id) } } -TSFieldId ts_language_field_id_for_name(const TSLanguage *self, const char *name, t_u32 name_length) +TSFieldId ts_language_field_id_for_name(const TSLanguage *self, t_const_str name, t_u32 name_length) { t_u16 count = (t_u16)ts_language_field_count(self); for (TSSymbol i = 1; i < count + 1; i++) @@ -231,7 +231,7 @@ TSFieldId ts_language_field_id_for_name(const TSLanguage *self, const char *name // return iterator->symbol; // } -// const char *ts_lookahead_iterator_current_symbol_name(const TSLookaheadIterator *self) { +// t_const_str ts_lookahead_iterator_current_symbol_name(const TSLookaheadIterator *self) { // const LookaheadIterator *iterator = (const LookaheadIterator *)self; // return ts_language_symbol_name(iterator->language, iterator->symbol); // } diff --git a/parser/src/language.h b/parser/src/language.h index 00076a4b..64d52371 100644 --- a/parser/src/language.h +++ b/parser/src/language.h @@ -58,11 +58,11 @@ static inline t_u16 ts_language_lookup(const TSLanguage *self, TSStateId state, t_u32 index = self->small_parse_table_map[state - self->large_state_count]; const t_u16 *data = &self->small_parse_table[index]; t_u16 group_count = *(data++); - for (unsigned i = 0; i < group_count; i++) + for (t_u32 i = 0; i < group_count; i++) { t_u16 section_value = *(data++); t_u16 symbol_count = *(data++); - for (unsigned j = 0; j < symbol_count; j++) + for (t_u32 j = 0; j < symbol_count; j++) { if (*(data++) == symbol) return section_value; @@ -81,7 +81,7 @@ static inline bool ts_language_has_actions(const TSLanguage *self, TSStateId sta return ts_language_lookup(self, state, symbol) != 0; } -static inline const bool *ts_language_enabled_external_tokens(const TSLanguage *self, unsigned external_scanner_state) +static inline const bool *ts_language_enabled_external_tokens(const TSLanguage *self, t_u32 external_scanner_state) { if (external_scanner_state == 0) { @@ -124,7 +124,7 @@ static inline void ts_language_aliases_for_symbol(const TSLanguage *self, TSSymb *start = &self->public_symbol_map[original_symbol]; *end = *start + 1; - unsigned idx = 0; + t_u32 idx = 0; for (;;) { TSSymbol symbol = self->alias_map[idx++]; diff --git a/parser/src/lexer.c b/parser/src/lexer.c index 03e5f4d6..7fbe47c0 100644 --- a/parser/src/lexer.c +++ b/parser/src/lexer.c @@ -95,7 +95,7 @@ static void ts_lexer_goto(Lexer *self, Length position) // Move to the first valid position at or after the given position. bool found_included_range = false; - for (unsigned i = 0; i < self->included_range_count; i++) + for (t_u32 i = 0; i < self->included_range_count; i++) { TSRange *included_range = &self->included_ranges[i]; if (included_range->end_byte > self->current_position.bytes && included_range->end_byte > included_range->start_byte) @@ -416,7 +416,7 @@ bool ts_lexer_set_included_ranges(Lexer *self, const TSRange *ranges, t_u32 coun else { t_u32 previous_byte = 0; - for (unsigned i = 0; i < count; i++) + for (t_u32 i = 0; i < count; i++) { const TSRange *range = &ranges[i]; if (range->start_byte < previous_byte || range->end_byte < range->start_byte) diff --git a/parser/src/lexer.h b/parser/src/lexer.h index 04acf8a8..25a3340c 100644 --- a/parser/src/lexer.h +++ b/parser/src/lexer.h @@ -14,7 +14,7 @@ typedef struct Lexer Length token_end_position; TSRange *included_ranges; - const char *chunk; + const t_u8 *chunk; TSInput input; TSLogger logger; @@ -25,7 +25,7 @@ typedef struct Lexer t_u32 lookahead_size; bool did_get_column; - char debug_buffer[TREE_SITTER_SERIALIZATION_BUFFER_SIZE]; + t_u8 debug_buffer[TREE_SITTER_SERIALIZATION_BUFFER_SIZE]; } Lexer; void ts_lexer_init(Lexer *); diff --git a/parser/src/node.c b/parser/src/node.c index 4af56084..126d81a9 100644 --- a/parser/src/node.c +++ b/parser/src/node.c @@ -187,7 +187,7 @@ static inline TSNode ts_node__child(TSNode self, t_u32 child_index, bool include static bool ts_subtree_has_trailing_empty_descendant(Subtree self, Subtree other) { - for (unsigned i = ts_subtree_child_count(self) - 1; i + 1 > 0; i--) + for (t_u32 i = ts_subtree_child_count(self) - 1; i + 1 > 0; i--) { Subtree child = ts_subtree_children(self)[i]; if (ts_subtree_total_bytes(child) > 0) @@ -484,7 +484,7 @@ TSSymbol ts_node_symbol(TSNode self) return ts_language_public_symbol(self.tree->language, symbol); } -const char *ts_node_type(TSNode self) +t_const_str ts_node_type(TSNode self) { TSSymbol symbol = ts_node__alias(&self); if (!symbol) @@ -502,7 +502,7 @@ TSSymbol ts_node_grammar_symbol(TSNode self) return ts_subtree_symbol(ts_node__subtree(self)); } -const char *ts_node_grammar_type(TSNode self) +t_const_str ts_node_grammar_type(TSNode self) { TSSymbol symbol = ts_subtree_symbol(ts_node__subtree(self)); return ts_language_symbol_name(self.tree->language, symbol); @@ -712,7 +712,7 @@ recur: return ts_node__null(); } -static inline const char *ts_node__field_name_from_language(TSNode self, t_u32 structural_child_index) +static inline t_const_str ts_node__field_name_from_language(TSNode self, t_u32 structural_child_index) { const TSFieldMapEntry *field_map, *field_map_end; ts_language_field_map(self.tree->language, ts_node__subtree(self).ptr->production_id, &field_map, &field_map_end); @@ -726,11 +726,11 @@ static inline const char *ts_node__field_name_from_language(TSNode self, t_u32 s return NULL; } -const char *ts_node_field_name_for_child(TSNode self, t_u32 child_index) +t_const_str ts_node_field_name_for_child(TSNode self, t_u32 child_index) { TSNode result = self; bool did_descend = true; - const char *inherited_field_name = NULL; + t_const_str inherited_field_name = NULL; while (did_descend) { @@ -749,7 +749,7 @@ const char *ts_node_field_name_for_child(TSNode self, t_u32 child_index) { return NULL; } - const char *field_name = ts_node__field_name_from_language(result, iterator.structural_child_index - 1); + t_const_str field_name = ts_node__field_name_from_language(result, iterator.structural_child_index - 1); if (field_name) return field_name; return inherited_field_name; @@ -762,7 +762,7 @@ const char *ts_node_field_name_for_child(TSNode self, t_u32 child_index) t_u32 grandchild_count = ts_node__relevant_child_count(child, true); if (grandchild_index < grandchild_count) { - const char *field_name = ts_node__field_name_from_language(result, iterator.structural_child_index - 1); + t_const_str field_name = ts_node__field_name_from_language(result, iterator.structural_child_index - 1); if (field_name) inherited_field_name = field_name; @@ -779,7 +779,7 @@ const char *ts_node_field_name_for_child(TSNode self, t_u32 child_index) return NULL; } -TSNode ts_node_child_by_field_name(TSNode self, const char *name, t_u32 name_length) +TSNode ts_node_child_by_field_name(TSNode self, t_const_str name, t_u32 name_length) { TSFieldId field_id = ts_language_field_id_for_name(self.tree->language, name, name_length); return ts_node_child_by_field_id(self, field_id); @@ -884,7 +884,7 @@ void ts_node_edit(TSNode *self, const TSInputEdit *edit) TSSymbol ts_node_field_id_for_child(TSNode self, t_u32 child_index) { - const char *name = ts_node_field_name_for_child(self, child_index); + t_const_str name = ts_node_field_name_for_child(self, child_index); if (name == NULL) return (0); return (ts_language_field_id_for_name(ts_node_language(self), name, strlen(name))); diff --git a/parser/src/parser.c b/parser/src/parser.c index debb8bac..89765a0f 100644 --- a/parser/src/parser.c +++ b/parser/src/parser.c @@ -23,11 +23,11 @@ #define TREE_NAME(tree) SYM_NAME(ts_subtree_symbol(tree)) -static const unsigned MAX_VERSION_COUNT = 6; -static const unsigned MAX_VERSION_COUNT_OVERFLOW = 4; -static const unsigned MAX_SUMMARY_DEPTH = 16; -static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE; -static const unsigned OP_COUNT_PER_TIMEOUT_CHECK = 100; +static const t_u32 MAX_VERSION_COUNT = 6; +static const t_u32 MAX_VERSION_COUNT_OVERFLOW = 4; +static const t_u32 MAX_SUMMARY_DEPTH = 16; +static const t_u32 MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE; +static const t_u32 OP_COUNT_PER_TIMEOUT_CHECK = 100; typedef struct TokenCache { @@ -49,20 +49,20 @@ struct TSParser SubtreeArray scratch_trees; TokenCache token_cache; void *external_scanner_payload; - unsigned accept_count; - unsigned operation_count; + t_u32 accept_count; + t_u32 operation_count; const volatile size_t *cancellation_flag; Subtree old_tree; - unsigned included_range_difference_index; + t_u32 included_range_difference_index; bool has_scanner_error; }; typedef struct ErrorStatus { - unsigned cost; - unsigned node_count; - int dynamic_precedence; - bool is_in_error; + t_u32 cost; + t_u32 node_count; + int dynamic_precedence; + bool is_in_error; } ErrorStatus; typedef enum ErrorComparison @@ -76,20 +76,20 @@ typedef enum ErrorComparison typedef struct TSStringInput { - const char *string; + const t_u8 *string; t_u32 length; } TSStringInput; // StringInput -static const char *ts_string_input_read(void *_self, t_u32 byte, TSPoint point, t_u32 *length) +static const t_u8 *ts_string_input_read(void *_self, t_u32 byte, TSPoint point, t_u32 *length) { (void)point; TSStringInput *self = (TSStringInput *)_self; if (byte >= self->length) { *length = 0; - return ""; + return (const t_u8 *)""; } else { @@ -234,8 +234,8 @@ static ErrorComparison ts_parser__compare_versions(TSParser *self, ErrorStatus a static ErrorStatus ts_parser__version_status(TSParser *self, StackVersion version) { - unsigned cost = ts_stack_error_cost(self->stack, version); - bool is_paused = ts_stack_is_paused(self->stack, version); + t_u32 cost = ts_stack_error_cost(self->stack, version); + bool is_paused = ts_stack_is_paused(self->stack, version); if (is_paused) cost += ERROR_COST_PER_SKIPPED_TREE; return (ErrorStatus){.cost = cost, @@ -244,7 +244,7 @@ static ErrorStatus ts_parser__version_status(TSParser *self, StackVersion versio .is_in_error = is_paused || ts_stack_state(self->stack, version) == ERROR_STATE}; } -static bool ts_parser__better_version_exists(TSParser *self, StackVersion version, bool is_in_error, unsigned cost) +static bool ts_parser__better_version_exists(TSParser *self, StackVersion version, bool is_in_error, t_u32 cost) { if (self->finished_tree.ptr && ts_subtree_error_cost(self->finished_tree) <= cost) { @@ -313,7 +313,7 @@ static void ts_parser__external_scanner_destroy(TSParser *self) self->external_scanner_payload = NULL; } -static unsigned ts_parser__external_scanner_serialize(TSParser *self) +static t_u32 ts_parser__external_scanner_serialize(TSParser *self) { t_u32 length = self->language->external_scanner.serialize(self->external_scanner_payload, self->lexer.debug_buffer); @@ -323,7 +323,7 @@ static unsigned ts_parser__external_scanner_serialize(TSParser *self) static void ts_parser__external_scanner_deserialize(TSParser *self, Subtree external_token) { - const char *data = NULL; + const t_u8 *data = NULL; t_u32 length = 0; if (external_token.ptr) { @@ -829,7 +829,7 @@ static bool ts_parser__do_all_potential_reductions(TSParser *self, StackVersion bool can_shift_lookahead_symbol = false; StackVersion version = starting_version; - for (unsigned i = 0; true; i++) + for (t_u32 i = 0; true; i++) { t_u32 version_count = ts_stack_version_count(self->stack); if (version >= version_count) @@ -928,12 +928,12 @@ static bool ts_parser__do_all_potential_reductions(TSParser *self, StackVersion return can_shift_lookahead_symbol; } -static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, unsigned depth, TSStateId goal_state) +static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, t_u32 depth, TSStateId goal_state) { StackSliceArray pop = ts_stack_pop_count(self->stack, version, depth); StackVersion previous_version = STACK_VERSION_NONE; - for (unsigned i = 0; i < pop.size; i++) + for (t_u32 i = 0; i < pop.size; i++) { StackSlice slice = pop.contents[i]; @@ -961,7 +961,7 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un if (error_child_count > 0) { array_splice(&slice.subtrees, 0, 0, error_child_count, ts_subtree_children(error_tree)); - for (unsigned j = 0; j < error_child_count; j++) + for (t_u32 j = 0; j < error_child_count; j++) { ts_subtree_retain(slice.subtrees.contents[j]); } @@ -981,7 +981,7 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un array_delete(&slice.subtrees); } - for (unsigned j = 0; j < self->trailing_extras.size; j++) + for (t_u32 j = 0; j < self->trailing_extras.size; j++) { Subtree tree = self->trailing_extras.contents[j]; ts_stack_push(self->stack, slice.version, tree, false, goal_state); @@ -996,11 +996,11 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, un static void ts_parser__recover(TSParser *self, StackVersion version, Subtree lookahead) { bool did_recover = false; - unsigned previous_version_count = ts_stack_version_count(self->stack); + t_u32 previous_version_count = ts_stack_version_count(self->stack); Length position = ts_stack_position(self->stack, version); StackSummary *summary = ts_stack_get_summary(self->stack, version); - unsigned node_count_since_error = ts_stack_node_count_since_error(self->stack, version); - unsigned current_error_cost = ts_stack_error_cost(self->stack, version); + t_u32 node_count_since_error = ts_stack_node_count_since_error(self->stack, version); + t_u32 current_error_cost = ts_stack_error_cost(self->stack, version); // When the parser is in the error state, there are two strategies for recovering with a // given lookahead token: @@ -1016,7 +1016,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree loo // if the current lookahead token would be valid in that state. if (summary && !ts_subtree_is_error(lookahead)) { - for (unsigned i = 0; i < summary->size; i++) + for (t_u32 i = 0; i < summary->size; i++) { StackSummaryEntry entry = summary->contents[i]; @@ -1024,13 +1024,13 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree loo continue; if (entry.position.bytes == position.bytes) continue; - unsigned depth = entry.depth; + t_u32 depth = entry.depth; if (node_count_since_error > 0) depth++; // Do not recover in ways that create redundant stack versions. bool would_merge = false; - for (unsigned j = 0; j < previous_version_count; j++) + for (t_u32 j = 0; j < previous_version_count; j++) { if (ts_stack_state(self->stack, j) == entry.state && ts_stack_position(self->stack, j).bytes == position.bytes) { @@ -1042,9 +1042,9 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree loo continue; // Do not recover if the result would clearly be worse than some existing stack version. - unsigned new_cost = current_error_cost + entry.depth * ERROR_COST_PER_SKIPPED_TREE + - (position.bytes - entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR + - (position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE; + t_u32 new_cost = current_error_cost + entry.depth * ERROR_COST_PER_SKIPPED_TREE + + (position.bytes - entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR + + (position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE; if (ts_parser__better_version_exists(self, version, false, new_cost)) break; @@ -1065,7 +1065,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree loo // In the process of attempting to recover, some stack versions may have been created // and subsequently halted. Remove those versions. - for (unsigned i = previous_version_count; i < ts_stack_version_count(self->stack); i++) + for (t_u32 i = previous_version_count; i < ts_stack_version_count(self->stack); i++) { if (!ts_stack_is_active(self->stack, i)) { @@ -1105,8 +1105,8 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree loo } // Do not recover if the result would clearly be worse than some existing stack version. - unsigned new_cost = current_error_cost + ERROR_COST_PER_SKIPPED_TREE + ts_subtree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR + - ts_subtree_total_size(lookahead).extent.row * ERROR_COST_PER_SKIPPED_LINE; + t_u32 new_cost = current_error_cost + ERROR_COST_PER_SKIPPED_TREE + ts_subtree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR + + ts_subtree_total_size(lookahead).extent.row * ERROR_COST_PER_SKIPPED_LINE; if (ts_parser__better_version_exists(self, version, false, new_cost)) { ts_stack_halt(self->stack, version); @@ -1116,7 +1116,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree loo // If the current lookahead token is an extra token, mark it as extra. This means it won't // be counted in error cost calculations. - unsigned n; + t_u32 n; const TSParseAction *actions = ts_language_actions(self->language, 1, ts_subtree_symbol(lookahead), &n); if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].shift.extra) { @@ -1145,7 +1145,7 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree loo // arbitrarily and discard the rest. if (pop.size > 1) { - for (unsigned i = 1; i < pop.size; i++) + for (t_u32 i = 1; i < pop.size; i++) { ts_subtree_array_delete(&self->tree_pool, &pop.contents[i].subtrees); } @@ -1225,7 +1225,7 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version, Subtre v = (v == version) ? previous_version_count : v + 1; } - for (unsigned i = previous_version_count; i < version_count; i++) + for (t_u32 i = previous_version_count; i < version_count; i++) { bool did_merge = ts_stack_merge(self->stack, version, previous_version_count); assert(did_merge); @@ -1453,10 +1453,10 @@ static bool ts_parser__advance(TSParser *self, StackVersion version, bool allow_ } } -static unsigned ts_parser__condense_stack(TSParser *self) +static t_u32 ts_parser__condense_stack(TSParser *self) { - bool made_changes = false; - unsigned min_error_cost = UINT_MAX; + bool made_changes = false; + t_u32 min_error_cost = UINT_MAX; for (StackVersion i = 0; i < ts_stack_version_count(self->stack); i++) { // Prune any versions that have been marked for removal. @@ -1724,7 +1724,7 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) // After advancing each version of the stack, re-sort the versions by their cost, // removing any versions that are no longer worth pursuing. - unsigned min_error_cost = ts_parser__condense_stack(self); + t_u32 min_error_cost = ts_parser__condense_stack(self); // If there's already a finished parse tree that's better than any in-progress version, // then terminate parsing. Clear the parse stack to remove any extra references to subtrees @@ -1751,14 +1751,14 @@ exit: return result; } -TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, const char *string, t_u32 length) +TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, t_const_str string, t_u32 length) { return ts_parser_parse_string_encoding(self, old_tree, string, length, TSInputEncodingUTF8); } -TSTree *ts_parser_parse_string_encoding(TSParser *self, const TSTree *old_tree, const char *string, t_u32 length, TSInputEncoding encoding) +TSTree *ts_parser_parse_string_encoding(TSParser *self, const TSTree *old_tree, t_const_str string, t_u32 length, TSInputEncoding encoding) { - TSStringInput input = {string, length}; + TSStringInput input = {(const t_u8 *)string, length}; return ts_parser_parse(self, old_tree, (TSInput){ &input, diff --git a/parser/src/parser.h b/parser/src/parser.h index 3c9d60af..113e0a7c 100644 --- a/parser/src/parser.h +++ b/parser/src/parser.h @@ -111,8 +111,8 @@ struct TSLanguage const t_u16 *small_parse_table; const t_u32 *small_parse_table_map; const TSParseActionEntry *parse_actions; - const char *const *symbol_names; - const char *const *field_names; + t_const_str const *symbol_names; + t_const_str const *field_names; const TSFieldMapSlice *field_map_slices; const TSFieldMapEntry *field_map_entries; const TSSymbolMetadata *symbol_metadata; @@ -130,8 +130,8 @@ struct TSLanguage void *(*create)(void); void (*destroy)(void *); bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); - unsigned (*serialize)(void *, char *); - void (*deserialize)(void *, const char *, unsigned); + t_u32 (*serialize)(void *, t_u8 *); + void (*deserialize)(void *, const t_u8 *, t_u32); } external_scanner; const TSStateId *primary_state_ids; }; diff --git a/parser/src/point.h b/parser/src/point.h index ef73804c..f8924044 100644 --- a/parser/src/point.h +++ b/parser/src/point.h @@ -7,7 +7,7 @@ #define POINT_ZERO ((TSPoint){0, 0}) #define POINT_MAX ((TSPoint){UINT32_MAX, UINT32_MAX}) -static inline TSPoint point__new(unsigned row, unsigned column) +static inline TSPoint point__new(t_u32 row, t_u32 column) { TSPoint result = {row, column}; return result; diff --git a/parser/src/reduce_action.h b/parser/src/reduce_action.h index eecd7b79..1df0b730 100644 --- a/parser/src/reduce_action.h +++ b/parser/src/reduce_action.h @@ -10,7 +10,7 @@ typedef struct ReduceAction t_u32 count; TSSymbol symbol; int dynamic_precedence; - unsigned short production_id; + t_u16 production_id; } ReduceAction; typedef Array(ReduceAction) ReduceActionSet; diff --git a/parser/src/scanner.c b/parser/src/scanner.c index 9e5a1112..2f1b2e19 100644 --- a/parser/src/scanner.c +++ b/parser/src/scanner.c @@ -97,7 +97,7 @@ static inline void reset(Scanner *scanner) } } -static unsigned serialize(Scanner *scanner, char *buffer) +static t_u32 serialize(Scanner *scanner, t_u8 *buffer) { t_u32 size = 0; @@ -129,7 +129,7 @@ static unsigned serialize(Scanner *scanner, char *buffer) return size; } -static void deserialize(Scanner *scanner, const char *buffer, unsigned length) +static void deserialize(Scanner *scanner, const t_u8 *buffer, t_u32 length) { if (length == 0) { @@ -141,7 +141,7 @@ static void deserialize(Scanner *scanner, const char *buffer, unsigned length) scanner->last_glob_paren_depth = buffer[size++]; scanner->ext_was_in_double_quote = buffer[size++]; scanner->ext_saw_outside_quote = buffer[size++]; - t_u32 heredoc_count = (unsigned char)buffer[size++]; + t_u32 heredoc_count = (t_u8)buffer[size++]; for (t_u32 i = 0; i < heredoc_count; i++) { Heredoc *heredoc = NULL; @@ -1172,13 +1172,13 @@ bool tree_sitter_sh_external_scanner_scan(void *payload, TSLexer *lexer, const b return scan(scanner, lexer, valid_symbols); } -unsigned tree_sitter_sh_external_scanner_serialize(void *payload, char *state) +t_u32 tree_sitter_sh_external_scanner_serialize(void *payload, t_u8 *state) { Scanner *scanner = (Scanner *)payload; return serialize(scanner, state); } -void tree_sitter_sh_external_scanner_deserialize(void *payload, const char *state, unsigned length) +void tree_sitter_sh_external_scanner_deserialize(void *payload, const t_u8 *state, t_u32 length) { Scanner *scanner = (Scanner *)payload; deserialize(scanner, state, length); diff --git a/parser/src/stack.c b/parser/src/stack.c index 719e816f..418bab2c 100644 --- a/parser/src/stack.c +++ b/parser/src/stack.c @@ -27,10 +27,10 @@ struct StackNode TSStateId state; Length position; StackLink links[MAX_LINK_COUNT]; - short unsigned int link_count; + t_u16 link_count; t_u32 ref_count; - unsigned error_cost; - unsigned node_count; + t_u32 error_cost; + t_u32 node_count; int dynamic_precedence; }; @@ -55,7 +55,7 @@ typedef struct StackHead { StackNode *node; StackSummary *summary; - unsigned node_count_at_last_error; + t_u32 node_count_at_last_error; Subtree last_external_token; Subtree lookahead_when_paused; StackStatus status; @@ -71,7 +71,7 @@ struct Stack SubtreePool *subtree_pool; }; -typedef unsigned StackAction; +typedef t_u32 StackAction; enum StackAction { StackActionNone, @@ -101,7 +101,7 @@ recur: StackNode *first_predecessor = NULL; if (self->link_count > 0) { - for (unsigned i = self->link_count - 1; i > 0; i--) + for (t_u32 i = self->link_count - 1; i > 0; i--) { StackLink link = self->links[i]; if (link.subtree.ptr) @@ -255,7 +255,7 @@ static void stack_node_add_link(StackNode *self, StackLink link, SubtreePool *su return; stack_node_retain(link.node); - unsigned node_count = link.node->node_count; + t_u32 node_count = link.node->node_count; int dynamic_precedence = link.node->dynamic_precedence; self->links[self->link_count++] = link; @@ -504,10 +504,10 @@ void ts_stack_set_last_external_token(Stack *self, StackVersion version, Subtree head->last_external_token = token; } -unsigned ts_stack_error_cost(const Stack *self, StackVersion version) +t_u32 ts_stack_error_cost(const Stack *self, StackVersion version) { StackHead *head = array_get(&self->heads, version); - unsigned result = head->node->error_cost; + t_u32 result = head->node->error_cost; if (head->status == StackStatusPaused || (head->node->state == ERROR_STATE && !head->node->links[0].subtree.ptr)) { result += ERROR_COST_PER_RECOVERY; @@ -515,7 +515,7 @@ unsigned ts_stack_error_cost(const Stack *self, StackVersion version) return result; } -unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version) +t_u32 ts_stack_node_count_since_error(const Stack *self, StackVersion version) { StackHead *head = array_get(&self->heads, version); if (head->node->node_count < head->node_count_at_last_error) @@ -536,7 +536,7 @@ void ts_stack_push(Stack *self, StackVersion version, Subtree subtree, bool pend StackAction pop_count_callback(void *payload, const StackIterator *iterator) { - unsigned *goal_subtree_count = payload; + t_u32 *goal_subtree_count = payload; if (iterator->subtree_count == *goal_subtree_count) { return StackActionPop | StackActionStop; @@ -607,7 +607,7 @@ StackAction pop_error_callback(void *payload, const StackIterator *iterator) SubtreeArray ts_stack_pop_error(Stack *self, StackVersion version) { StackNode *node = array_get(&self->heads, version)->node; - for (unsigned i = 0; i < node->link_count; i++) + for (t_u32 i = 0; i < node->link_count; i++) { if (node->links[i].subtree.ptr && ts_subtree_is_error(node->links[i].subtree)) { @@ -639,17 +639,17 @@ StackSliceArray ts_stack_pop_all(Stack *self, StackVersion version) typedef struct SummarizeStackSession { StackSummary *summary; - unsigned max_depth; + t_u32 max_depth; } SummarizeStackSession; StackAction summarize_stack_callback(void *payload, const StackIterator *iterator) { SummarizeStackSession *session = payload; TSStateId state = iterator->node->state; - unsigned depth = iterator->subtree_count; + t_u32 depth = iterator->subtree_count; if (depth > session->max_depth) return StackActionStop; - for (unsigned i = session->summary->size - 1; i + 1 > 0; i--) + for (t_u32 i = session->summary->size - 1; i + 1 > 0; i--) { StackSummaryEntry entry = session->summary->contents[i]; if (entry.depth < depth) @@ -665,7 +665,7 @@ StackAction summarize_stack_callback(void *payload, const StackIterator *iterato return StackActionNone; } -void ts_stack_record_summary(Stack *self, StackVersion version, unsigned max_depth) +void ts_stack_record_summary(Stack *self, StackVersion version, t_u32 max_depth) { SummarizeStackSession session = {.summary = mem_alloc(sizeof(StackSummary)), .max_depth = max_depth}; array_init(session.summary); diff --git a/parser/src/stack.h b/parser/src/stack.h index 257e8c1b..18faa3bb 100644 --- a/parser/src/stack.h +++ b/parser/src/stack.h @@ -8,7 +8,7 @@ typedef struct Stack Stack; -typedef unsigned StackVersion; +typedef t_u32 StackVersion; #define STACK_VERSION_NONE ((StackVersion)-1) typedef struct StackSlice @@ -21,7 +21,7 @@ typedef Array(StackSlice) StackSliceArray; typedef struct StackSummaryEntry { Length position; - unsigned depth; + t_u32 depth; TSStateId state; } StackSummaryEntry; @@ -76,7 +76,7 @@ StackSliceArray ts_stack_pop_all(Stack *, StackVersion); // Get the maximum number of tree nodes reachable from this version of the stack // since the last error was detected. -unsigned ts_stack_node_count_since_error(const Stack *, StackVersion); +t_u32 ts_stack_node_count_since_error(const Stack *, StackVersion); int ts_stack_dynamic_precedence(Stack *, StackVersion); @@ -84,14 +84,14 @@ bool ts_stack_has_advanced_since_error(const Stack *, StackVersion); // Compute a summary of all the parse states near the top of the given // version of the stack and store the summary for later retrieval. -void ts_stack_record_summary(Stack *, StackVersion, unsigned max_depth); +void ts_stack_record_summary(Stack *, StackVersion, t_u32 max_depth); // Retrieve a summary of all the parse states near the top of the // given version of the stack. StackSummary *ts_stack_get_summary(Stack *, StackVersion); // Get the total cost of all errors on the given version of the stack. -unsigned ts_stack_error_cost(const Stack *, StackVersion version); +t_u32 ts_stack_error_cost(const Stack *, StackVersion version); // Merge the given two stack versions if possible, returning true // if they were successfully merged and false otherwise. diff --git a/parser/src/subtree.c b/parser/src/subtree.c index 75c7a8b0..1b7a021e 100644 --- a/parser/src/subtree.c +++ b/parser/src/subtree.c @@ -23,7 +23,7 @@ typedef struct // ExternalScannerState -void ts_external_scanner_state_init(ExternalScannerState *self, const char *data, unsigned length) +void ts_external_scanner_state_init(ExternalScannerState *self, const t_u8 *data, t_u32 length) { self->length = length; if (length > sizeof(self->short_data)) @@ -56,19 +56,19 @@ void ts_external_scanner_state_delete(ExternalScannerState *self) } } -const char *ts_external_scanner_state_data(const ExternalScannerState *self) +const t_u8 *ts_external_scanner_state_data(const ExternalScannerState *self) { if (self->length > sizeof(self->short_data)) { - return self->long_data; + return (const t_u8 *)self->long_data; } else { - return self->short_data; + return (const t_u8 *)self->short_data; } } -bool ts_external_scanner_state_eq(const ExternalScannerState *self, const char *buffer, unsigned length) +bool ts_external_scanner_state_eq(const ExternalScannerState *self, const t_u8 *buffer, t_u32 length) { return self->length == length && memcmp(ts_external_scanner_state_data(self), buffer, length) == 0; } @@ -149,7 +149,7 @@ void ts_subtree_pool_delete(SubtreePool *self) { if (self->free_trees.contents) { - for (unsigned i = 0; i < self->free_trees.size; i++) + for (t_u32 i = 0; i < self->free_trees.size; i++) { mem_free(self->free_trees.contents[i].ptr); } @@ -315,13 +315,13 @@ MutableSubtree ts_subtree_make_mut(SubtreePool *pool, Subtree self) return result; } -static void ts_subtree__compress(MutableSubtree self, unsigned count, const TSLanguage *language, MutableSubtreeArray *stack) +static void ts_subtree__compress(MutableSubtree self, t_u32 count, const TSLanguage *language, MutableSubtreeArray *stack) { - unsigned initial_stack_size = stack->size; + t_u32 initial_stack_size = stack->size; MutableSubtree tree = self; TSSymbol symbol = tree.ptr->symbol; - for (unsigned i = 0; i < count; i++) + for (t_u32 i = 0; i < count; i++) { if (tree.ptr->ref_count > 1 || tree.ptr->child_count < 2) break; @@ -373,8 +373,8 @@ void ts_subtree_balance(Subtree self, SubtreePool *pool, const TSLanguage *langu long repeat_delta = (long)ts_subtree_repeat_depth(child1) - (long)ts_subtree_repeat_depth(child2); if (repeat_delta > 0) { - unsigned n = (unsigned)repeat_delta; - for (unsigned i = n / 2; i > 0; i /= 2) + t_u32 n = (t_u32)repeat_delta; + for (t_u32 i = n / 2; i > 0; i /= 2) { ts_subtree__compress(tree, i, language, &pool->tree_stack); n -= i; @@ -540,7 +540,7 @@ void ts_subtree_summarize_children(MutableSubtree self, const TSLanguage *langua // Create a new parent node with the given children. // // This takes ownership of the children array. -MutableSubtree ts_subtree_new_node(TSSymbol symbol, SubtreeArray *children, unsigned production_id, const TSLanguage *language) +MutableSubtree ts_subtree_new_node(TSSymbol symbol, SubtreeArray *children, t_u32 production_id, const TSLanguage *language) { TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); bool fragile = symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat; @@ -903,10 +903,10 @@ static size_t ts_subtree__write_char_to_string(char *str, size_t n, t_i32 chr) return snprintf(str, n, "%d", chr); } -static const char *const ROOT_FIELD = "__ROOT__"; +static t_const_str const ROOT_FIELD = "__ROOT__"; static size_t ts_subtree__write_to_string(Subtree self, char *string, size_t limit, const TSLanguage *language, bool include_all, - TSSymbol alias_symbol, bool alias_is_named, const char *field_name) + TSSymbol alias_symbol, bool alias_is_named, t_const_str field_name) { if (!self.ptr) return snprintf(string, limit, "(NULL)"); @@ -936,7 +936,7 @@ static size_t ts_subtree__write_to_string(Subtree self, char *string, size_t lim else { TSSymbol symbol = alias_symbol ? alias_symbol : ts_subtree_symbol(self); - const char *symbol_name = ts_language_symbol_name(language, symbol); + t_const_str symbol_name = ts_language_symbol_name(language, symbol); if (ts_subtree_missing(self)) { cursor += snprintf(*writer, limit, "(MISSING "); @@ -958,7 +958,7 @@ static size_t ts_subtree__write_to_string(Subtree self, char *string, size_t lim else if (is_root) { TSSymbol symbol = alias_symbol ? alias_symbol : ts_subtree_symbol(self); - const char *symbol_name = ts_language_symbol_name(language, symbol); + t_const_str symbol_name = ts_language_symbol_name(language, symbol); if (ts_subtree_child_count(self) > 0) { cursor += snprintf(*writer, limit, "(%s", symbol_name); @@ -993,7 +993,7 @@ static size_t ts_subtree__write_to_string(Subtree self, char *string, size_t lim bool subtree_alias_is_named = subtree_alias_symbol ? ts_language_symbol_metadata(language, subtree_alias_symbol).named : false; - const char *child_field_name = is_visible ? NULL : field_name; + t_const_str child_field_name = is_visible ? NULL : field_name; for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) { if (!map->inherited && map->child_index == structural_child_index) diff --git a/parser/src/subtree.h b/parser/src/subtree.h index c74c86de..b14b60a2 100644 --- a/parser/src/subtree.h +++ b/parser/src/subtree.h @@ -179,9 +179,9 @@ typedef struct SubtreePool MutableSubtreeArray tree_stack; } SubtreePool; -void ts_external_scanner_state_init(ExternalScannerState *, const char *, unsigned); -const char *ts_external_scanner_state_data(const ExternalScannerState *); -bool ts_external_scanner_state_eq(const ExternalScannerState *self, const char *, unsigned); +void ts_external_scanner_state_init(ExternalScannerState *, const t_u8 *, t_u32); +const t_u8 *ts_external_scanner_state_data(const ExternalScannerState *); +bool ts_external_scanner_state_eq(const ExternalScannerState *self, const t_u8 *, t_u32); void ts_external_scanner_state_delete(ExternalScannerState *self); void ts_subtree_array_copy(SubtreeArray, SubtreeArray *); @@ -195,7 +195,7 @@ void ts_subtree_pool_delete(SubtreePool *); Subtree ts_subtree_new_leaf(SubtreePool *, TSSymbol, Length, Length, t_u32, TSStateId, bool, bool, bool, const TSLanguage *); Subtree ts_subtree_new_error(SubtreePool *, t_i32, Length, Length, t_u32, TSStateId, const TSLanguage *); -MutableSubtree ts_subtree_new_node(TSSymbol, SubtreeArray *, unsigned, const TSLanguage *); +MutableSubtree ts_subtree_new_node(TSSymbol, SubtreeArray *, t_u32, const TSLanguage *); Subtree ts_subtree_new_error_node(SubtreeArray *, bool, const TSLanguage *); Subtree ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, Length, t_u32, const TSLanguage *); MutableSubtree ts_subtree_make_mut(SubtreePool *, Subtree); diff --git a/parser/src/tree.c b/parser/src/tree.c index c1f48e15..7adf8b11 100644 --- a/parser/src/tree.c +++ b/parser/src/tree.c @@ -8,7 +8,7 @@ #include "./subtree.h" #include "api.h" -TSTree *ts_tree_new(Subtree root, const TSLanguage *language, const TSRange *included_ranges, unsigned included_range_count) +TSTree *ts_tree_new(Subtree root, const TSLanguage *language, const TSRange *included_ranges, t_u32 included_range_count) { TSTree *result = mem_alloc(sizeof(TSTree)); result->root = root; @@ -56,7 +56,7 @@ const TSLanguage *ts_tree_language(const TSTree *self) void ts_tree_edit(TSTree *self, const TSInputEdit *edit) { - for (unsigned i = 0; i < self->included_range_count; i++) + for (t_u32 i = 0; i < self->included_range_count; i++) { TSRange *range = &self->included_ranges[i]; if (range->end_byte >= edit->old_end_byte) diff --git a/parser/src/tree.h b/parser/src/tree.h index d4b8487d..c8f68723 100644 --- a/parser/src/tree.h +++ b/parser/src/tree.h @@ -18,10 +18,10 @@ struct TSTree Subtree root; const TSLanguage *language; TSRange *included_ranges; - unsigned included_range_count; + t_u32 included_range_count; }; -TSTree *ts_tree_new(Subtree root, const TSLanguage *language, const TSRange *, unsigned); +TSTree *ts_tree_new(Subtree root, const TSLanguage *language, const TSRange *, t_u32); TSNode ts_node_new(const TSTree *, const Subtree *, Length, TSSymbol); #endif // TREE_SITTER_TREE_H_