From 1b2f8c62620ba26137c4c4d80b7b65b8e46ec789 Mon Sep 17 00:00:00 2001 From: Maieul BOYER Date: Tue, 3 Sep 2024 14:06:38 +0200 Subject: [PATCH] parser.c --- parser/src/parser.c | 592 ++++++++++++++++++++++++++------------------ 1 file changed, 353 insertions(+), 239 deletions(-) diff --git a/parser/src/parser.c b/parser/src/parser.c index 70fc1d37..eac3cb56 100644 --- a/parser/src/parser.c +++ b/parser/src/parser.c @@ -6,7 +6,7 @@ /* By: maiboyer +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/09/02 20:48:07 by maiboyer #+# #+# */ -/* Updated: 2024/09/02 21:14:40 by maiboyer ### ########.fr */ +/* Updated: 2024/09/03 14:06:04 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -99,26 +99,37 @@ static const t_u8 *ts_string_input_read(void *_self, t_u32 byte, TSPoint point, // Parser - Private static bool ts_parser__breakdown_top_of_stack(TSParser *self, t_stack_version version) { - bool did_break_down = false; - bool pending = false; + t_stack_slice_array pop; + bool did_break_down; + bool pending; + did_break_down = false; + pending = false; + t_stack_slice slice; + TSStateId state; + t_subtree parent; + t_subtree child; + t_subtree tree; + t_u32 i; + t_u32 n; + t_u32 j; do { - t_stack_slice_array pop = ts_stack_pop_pending(self->stack, version); + pop = ts_stack_pop_pending(self->stack, version); if (!pop.size) break; did_break_down = true; pending = false; - for (t_u32 i = 0; i < pop.size; i++) + for (i = 0; i < pop.size; i++) { - t_stack_slice slice = pop.contents[i]; - TSStateId state = ts_stack_state(self->stack, slice.version); - t_subtree parent = *slice.subtrees.buffer; + slice = pop.contents[i]; + state = ts_stack_state(self->stack, slice.version); + parent = *slice.subtrees.buffer; - for (t_u32 j = 0, n = ts_subtree_child_count(parent); j < n; j++) + for (j = 0, n = ts_subtree_child_count(parent); j < n; j++) { - t_subtree child = ts_subtree_children(parent)[j]; + child = ts_subtree_children(parent)[j]; pending = ts_subtree_child_count(child) > 0; if (ts_subtree_is_error(child)) @@ -130,17 +141,17 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self, t_stack_version ve state = ts_language_next_state(self->language, state, ts_subtree_symbol(child)); } - (child->ref_count++); + child->ref_count++; ts_stack_push(self->stack, slice.version, child, pending, state); } - for (t_u32 j = 1; j < slice.subtrees.len; j++) + for (j = 1; j < slice.subtrees.len; j++) { - t_subtree tree = slice.subtrees.buffer[j]; + tree = slice.subtrees.buffer[j]; ts_stack_push(self->stack, slice.version, tree, false, state); } - ts_subtree_release(/*&self->tree_pool,*/ parent); + ts_subtree_release(parent); array_delete(&slice.subtrees); } } while (pending); @@ -208,8 +219,10 @@ static t_error_comparison ts_parser__compare_versions(TSParser *self, t_error_st static t_error_status ts_parser__version_status(TSParser *self, t_stack_version version) { - t_u32 cost = ts_stack_error_cost(self->stack, version); - bool is_paused = ts_stack_is_paused(self->stack, version); + t_u32 cost; + bool is_paused; + cost = ts_stack_error_cost(self->stack, version); + is_paused = ts_stack_is_paused(self->stack, version); if (is_paused) cost += ERROR_COST_PER_SKIPPED_TREE; return (t_error_status){.cost = cost, @@ -220,24 +233,28 @@ static t_error_status ts_parser__version_status(TSParser *self, t_stack_version static bool ts_parser__better_version_exists(TSParser *self, t_stack_version version, bool is_in_error, t_u32 cost) { - if (self->finished_tree && ts_subtree_error_cost(self->finished_tree) <= cost) - { - return true; - } + t_error_status status_i; + Length position; + t_error_status status; + t_stack_version i; + t_stack_version n; - Length position = ts_stack_position(self->stack, version); - t_error_status status = { + if (self->finished_tree && ts_subtree_error_cost(self->finished_tree) <= cost) + return true; + + position = ts_stack_position(self->stack, version); + status = (t_error_status){ .cost = cost, .is_in_error = is_in_error, .dynamic_precedence = ts_stack_dynamic_precedence(self->stack, version), .node_count = ts_stack_node_count_since_error(self->stack, version), }; - for (t_stack_version i = 0, n = ts_stack_version_count(self->stack); i < n; i++) + for (i = 0, n = ts_stack_version_count(self->stack); i < n; i++) { if (i == version || !ts_stack_is_active(self->stack, i) || ts_stack_position(self->stack, i).bytes < position.bytes) continue; - t_error_status status_i = ts_parser__version_status(self, i); + status_i = ts_parser__version_status(self, i); switch (ts_parser__compare_versions(self, status, status_i)) { case ECTakeRight: @@ -268,29 +285,22 @@ static bool ts_parser__call_keyword_lex_fn(TSParser *self, TSLexMode lex_mode) static void ts_parser__external_scanner_create(TSParser *self) { - if (self->language && self->language->external_scanner.states) - { - - if (self->language->external_scanner.create) - { - self->external_scanner_payload = self->language->external_scanner.create(); - } - } + if (self->language && self->language->external_scanner.states && self->language->external_scanner.create) + self->external_scanner_payload = self->language->external_scanner.create(); } static void ts_parser__external_scanner_destroy(TSParser *self) { if (self->language && self->external_scanner_payload && self->language->external_scanner.destroy) - { self->language->external_scanner.destroy(self->external_scanner_payload); - } self->external_scanner_payload = NULL; } static t_u32 ts_parser__external_scanner_serialize(TSParser *self) { - t_u32 length = self->language->external_scanner.serialize(self->external_scanner_payload, self->lexer.debug_buffer); + t_u32 length; + length = self->language->external_scanner.serialize(self->external_scanner_payload, self->lexer.debug_buffer); if (length > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) me_abort("assertion failed in " __FILE__ " `length > TREE_SITTER_SERIALIZATION_BUFFER_SIZE`"); return length; @@ -298,8 +308,11 @@ static t_u32 ts_parser__external_scanner_serialize(TSParser *self) static void ts_parser__external_scanner_deserialize(TSParser *self, t_subtree external_token) { - const t_u8 *data = NULL; - t_u32 length = 0; + const t_u8 *data; + t_u32 length; + + data = NULL; + length = 0; if (external_token) { data = ts_external_scanner_state_data(&external_token->external_scanner_state); @@ -311,35 +324,59 @@ static void ts_parser__external_scanner_deserialize(TSParser *self, t_subtree ex static bool ts_parser__external_scanner_scan(TSParser *self, TSStateId external_lex_state) { - const bool *valid_external_tokens = ts_language_enabled_external_tokens(self->language, external_lex_state); + const bool *valid_external_tokens; + valid_external_tokens = ts_language_enabled_external_tokens(self->language, external_lex_state); return self->language->external_scanner.scan(self->external_scanner_payload, &self->lexer.data, valid_external_tokens); } static t_subtree ts_parser__lex(TSParser *self, t_stack_version version, TSStateId parse_state) { - TSLexMode lex_mode = self->language->lex_modes[parse_state]; + Length start_position; + t_subtree external_token; + TSLexMode lex_mode; + bool found_external_token; + bool error_mode; + bool skipped_error; + bool called_get_column; + t_i32 first_error_character; + Length error_start_position; + Length error_end_position; + t_u32 lookahead_end_byte; + t_u32 external_scanner_state_len; + bool external_scanner_state_changed; + bool found_token; + Length current_position; + t_subtree result; + Length padding; + Length size; + t_u32 lookahead_bytes; + bool is_keyword; + TSSymbol symbol; + t_u32 end_byte; + + lex_mode = self->language->lex_modes[parse_state]; if (lex_mode.lex_state == (t_u16)-1) return NULL; - const Length start_position = ts_stack_position(self->stack, version); - const t_subtree external_token = ts_stack_last_external_token(self->stack, version); + start_position = ts_stack_position(self->stack, version); + external_token = ts_stack_last_external_token(self->stack, version); - bool found_external_token = false; - bool error_mode = parse_state == ERROR_STATE; - bool skipped_error = false; - bool called_get_column = false; - t_i32 first_error_character = 0; - Length error_start_position = length_zero(); - Length error_end_position = length_zero(); - t_u32 lookahead_end_byte = 0; - t_u32 external_scanner_state_len = 0; - bool external_scanner_state_changed = false; + found_external_token = false; + error_mode = parse_state == ERROR_STATE; + skipped_error = false; + called_get_column = false; + first_error_character = 0; + error_start_position = length_zero(); + error_end_position = length_zero(); + lookahead_end_byte = 0; + external_scanner_state_len = 0; + external_scanner_state_changed = false; ts_lexer_reset(&self->lexer, start_position); for (;;) { - bool found_token = false; - Length current_position = self->lexer.current_position; + found_token = false; + current_position = self->lexer.current_position; if (lex_mode.external_lex_state != 0) { @@ -419,22 +456,20 @@ static t_subtree ts_parser__lex(TSParser *self, t_stack_version version, TSState error_end_position = self->lexer.current_position; } - t_subtree result; if (skipped_error) { - Length padding = length_sub(error_start_position, start_position); - Length size = length_sub(error_end_position, error_start_position); - t_u32 lookahead_bytes = lookahead_end_byte - error_end_position.bytes; - result = - ts_subtree_new_error(/*&self->tree_pool, */ first_error_character, padding, size, lookahead_bytes, parse_state, self->language); + padding = length_sub(error_start_position, start_position); + size = length_sub(error_end_position, error_start_position); + lookahead_bytes = lookahead_end_byte - error_end_position.bytes; + result = ts_subtree_new_error(first_error_character, padding, size, lookahead_bytes, parse_state, self->language); } else { - bool is_keyword = false; - TSSymbol symbol = self->lexer.data.result_symbol; - Length padding = length_sub(self->lexer.token_start_position, start_position); - Length size = length_sub(self->lexer.token_end_position, self->lexer.token_start_position); - t_u32 lookahead_bytes = lookahead_end_byte - self->lexer.token_end_position.bytes; + is_keyword = false; + symbol = self->lexer.data.result_symbol; + padding = length_sub(self->lexer.token_start_position, start_position); + size = length_sub(self->lexer.token_end_position, self->lexer.token_start_position); + lookahead_bytes = lookahead_end_byte - self->lexer.token_end_position.bytes; if (found_external_token) { @@ -442,7 +477,7 @@ static t_subtree ts_parser__lex(TSParser *self, t_stack_version version, TSState } else if (symbol == self->language->keyword_capture_token && symbol != 0) { - t_u32 end_byte = self->lexer.token_end_position.bytes; + end_byte = self->lexer.token_end_position.bytes; ts_lexer_reset(&self->lexer, self->lexer.token_start_position); ts_lexer_start(&self->lexer); @@ -455,14 +490,13 @@ static t_subtree ts_parser__lex(TSParser *self, t_stack_version version, TSState } } - result = ts_subtree_new_leaf(/*&self->tree_pool,*/ symbol, padding, size, lookahead_bytes, parse_state, found_external_token, - called_get_column, is_keyword, self->language); + result = ts_subtree_new_leaf(symbol, padding, size, lookahead_bytes, parse_state, found_external_token, called_get_column, + is_keyword, self->language); if (found_external_token) { - t_subtree mut_result = (result); - ts_external_scanner_state_init(&mut_result->external_scanner_state, self->lexer.debug_buffer, external_scanner_state_len); - mut_result->has_external_scanner_state_change = external_scanner_state_changed; + ts_external_scanner_state_init(&result->external_scanner_state, self->lexer.debug_buffer, external_scanner_state_len); + result->has_external_scanner_state_change = external_scanner_state_changed; } } @@ -475,12 +509,12 @@ static t_subtree ts_parser__lex(TSParser *self, t_stack_version version, TSState // and finally, as a default, by a recursive comparison of the trees' symbols. static bool ts_parser__select_tree(TSParser *self, t_subtree left, t_subtree right) { + int comparison; (void)(self); if (!left) return true; if (!right) return false; - if (ts_subtree_error_cost(right) < ts_subtree_error_cost(left)) { return true; @@ -504,7 +538,7 @@ static bool ts_parser__select_tree(TSParser *self, t_subtree left, t_subtree rig if (ts_subtree_error_cost(left) > 0) return true; - int comparison = ts_subtree_compare(left, right /* , &self->tree_pool*/); + comparison = ts_subtree_compare(left, right); switch (comparison) { case -1: @@ -529,38 +563,54 @@ static bool ts_parser__select_children(TSParser *self, t_subtree left, const t_v static void ts_parser__shift(TSParser *self, t_stack_version version, TSStateId state, t_subtree lookahead, bool extra) { - bool is_leaf = ts_subtree_child_count(lookahead) == 0; - t_subtree subtree_to_push = lookahead; + t_subtree result; + bool is_leaf; + t_subtree subtree_to_push; + is_leaf = ts_subtree_child_count(lookahead) == 0; + subtree_to_push = lookahead; if (extra != ts_subtree_extra(lookahead) && is_leaf) { - t_subtree result = ts_subtree_ensure_owner(/*&self->tree_pool,*/ lookahead); + result = ts_subtree_ensure_owner(lookahead); ts_subtree_set_extra(&result, extra); subtree_to_push = (result); } ts_stack_push(self->stack, version, subtree_to_push, !is_leaf, state); if (ts_subtree_has_external_tokens(subtree_to_push)) - { ts_stack_set_last_external_token(self->stack, version, ts_subtree_last_external_token(subtree_to_push)); - } } static t_stack_version ts_parser__reduce(TSParser *self, t_stack_version version, TSSymbol symbol, t_u32 count, int dynamic_precedence, t_u16 production_id, bool is_fragile, bool end_of_non_terminal_extra) { - t_u32 initial_version_count = ts_stack_version_count(self->stack); + t_u32 initial_version_count; + t_stack_slice_array pop; + t_u32 removed_version_count; + t_stack_slice slice; + t_stack_version slice_version; + t_stack_slice next_slice; + t_u32 i; + t_vec_subtree children; + t_subtree parent; + t_vec_subtree next_slice_children; + TSStateId state; + TSStateId next_state; + t_u32 j; + t_stack_version k; + + initial_version_count = ts_stack_version_count(self->stack); // Pop the given number of nodes from the given version of the parse stack. // If stack versions have previously merged, then there may be more than one // path back through the stack. For each path, create a new parent node to // contain the popped children, and push it onto the stack in place of the // children. - t_stack_slice_array pop = ts_stack_pop_count(self->stack, version, count); - t_u32 removed_version_count = 0; - for (t_u32 i = 0; i < pop.size; i++) + pop = ts_stack_pop_count(self->stack, version, count); + removed_version_count = 0; + for (i = 0; i < pop.size; i++) { - t_stack_slice slice = pop.contents[i]; - t_stack_version slice_version = slice.version - removed_version_count; + slice = pop.contents[i]; + slice_version = slice.version - removed_version_count; // This is where new versions are added to the parse stack. The versions // will all be sorted and truncated at the end of the outer parsing loop. @@ -569,14 +619,14 @@ static t_stack_version ts_parser__reduce(TSParser *self, t_stack_version version if (slice_version > MAX_VERSION_COUNT + MAX_VERSION_COUNT_OVERFLOW) { ts_stack_remove_version(self->stack, slice_version); - ts_subtree_array_delete(/*&self->tree_pool,*/ &slice.subtrees); + ts_subtree_array_delete(&slice.subtrees); removed_version_count++; while (i + 1 < pop.size) { - t_stack_slice next_slice = pop.contents[i + 1]; + next_slice = pop.contents[i + 1]; if (next_slice.version != slice.version) break; - ts_subtree_array_delete(/*&self->tree_pool,*/ &next_slice.subtrees); + ts_subtree_array_delete(&next_slice.subtrees); i++; } continue; @@ -585,10 +635,10 @@ static t_stack_version ts_parser__reduce(TSParser *self, t_stack_version version // Extra tokens on top of the stack should not be included in this new parent // node. They will be re-pushed onto the stack after the parent node is // created and pushed. - t_vec_subtree children = slice.subtrees; + children = slice.subtrees; ts_subtree_array_remove_trailing_extras(&children, &self->trailing_extras); - t_subtree parent = ts_subtree_new_node(symbol, &children, production_id, self->language); + parent = ts_subtree_new_node(symbol, &children, production_id, self->language); // This pop operation may have caused multiple stack versions to collapse // into one, because they all diverged from a common state. In that case, @@ -596,12 +646,12 @@ static t_stack_version ts_parser__reduce(TSParser *self, t_stack_version version // delete the rest of the tree arrays. while (i + 1 < pop.size) { - t_stack_slice next_slice = pop.contents[i + 1]; + next_slice = pop.contents[i + 1]; if (next_slice.version != slice.version) break; i++; - t_vec_subtree next_slice_children = next_slice.subtrees; + next_slice_children = next_slice.subtrees; ts_subtree_array_remove_trailing_extras(&next_slice_children, &self->trailing_extras2); if (ts_parser__select_children(self, (parent), &next_slice_children)) @@ -618,8 +668,8 @@ static t_stack_version ts_parser__reduce(TSParser *self, t_stack_version version } } - TSStateId state = ts_stack_state(self->stack, slice_version); - TSStateId next_state = ts_language_next_state(self->language, state, symbol); + state = ts_stack_state(self->stack, slice_version); + next_state = ts_language_next_state(self->language, state, symbol); if (end_of_non_terminal_extra && next_state == state) { parent->extra = true; @@ -639,16 +689,15 @@ static t_stack_version ts_parser__reduce(TSParser *self, t_stack_version version // Push the parent node onto the stack, along with any extra tokens that // were previously on top of the stack. ts_stack_push(self->stack, slice_version, (parent), false, next_state); - for (t_u32 j = 0; j < self->trailing_extras.len; j++) + for (j = 0; j < self->trailing_extras.len; j++) { ts_stack_push(self->stack, slice_version, self->trailing_extras.buffer[j], false, next_state); } - - for (t_stack_version j = 0; j < slice_version; j++) + for (k = 0; k < slice_version; k++) { - if (j == version) + if (k == version) continue; - if (ts_stack_merge(self->stack, j, slice_version)) + if (ts_stack_merge(self->stack, k, slice_version)) { removed_version_count++; break; @@ -662,27 +711,36 @@ static t_stack_version ts_parser__reduce(TSParser *self, t_stack_version version static void ts_parser__accept(TSParser *self, t_stack_version version, t_subtree lookahead) { + t_u32 child_count; + const t_subtree *children; + t_stack_slice_array pop; + t_vec_subtree trees; + t_subtree root; + t_u32 i; + t_u32 j; + t_u32 k; + t_subtree tree; assert(ts_subtree_is_eof(lookahead)); ts_stack_push(self->stack, version, lookahead, false, 1); - t_stack_slice_array pop = ts_stack_pop_all(self->stack, version); - for (t_u32 i = 0; i < pop.size; i++) + pop = ts_stack_pop_all(self->stack, version); + for (i = 0; i < pop.size; i++) { - t_vec_subtree trees = pop.contents[i].subtrees; - t_subtree root = NULL; + trees = pop.contents[i].subtrees; + root = NULL; - for (t_u32 j = trees.len - 1; j + 1 > 0; j--) + for (j = trees.len - 1; j + 1 > 0; j--) { - t_subtree tree = trees.buffer[j]; + tree = trees.buffer[j]; if (!ts_subtree_extra(tree)) { - t_u32 child_count = ts_subtree_child_count(tree); - const t_subtree *children = ts_subtree_children(tree); - for (t_u32 k = 0; k < child_count; k++) + child_count = ts_subtree_child_count(tree); + children = ts_subtree_children(tree); + for (k = 0; k < child_count; k++) children[k]->ref_count++; vec_subtree_splice(&trees, vec_subtree_splice_args(j, 1, child_count, children)); root = (ts_subtree_new_node(ts_subtree_symbol(tree), &trees, tree->production_id, self->language)); - ts_subtree_release(/*&self->tree_pool, */ tree); + ts_subtree_release(tree); break; } } @@ -707,18 +765,35 @@ static void ts_parser__accept(TSParser *self, t_stack_version version, t_subtree static bool ts_parser__do_all_potential_reductions(TSParser *self, t_stack_version starting_version, TSSymbol lookahead_symbol) { - t_u32 initial_version_count = ts_stack_version_count(self->stack); + t_u32 initial_version_count; + bool can_shift_lookahead_symbol; + t_stack_version version; + t_u32 i; + t_u32 version_count; + bool merged; + t_stack_version j; + TSStateId state; + bool has_shift_action; + TSSymbol first_symbol; + TSSymbol end_symbol; + t_stack_version reduction_version; + ReduceAction reduce_action; + t_u32 k; + TSSymbol symbol; + TableEntry entry; + TSParseAction action; - bool can_shift_lookahead_symbol = false; - t_stack_version version = starting_version; - for (t_u32 i = 0; true; i++) + initial_version_count = ts_stack_version_count(self->stack); + can_shift_lookahead_symbol = false; + version = starting_version; + for (i = 0; true; i++) { - t_u32 version_count = ts_stack_version_count(self->stack); + version_count = ts_stack_version_count(self->stack); if (version >= version_count) break; - bool merged = false; - for (t_stack_version j = initial_version_count; j < version; j++) + merged = false; + for (j = initial_version_count; j < version; j++) { if (ts_stack_merge(self->stack, j, version)) { @@ -729,11 +804,10 @@ static bool ts_parser__do_all_potential_reductions(TSParser *self, t_stack_versi if (merged) continue; - TSStateId state = ts_stack_state(self->stack, version); - bool has_shift_action = false; + state = ts_stack_state(self->stack, version); + has_shift_action = false; array_clear(&self->reduce_actions); - TSSymbol first_symbol, end_symbol; if (lookahead_symbol != 0) { first_symbol = lookahead_symbol; @@ -745,13 +819,12 @@ static bool ts_parser__do_all_potential_reductions(TSParser *self, t_stack_versi end_symbol = self->language->token_count; } - for (TSSymbol symbol = first_symbol; symbol < end_symbol; symbol++) + for (symbol = first_symbol; symbol < end_symbol; symbol++) { - TableEntry entry; ts_language_table_entry(self->language, state, symbol, &entry); - for (t_u32 j = 0; j < entry.action_count; j++) + for (k = 0; k < entry.action_count; k++) { - TSParseAction action = entry.actions[j]; + action = entry.actions[k]; switch (action.type) { case TSParseActionTypeShift: @@ -774,37 +847,28 @@ static bool ts_parser__do_all_potential_reductions(TSParser *self, t_stack_versi } } - t_stack_version reduction_version = STACK_VERSION_NONE; - for (t_u32 j = 0; j < self->reduce_actions.size; j++) + reduction_version = STACK_VERSION_NONE; + for (k = 0; k < self->reduce_actions.size; k++) { - ReduceAction action = self->reduce_actions.contents[j]; - - reduction_version = - ts_parser__reduce(self, version, action.symbol, action.count, action.dynamic_precedence, action.production_id, true, false); + reduce_action = self->reduce_actions.contents[k]; + reduction_version = ts_parser__reduce(self, version, reduce_action.symbol, reduce_action.count, + reduce_action.dynamic_precedence, reduce_action.production_id, true, false); } if (has_shift_action) - { can_shift_lookahead_symbol = true; - } else if (reduction_version != STACK_VERSION_NONE && i < MAX_VERSION_COUNT) { ts_stack_renumber_version(self->stack, reduction_version, version); continue; } else if (lookahead_symbol != 0) - { ts_stack_remove_version(self->stack, version); - } if (version == starting_version) - { version = version_count; - } else - { version++; - } } return can_shift_lookahead_symbol; @@ -812,16 +876,27 @@ static bool ts_parser__do_all_potential_reductions(TSParser *self, t_stack_versi static bool ts_parser__recover_to_state(TSParser *self, t_stack_version version, t_u32 depth, TSStateId goal_state) { - t_stack_slice_array pop = ts_stack_pop_count(self->stack, version, depth); - t_stack_version previous_version = STACK_VERSION_NONE; + t_stack_slice_array pop; + t_stack_version previous_version; + t_stack_slice slice; + t_u32 i; + t_u32 j; + t_vec_subtree error_trees; + t_subtree error_tree; + t_u32 error_child_count; + t_subtree tree; + t_subtree error; - for (t_u32 i = 0; i < pop.size; i++) + previous_version = STACK_VERSION_NONE; + pop = ts_stack_pop_count(self->stack, version, depth); + + for (i = 0; i < pop.size; i++) { - t_stack_slice slice = pop.contents[i]; + slice = pop.contents[i]; if (slice.version == previous_version) { - ts_subtree_array_delete(/*&self->tree_pool,*/ &slice.subtrees); + ts_subtree_array_delete(&slice.subtrees); array_erase(&pop, i--); continue; } @@ -829,32 +904,30 @@ static bool ts_parser__recover_to_state(TSParser *self, t_stack_version version, if (ts_stack_state(self->stack, slice.version) != goal_state) { ts_stack_halt(self->stack, slice.version); - ts_subtree_array_delete(/*&self->tree_pool,*/ &slice.subtrees); + ts_subtree_array_delete(&slice.subtrees); array_erase(&pop, i--); continue; } - t_vec_subtree error_trees = ts_stack_pop_error(self->stack, slice.version); + error_trees = ts_stack_pop_error(self->stack, slice.version); if (error_trees.len > 0) { - t_subtree error_tree = error_trees.buffer[0]; - t_u32 error_child_count = ts_subtree_child_count(error_tree); + error_tree = error_trees.buffer[0]; + error_child_count = ts_subtree_child_count(error_tree); if (error_child_count > 0) { vec_subtree_splice(&slice.subtrees, vec_subtree_splice_args(0, 0, error_child_count, ts_subtree_children(error_tree))); - for (t_u32 j = 0; j < error_child_count; j++) - { - (slice.subtrees.buffer[j]->ref_count++); - } + for (j = 0; j < error_child_count; j++) + slice.subtrees.buffer[j]->ref_count++; } - ts_subtree_array_delete(/*&self->tree_pool,*/ &error_trees); + ts_subtree_array_delete(&error_trees); } ts_subtree_array_remove_trailing_extras(&slice.subtrees, &self->trailing_extras); if (slice.subtrees.len > 0) { - t_subtree error = ts_subtree_new_error_node(&slice.subtrees, true, self->language); + error = ts_subtree_new_error_node(&slice.subtrees, true, self->language); ts_stack_push(self->stack, slice.version, error, false, goal_state); } else @@ -862,9 +935,9 @@ static bool ts_parser__recover_to_state(TSParser *self, t_stack_version version, vec_subtree_free(slice.subtrees); } - for (t_u32 j = 0; j < self->trailing_extras.len; j++) + for (j = 0; j < self->trailing_extras.len; j++) { - t_subtree tree = self->trailing_extras.buffer[j]; + tree = self->trailing_extras.buffer[j]; ts_stack_push(self->stack, slice.version, tree, false, goal_state); } @@ -876,12 +949,32 @@ static bool ts_parser__recover_to_state(TSParser *self, t_stack_version version, static void ts_parser__recover(TSParser *self, t_stack_version version, t_subtree lookahead) { - bool did_recover = false; - t_u32 previous_version_count = ts_stack_version_count(self->stack); - Length position = ts_stack_position(self->stack, version); - t_stack_summary *summary = ts_stack_get_summary(self->stack, version); - t_u32 node_count_since_error = ts_stack_node_count_since_error(self->stack, version); - t_u32 current_error_cost = ts_stack_error_cost(self->stack, version); + Length position; + bool did_recover; + bool would_merge; + t_stack_slice_array pop; + t_stack_summary *summary; + t_stack_summary_entry entry; + t_subtree parent; + t_u32 current_error_cost; + t_u32 depth; + t_u32 i; + t_u32 j; + t_u32 new_cost; + t_u32 node_count_since_error; + t_u32 previous_version_count; + t_vec_subtree children; + t_u32 n; + const TSParseAction *actions; + t_subtree error_repeat; + t_subtree mutable_lookahead; + + did_recover = false; + previous_version_count = ts_stack_version_count(self->stack); + position = ts_stack_position(self->stack, version); + summary = ts_stack_get_summary(self->stack, version); + node_count_since_error = ts_stack_node_count_since_error(self->stack, version); + current_error_cost = ts_stack_error_cost(self->stack, version); // When the parser is in the error state, there are two strategies for recovering with a // given lookahead token: @@ -897,21 +990,21 @@ static void ts_parser__recover(TSParser *self, t_stack_version version, t_subtre // if the current lookahead token would be valid in that state. if (summary && !ts_subtree_is_error(lookahead)) { - for (t_u32 i = 0; i < summary->size; i++) + for (i = 0; i < summary->size; i++) { - t_stack_summary_entry entry = summary->contents[i]; + entry = summary->contents[i]; if (entry.state == ERROR_STATE) continue; if (entry.position.bytes == position.bytes) continue; - t_u32 depth = entry.depth; + depth = entry.depth; if (node_count_since_error > 0) depth++; // Do not recover in ways that create redundant stack versions. - bool would_merge = false; - for (t_u32 j = 0; j < previous_version_count; j++) + would_merge = false; + for (j = 0; j < previous_version_count; j++) { if (ts_stack_state(self->stack, j) == entry.state && ts_stack_position(self->stack, j).bytes == position.bytes) { @@ -923,9 +1016,9 @@ static void ts_parser__recover(TSParser *self, t_stack_version version, t_subtre continue; // Do not recover if the result would clearly be worse than some existing stack version. - t_u32 new_cost = current_error_cost + entry.depth * ERROR_COST_PER_SKIPPED_TREE + - (position.bytes - entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR + - (position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE; + new_cost = current_error_cost + entry.depth * ERROR_COST_PER_SKIPPED_TREE + + (position.bytes - entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR + + (position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE; if (ts_parser__better_version_exists(self, version, false, new_cost)) break; @@ -944,13 +1037,9 @@ static void ts_parser__recover(TSParser *self, t_stack_version version, t_subtre // In the process of attempting to recover, some stack versions may have been created // and subsequently halted. Remove those versions. - for (t_u32 i = previous_version_count; i < ts_stack_version_count(self->stack); i++) - { + for (i = previous_version_count; i < ts_stack_version_count(self->stack); i++) if (!ts_stack_is_active(self->stack, i)) - { ts_stack_remove_version(self->stack, i--); - } - } // If strategy 1 succeeded, a new stack version will have been created which is able to handle // the current lookahead token. Now, in addition, try strategy 2 described above: skip the @@ -960,14 +1049,14 @@ static void ts_parser__recover(TSParser *self, t_stack_version version, t_subtre if (did_recover && ts_stack_version_count(self->stack) > MAX_VERSION_COUNT) { ts_stack_halt(self->stack, version); - ts_subtree_release(/*&self->tree_pool,*/ lookahead); + ts_subtree_release(lookahead); return; } if (did_recover && ts_subtree_has_external_scanner_state_change(lookahead)) { ts_stack_halt(self->stack, version); - ts_subtree_release(/*&self->tree_pool,*/ lookahead); + ts_subtree_release(lookahead); return; } @@ -975,45 +1064,44 @@ static void ts_parser__recover(TSParser *self, t_stack_version version, t_subtre // in an ERROR node and terminate. if (ts_subtree_is_eof(lookahead)) { - t_vec_subtree children = vec_subtree_new(16, NULL); - t_subtree parent = ts_subtree_new_error_node(&children, false, self->language); + children = vec_subtree_new(16, NULL); + parent = ts_subtree_new_error_node(&children, false, self->language); ts_stack_push(self->stack, version, parent, false, 1); ts_parser__accept(self, version, lookahead); return; } // Do not recover if the result would clearly be worse than some existing stack version. - t_u32 new_cost = current_error_cost + ERROR_COST_PER_SKIPPED_TREE + ts_subtree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR + - ts_subtree_total_size(lookahead).extent.row * ERROR_COST_PER_SKIPPED_LINE; + new_cost = current_error_cost + ERROR_COST_PER_SKIPPED_TREE + ts_subtree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR + + ts_subtree_total_size(lookahead).extent.row * ERROR_COST_PER_SKIPPED_LINE; if (ts_parser__better_version_exists(self, version, false, new_cost)) { ts_stack_halt(self->stack, version); - ts_subtree_release(/*&self->tree_pool,*/ lookahead); + ts_subtree_release(lookahead); return; } // If the current lookahead token is an extra token, mark it as extra. This means it won't // be counted in error cost calculations. - t_u32 n; - const TSParseAction *actions = ts_language_actions(self->language, 1, ts_subtree_symbol(lookahead), &n); + actions = ts_language_actions(self->language, 1, ts_subtree_symbol(lookahead), &n); if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].shift.extra) { - t_subtree mutable_lookahead = ts_subtree_ensure_owner(/*&self->tree_pool,*/ lookahead); + mutable_lookahead = ts_subtree_ensure_owner(lookahead); ts_subtree_set_extra(&mutable_lookahead, true); lookahead = (mutable_lookahead); } // Wrap the lookahead token in an ERROR. - t_vec_subtree children = vec_subtree_new(1, NULL); + children = vec_subtree_new(1, NULL); vec_subtree_push(&children, lookahead); - t_subtree error_repeat = ts_subtree_new_node(ts_builtin_sym_error_repeat, &children, 0, self->language); + error_repeat = ts_subtree_new_node(ts_builtin_sym_error_repeat, &children, 0, self->language); // If other tokens have already been skipped, so there is already an ERROR at the top of the // stack, then pop that ERROR off the stack and wrap the two ERRORs together into one larger // ERROR. if (node_count_since_error > 0) { - t_stack_slice_array pop = ts_stack_pop_count(self->stack, version, 1); + pop = ts_stack_pop_count(self->stack, version, 1); // TODO: Figure out how to make this condition occur. // See https://github.com/atom/atom/issues/18450#issuecomment-439579778 @@ -1021,14 +1109,10 @@ static void ts_parser__recover(TSParser *self, t_stack_version version, t_subtre // arbitrarily and discard the rest. if (pop.size > 1) { - for (t_u32 i = 1; i < pop.size; i++) - { + for (i = 1; i < pop.size; i++) ts_subtree_array_delete(/*&self->tree_pool,*/ &pop.contents[i].subtrees); - } while (ts_stack_version_count(self->stack) > pop.contents[0].version + 1) - { ts_stack_remove_version(self->stack, pop.contents[0].version + 1); - } } ts_stack_renumber_version(self->stack, pop.contents[0].version, version); @@ -1039,37 +1123,44 @@ static void ts_parser__recover(TSParser *self, t_stack_version version, t_subtre // Push the new ERROR onto the stack. ts_stack_push(self->stack, version, (error_repeat), false, ERROR_STATE); if (ts_subtree_has_external_tokens(lookahead)) - { ts_stack_set_last_external_token(self->stack, version, ts_subtree_last_external_token(lookahead)); - } } static void ts_parser__handle_error(TSParser *self, t_stack_version version, t_subtree lookahead) { - t_u32 previous_version_count = ts_stack_version_count(self->stack); + TSSymbol missing_symbol; + bool did_insert_missing_token; + TSStateId state; + TSStateId state_after_missing_symbol; + Length padding; + t_u32 lookahead_bytes; + t_stack_version version_with_missing_tree; + t_subtree missing_tree; + t_u32 previous_version_count; + t_u32 version_count; + Length position; + previous_version_count = ts_stack_version_count(self->stack); // Perform any reductions that can happen in this state, regardless of the lookahead. After // skipping one or more invalid tokens, the parser might find a token that would have allowed // a reduction to take place. ts_parser__do_all_potential_reductions(self, version, 0); - t_u32 version_count = ts_stack_version_count(self->stack); - Length position = ts_stack_position(self->stack, version); + version_count = ts_stack_version_count(self->stack); + position = ts_stack_position(self->stack, version); // Push a discontinuity onto the stack. Merge all of the stack versions that // were created in the previous step. - bool did_insert_missing_token = false; + did_insert_missing_token = false; for (t_stack_version v = version; v < version_count;) { if (!did_insert_missing_token) { - TSStateId state = ts_stack_state(self->stack, v); - for (TSSymbol missing_symbol = 1; missing_symbol < (t_u16)self->language->token_count; missing_symbol++) + state = ts_stack_state(self->stack, v); + for (missing_symbol = 1; missing_symbol < (t_u16)self->language->token_count; missing_symbol++) { - TSStateId state_after_missing_symbol = ts_language_next_state(self->language, state, missing_symbol); + state_after_missing_symbol = ts_language_next_state(self->language, state, missing_symbol); if (state_after_missing_symbol == 0 || state_after_missing_symbol == state) - { continue; - } if (ts_language_has_reduce_action(self->language, state_after_missing_symbol, ts_subtree_leaf_symbol(lookahead))) { @@ -1078,12 +1169,11 @@ static void ts_parser__handle_error(TSParser *self, t_stack_version version, t_s // must be assigned to position it within the next included range. ts_lexer_reset(&self->lexer, position); ts_lexer__mark_end((void *)&self->lexer); - Length padding = length_sub(self->lexer.token_end_position, position); - t_u32 lookahead_bytes = ts_subtree_total_bytes(lookahead) + ts_subtree_lookahead_bytes(lookahead); + padding = length_sub(self->lexer.token_end_position, position); + lookahead_bytes = ts_subtree_total_bytes(lookahead) + ts_subtree_lookahead_bytes(lookahead); - t_stack_version version_with_missing_tree = ts_stack_copy_version(self->stack, v); - t_subtree missing_tree = - ts_subtree_new_missing_leaf(/*&self->tree_pool,*/ missing_symbol, padding, lookahead_bytes, self->language); + version_with_missing_tree = ts_stack_copy_version(self->stack, v); + missing_tree = ts_subtree_new_missing_leaf(missing_symbol, padding, lookahead_bytes, self->language); ts_stack_push(self->stack, version_with_missing_tree, missing_tree, false, state_after_missing_symbol); if (ts_parser__do_all_potential_reductions(self, version_with_missing_tree, ts_subtree_leaf_symbol(lookahead))) @@ -1101,9 +1191,7 @@ static void ts_parser__handle_error(TSParser *self, t_stack_version version, t_s for (t_u32 i = previous_version_count; i < version_count; i++) { - bool did_merge = ts_stack_merge(self->stack, version, previous_version_count); - assert(did_merge); - (void)did_merge; // fix warning/error with clang -Os + ts_stack_merge(self->stack, version, previous_version_count); } ts_stack_record_summary(self->stack, version, MAX_SUMMARY_DEPTH); @@ -1119,12 +1207,18 @@ static void ts_parser__handle_error(TSParser *self, t_stack_version version, t_s static bool ts_parser__advance(TSParser *self, t_stack_version version, bool allow_node_reuse) { (void)(allow_node_reuse); - TSStateId state = ts_stack_state(self->stack, version); + TSStateId state; + t_subtree mutable_lookahead; - t_subtree lookahead = NULL; - TableEntry table_entry = {.action_count = 0}; + t_subtree lookahead; + TableEntry table_entry; + lookahead = NULL; + table_entry = (TableEntry){.action_count = 0}; + state = ts_stack_state(self->stack, version); - bool needs_lex = true; + bool needs_lex; + t_u32 i; + needs_lex = true; for (;;) { // Otherwise, re-run the lexer. @@ -1153,10 +1247,12 @@ static bool ts_parser__advance(TSParser *self, t_stack_version version, bool all // an ambiguous state. REDUCE actions always create a new stack // version, whereas SHIFT actions update the existing stack version // and terminate this loop. - t_stack_version last_reduction_version = STACK_VERSION_NONE; - for (t_u32 i = 0; i < table_entry.action_count; i++) + t_stack_version last_reduction_version; + last_reduction_version = STACK_VERSION_NONE; + for (i = 0; i < table_entry.action_count; i++) { - TSParseAction action = table_entry.actions[i]; + TSParseAction action; + action = table_entry.actions[i]; switch (action.type) { @@ -1183,9 +1279,12 @@ static bool ts_parser__advance(TSParser *self, t_stack_version version, bool all } case TSParseActionTypeReduce: { - bool is_fragile = table_entry.action_count > 1; - bool end_of_non_terminal_extra = lookahead == NULL; - t_stack_version reduction_version = + bool is_fragile; + is_fragile = table_entry.action_count > 1; + bool end_of_non_terminal_extra; + end_of_non_terminal_extra = lookahead == NULL; + t_stack_version reduction_version; + reduction_version = ts_parser__reduce(self, version, action.reduce.symbol, action.reduce.child_count, action.reduce.dynamic_precedence, action.reduce.production_id, is_fragile, end_of_non_terminal_extra); if (reduction_version != STACK_VERSION_NONE) @@ -1246,9 +1345,9 @@ static bool ts_parser__advance(TSParser *self, t_stack_version version, bool all ts_language_table_entry(self->language, state, self->language->keyword_capture_token, &table_entry); if (table_entry.action_count > 0) { - t_subtree mutable_lookahead = ts_subtree_ensure_owner(/*&self->tree_pool,*/ lookahead); + mutable_lookahead = ts_subtree_ensure_owner(lookahead); ts_subtree_set_symbol(&mutable_lookahead, self->language->keyword_capture_token, self->language); - lookahead = (mutable_lookahead); + lookahead = mutable_lookahead; continue; } } @@ -1287,8 +1386,16 @@ static bool ts_parser__advance(TSParser *self, t_stack_version version, bool all static t_u32 ts_parser__condense_stack(TSParser *self) { - t_u32 min_error_cost = UINT_MAX; - for (t_stack_version i = 0; i < ts_stack_version_count(self->stack); i++) + t_error_status status_i; + t_error_status status_j; + t_stack_version i; + t_stack_version j; + t_stack_version n; + t_subtree lookahead; + t_u32 min_error_cost; + + min_error_cost = UINT_MAX; + for (i = 0; i < ts_stack_version_count(self->stack); i++) { // Prune any versions that have been marked for removal. if (ts_stack_is_halted(self->stack, i)) @@ -1300,7 +1407,7 @@ static t_u32 ts_parser__condense_stack(TSParser *self) // Keep track of the minimum error cost of any stack version so // that it can be returned. - t_error_status status_i = ts_parser__version_status(self, i); + status_i = ts_parser__version_status(self, i); if (!status_i.is_in_error && status_i.cost < min_error_cost) { min_error_cost = status_i.cost; @@ -1309,9 +1416,9 @@ static t_u32 ts_parser__condense_stack(TSParser *self) // Examine each pair of stack versions, removing any versions that // are clearly worse than another version. Ensure that the versions // are ordered from most promising to least promising. - for (t_stack_version j = 0; j < i; j++) + for (j = 0; j < i; j++) { - t_error_status status_j = ts_parser__version_status(self, j); + status_j = ts_parser__version_status(self, j); switch (ts_parser__compare_versions(self, status_j, status_i)) { @@ -1354,9 +1461,7 @@ static t_u32 ts_parser__condense_stack(TSParser *self) // Enforce a hard upper bound on the number of stack versions by // discarding the least promising versions. while (ts_stack_version_count(self->stack) > MAX_VERSION_COUNT) - { ts_stack_remove_version(self->stack, MAX_VERSION_COUNT); - } // If the best-performing stack version is currently paused, or all // versions are paused, then resume the best paused version and begin @@ -1364,14 +1469,14 @@ static t_u32 ts_parser__condense_stack(TSParser *self) if (ts_stack_version_count(self->stack) > 0) { bool has_unpaused_version = false; - for (t_stack_version i = 0, n = ts_stack_version_count(self->stack); i < n; i++) + for (i = 0, n = ts_stack_version_count(self->stack); i < n; i++) { if (ts_stack_is_paused(self->stack, i)) { if (!has_unpaused_version && self->accept_count < MAX_VERSION_COUNT) { min_error_cost = ts_stack_error_cost(self->stack, i); - t_subtree lookahead = ts_stack_resume(self->stack, i); + lookahead = ts_stack_resume(self->stack, i); ts_parser__handle_error(self, i, lookahead); has_unpaused_version = true; } @@ -1401,6 +1506,7 @@ static bool ts_parser_has_outstanding_parse(TSParser *self) TSParser *ts_parser_new(void) { TSParser *self; + self = mem_alloc(sizeof(*self)); ts_lexer_init(&self->lexer); array_init(&self->reduce_actions); @@ -1459,13 +1565,19 @@ void ts_parser_reset(TSParser *self) TSTree *ts_parser_parse(TSParser *self, TSInput input) { - TSTree *result = NULL; + TSTree *result; + t_u32 position; + t_u32 last_position; + t_u32 version_count; + t_stack_version version; + bool allow_node_reuse; + t_u32 min_error_cost; + + result = NULL; if (!self->language || !input.read) return NULL; ts_lexer_set_input(&self->lexer, input); - // self->included_range_difference_index = 0; - if (!ts_parser_has_outstanding_parse(self)) { ts_parser__external_scanner_create(self); @@ -1478,14 +1590,15 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input) self->operation_count = 0; - t_u32 position = 0; - t_u32 last_position = 0; - t_u32 version_count = 0; + position = 0; + last_position = 0; + version_count = 0; + version = 0; do { - for (t_stack_version version = 0; version_count = ts_stack_version_count(self->stack), version < version_count; version++) + for (version = 0; version_count = ts_stack_version_count(self->stack), version < version_count; version++) { - bool allow_node_reuse = version_count == 1; + allow_node_reuse = version_count == 1; while (ts_stack_is_active(self->stack, version)) { if (!ts_parser__advance(self, version, allow_node_reuse)) @@ -1508,7 +1621,7 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input) // After advancing each version of the stack, re-sort the versions by their cost, // removing any versions that are no longer worth pursuing. - t_u32 min_error_cost = ts_parser__condense_stack(self); + min_error_cost = ts_parser__condense_stack(self); // If there's already a finished parse tree that's better than any in-progress version, // then terminate parsing. Clear the parse stack to remove any extra references to subtrees @@ -1532,7 +1645,8 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input) TSTree *ts_parser_parse_string(TSParser *self, t_const_str string, t_u32 length) { - t_string_input input = {(const t_u8 *)string, length}; + t_string_input input; + input = (t_string_input){(const t_u8 *)string, length}; return ts_parser_parse(self, (TSInput){ &input, ts_string_input_read,