raphael/minishell
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Upodated stuff

Browse source
This commit is contained in:
Maieul BOYER 2024-09-05 19:04:57 +00:00
parent c5bc77f467
commit 7401038bf7
5 changed files with 397 additions and 399 deletions
Download patch file Download diff file Expand all files Collapse all files

1
parser/Filelist.parser.mk
View file

@ -46,6 +46,7 @@ stack/stack_manipulate3 \
stack/stack_node \
stack/stack_summary \
stack/stack_version \
subtree/subtree_balance \
subtree/subtree_funcs \
subtree/subtree_helper \
subtree/subtree_new \

1
parser/src/lex.c
View file

@ -1,5 +1,4 @@
#include "../static/char_set/charset_inline.h"
#include "parser/api.h"
#include "gmr/symbols.h"
bool ts_lex(t_lexer *lexer, t_state_id state)

331
parser/src/parser.c
View file

@ -25,10 +25,10 @@
#include <assert.h>
#include <stdio.h>
static const t_u32 MAX_VERSION_COUNT = 4;
static const t_u32 MAX_VERSION_COUNT_OVERFLOW = 6;
static const t_u32 MAX_SUMMARY_DEPTH = 1;
static const t_u32 MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE;
#define MAX_VERSION_COUNT 40
#define MAX_VERSION_COUNT_OVERFLOW 60
#define MAX_SUMMARY_DEPTH 1
#define MAX_COST_DIFFERENCE 16 * ERROR_COST_PER_SKIPPED_TREE
typedef struct s_error_status t_error_status;
typedef enum e_error_comparison t_error_comparison;
@ -78,8 +78,7 @@ struct s_string_input
// StringInput
static const t_u8 *ts_string_input_read(void *_self, t_u32 byte,
TSPoint point, t_u32 *length)
static const t_u8 *ts_string_input_read(void *_self, t_u32 byte, TSPoint point, t_u32 *length)
{
t_string_input *self;
@ -98,8 +97,7 @@ static const t_u8 *ts_string_input_read(void *_self, t_u32 byte,
}
// Parser - Private
static bool ts_parser__breakdown_top_of_stack(TSParser *self,
t_stack_version version)
static bool ts_parser__breakdown_top_of_stack(TSParser *self, t_stack_version version)
{
t_stack_slice_array pop;
bool did_break_down;
@ -137,12 +135,10 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self,
}
else if (!ts_subtree_extra(child))
{
state = ts_language_next_state(self->language, state,
ts_subtree_symbol(child));
state = ts_language_next_state(self->language, state, ts_subtree_symbol(child));
}
child->ref_count++;
ts_stack_push(self->stack, slice.version, child, pending,
state);
ts_stack_push(self->stack, slice.version, child, pending, state);
}
for (j = 1; j < slice.subtrees.len; j++)
{
@ -156,8 +152,7 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self,
return (did_break_down);
}
static t_error_comparison ts_parser__compare_versions(TSParser *self,
t_error_status a, t_error_status b)
static t_error_comparison ts_parser__compare_versions(TSParser *self, t_error_status a, t_error_status b)
{
(void)self;
if (!a.is_in_error && b.is_in_error)
@ -211,8 +206,7 @@ static t_error_comparison ts_parser__compare_versions(TSParser *self,
return (ECNone);
}
static t_error_status ts_parser__version_status(TSParser *self,
t_stack_version version)
static t_error_status ts_parser__version_status(TSParser *self, t_stack_version version)
{
t_u32 cost;
bool is_paused;
@ -224,12 +218,10 @@ static t_error_status ts_parser__version_status(TSParser *self,
return ((t_error_status){.cost = cost,
.node_count = ts_stack_node_count_since_error(self->stack, version),
.dynamic_precedence = ts_stack_dynamic_precedence(self->stack, version),
.is_in_error = is_paused || ts_stack_state(self->stack,
version) == ERROR_STATE});
.is_in_error = is_paused || ts_stack_state(self->stack, version) == ERROR_STATE});
}
static bool ts_parser__better_version_exists(TSParser *self,
t_stack_version version, bool is_in_error, t_u32 cost)
static bool ts_parser__better_version_exists(TSParser *self, t_stack_version version, bool is_in_error, t_u32 cost)
{
t_error_status status_i;
Length position;
@ -237,8 +229,7 @@ static bool ts_parser__better_version_exists(TSParser *self,
t_stack_version i;
t_stack_version n;
if (self->finished_tree
&& ts_subtree_error_cost(self->finished_tree) <= cost)
if (self->finished_tree && ts_subtree_error_cost(self->finished_tree) <= cost)
return (true);
position = ts_stack_position(self->stack, version);
status = (t_error_status){
@ -249,8 +240,7 @@ static bool ts_parser__better_version_exists(TSParser *self,
};
for (i = 0, n = ts_stack_version_count(self->stack); i < n; i++)
{
if (i == version || !ts_stack_is_active(self->stack, i)
|| ts_stack_position(self->stack, i).bytes < position.bytes)
if (i == version || !ts_stack_is_active(self->stack, i) || ts_stack_position(self->stack, i).bytes < position.bytes)
continue;
status_i = ts_parser__version_status(self, i);
switch (ts_parser__compare_versions(self, status, status_i))
@ -281,15 +271,13 @@ static bool ts_parser__better_version_exists(TSParser *self,
static void ts_parser__external_scanner_create(TSParser *self)
{
if (self->language && self->language->external_scanner.states
&& self->language->external_scanner.create)
if (self->language && self->language->external_scanner.states && self->language->external_scanner.create)
self->external_scanner_payload = self->language->external_scanner.create();
}
static void ts_parser__external_scanner_destroy(TSParser *self)
{
if (self->language && self->external_scanner_payload
&& self->language->external_scanner.destroy)
if (self->language && self->external_scanner_payload && self->language->external_scanner.destroy)
self->language->external_scanner.destroy(self->external_scanner_payload);
self->external_scanner_payload = NULL;
}
@ -298,17 +286,14 @@ static t_u32 ts_parser__external_scanner_serialize(TSParser *self)
{
t_u32 length;
length = self->language->external_scanner.serialize(self->external_scanner_payload,
self->lexer.debug_buffer);
length = self->language->external_scanner.serialize(self->external_scanner_payload, self->lexer.debug_buffer);
if (length > TREE_SITTER_SERIALIZATION_BUFFER_SIZE)
me_abort("assertion failed in " __FILE__ " `length > "
"TREE_SITTER_SERIALIZATION_BUFFER_SIZE`");
return length;
}
static void ts_parser__external_scanner_deserialize(TSParser *self,
t_subtree external_token)
static void ts_parser__external_scanner_deserialize(TSParser *self, t_subtree external_token)
{
const t_u8 *data;
t_u32 length;
@ -321,23 +306,18 @@ static void ts_parser__external_scanner_deserialize(TSParser *self,
length = external_token->external_scanner_state.length;
printf("HERE\n");
}
self->language->external_scanner.deserialize(self->external_scanner_payload,
data, length);
self->language->external_scanner.deserialize(self->external_scanner_payload, data, length);
}
static bool ts_parser__external_scanner_scan(TSParser *self,
TSStateId external_lex_state)
static bool ts_parser__external_scanner_scan(TSParser *self, TSStateId external_lex_state)
{
const bool *valid_external_tokens;
valid_external_tokens = ts_language_enabled_external_tokens(self->language,
external_lex_state);
return self->language->external_scanner.scan(self->external_scanner_payload,
&self->lexer.data, valid_external_tokens);
valid_external_tokens = ts_language_enabled_external_tokens(self->language, external_lex_state);
return self->language->external_scanner.scan(self->external_scanner_payload, &self->lexer.data, valid_external_tokens);
}
static t_subtree ts_parser__lex(TSParser *self, t_stack_version version,
TSStateId parse_state)
static t_subtree ts_parser__lex(TSParser *self, t_stack_version version, TSStateId parse_state)
{
Length start_position;
t_subtree external_token;
@ -386,8 +366,7 @@ static t_subtree ts_parser__lex(TSParser *self, t_stack_version version,
{
ts_lexer_start(&self->lexer);
ts_parser__external_scanner_deserialize(self, external_token);
found_token = ts_parser__external_scanner_scan(self,
lex_mode.external_lex_state);
found_token = ts_parser__external_scanner_scan(self, lex_mode.external_lex_state);
if (self->has_scanner_error)
return NULL;
ts_lexer_finish(&self->lexer, &lookahead_end_byte);
@ -414,10 +393,8 @@ static t_subtree ts_parser__lex(TSParser *self, t_stack_version version,
// terminated at the start of an included
// range, it is marked as ending at the
// *end* of the preceding included range.
if (self->lexer.token_end_position.bytes <= current_position.bytes
&& (error_mode
|| !ts_stack_has_advanced_since_error(self->stack,
version)) && !external_scanner_state_changed)
if (self->lexer.token_end_position.bytes <= current_position.bytes &&
(error_mode || !ts_stack_has_advanced_since_error(self->stack, version)) && !external_scanner_state_changed)
{
found_token = false;
}
@ -465,18 +442,15 @@ static t_subtree ts_parser__lex(TSParser *self, t_stack_version version,
padding = length_sub(error_start_position, start_position);
size = length_sub(error_end_position, error_start_position);
lookahead_bytes = lookahead_end_byte - error_end_position.bytes;
result = ts_subtree_new_error(first_error_character, padding, size,
lookahead_bytes, parse_state, self->language);
result = ts_subtree_new_error(first_error_character, padding, size, lookahead_bytes, parse_state, self->language);
}
else
{
is_keyword = false;
symbol = self->lexer.data.result_symbol;
padding = length_sub(self->lexer.token_start_position, start_position);
size = length_sub(self->lexer.token_end_position,
self->lexer.token_start_position);
lookahead_bytes = lookahead_end_byte
- self->lexer.token_end_position.bytes;
size = length_sub(self->lexer.token_end_position, self->lexer.token_start_position);
lookahead_bytes = lookahead_end_byte - self->lexer.token_end_position.bytes;
if (found_external_token)
{
symbol = self->language->external_scanner.symbol_map[symbol];
@ -487,20 +461,17 @@ static t_subtree ts_parser__lex(TSParser *self, t_stack_version version,
ts_lexer_reset(&self->lexer, self->lexer.token_start_position);
ts_lexer_start(&self->lexer);
is_keyword = self->language->keyword_lex_fn(&self->lexer.data, 0);
if (is_keyword && self->lexer.token_end_position.bytes == end_byte
&& ts_language_has_actions(self->language, parse_state,
self->lexer.data.result_symbol))
if (is_keyword && self->lexer.token_end_position.bytes == end_byte &&
ts_language_has_actions(self->language, parse_state, self->lexer.data.result_symbol))
{
symbol = self->lexer.data.result_symbol;
}
}
result = ts_subtree_new_leaf(symbol, padding, size, lookahead_bytes,
parse_state, found_external_token, called_get_column,
result = ts_subtree_new_leaf(symbol, padding, size, lookahead_bytes, parse_state, found_external_token, called_get_column,
is_keyword, self->language);
if (found_external_token)
{
ts_external_scanner_state_init(&result->external_scanner_state,
self->lexer.debug_buffer, external_scanner_state_len);
ts_external_scanner_state_init(&result->external_scanner_state, self->lexer.debug_buffer, external_scanner_state_len);
result->has_external_scanner_state_change = external_scanner_state_changed;
}
}
@ -513,8 +484,7 @@ static t_subtree ts_parser__lex(TSParser *self, t_stack_version version,
// The decision is based on the trees' error costs (if any),
// their dynamic precedence, and finally, as a default, by a
// recursive comparison of the trees' symbols.
static bool ts_parser__select_tree(TSParser *self, t_subtree left,
t_subtree right)
static bool ts_parser__select_tree(TSParser *self, t_subtree left, t_subtree right)
{
int comparison;
@ -555,19 +525,16 @@ static bool ts_parser__select_tree(TSParser *self, t_subtree left,
// Determine if a given tree's children should be replaced
// by an alternative array of children.
static bool ts_parser__select_children(TSParser *self, t_subtree left,
const t_vec_subtree *children)
static bool ts_parser__select_children(TSParser *self, t_subtree left, const t_vec_subtree *children)
{
t_subtree scratch_tree;
vec_subtree_copy_into(&self->scratch_trees, (void *)children);
scratch_tree = ts_subtree_new_node(ts_subtree_symbol(left),
&self->scratch_trees, 0, self->language);
scratch_tree = ts_subtree_new_node(ts_subtree_symbol(left), &self->scratch_trees, 0, self->language);
return (ts_parser__select_tree(self, left, (scratch_tree)));
}
static void ts_parser__shift(TSParser *self, t_stack_version version,
TSStateId state, t_subtree lookahead, bool extra)
static void ts_parser__shift(TSParser *self, t_stack_version version, TSStateId state, t_subtree lookahead, bool extra)
{
t_subtree result;
bool is_leaf;
@ -583,14 +550,11 @@ static void ts_parser__shift(TSParser *self, t_stack_version version,
}
ts_stack_push(self->stack, version, subtree_to_push, !is_leaf, state);
if (ts_subtree_has_external_tokens(subtree_to_push))
ts_stack_set_last_external_token(self->stack, version,
ts_subtree_last_external_token(subtree_to_push));
ts_stack_set_last_external_token(self->stack, version, ts_subtree_last_external_token(subtree_to_push));
}
static t_stack_version ts_parser__reduce(TSParser *self,
t_stack_version version, TSSymbol symbol, t_u32 count,
int dynamic_precedence, t_u16 production_id, bool is_fragile,
bool end_of_non_terminal_extra)
static t_stack_version ts_parser__reduce(TSParser *self, t_stack_version version, TSSymbol symbol, t_u32 count, int dynamic_precedence,
t_u16 production_id, bool is_fragile, bool end_of_non_terminal_extra)
{
t_u32 initial_version_count;
t_stack_slice_array pop;
@ -645,10 +609,8 @@ static t_stack_version ts_parser__reduce(TSParser *self,
// re-pushed onto the stack after the parent node is
// created and pushed.
children = slice.subtrees;
ts_subtree_array_remove_trailing_extras(&children,
&self->trailing_extras);
parent = ts_subtree_new_node(symbol, &children, production_id,
self->language);
ts_subtree_array_remove_trailing_extras(&children, &self->trailing_extras);
parent = ts_subtree_new_node(symbol, &children, production_id, self->language);
// This pop operation may have caused multiple stack
// versions to collapse into one, because they all
// diverged from a common state. In that case,
@ -662,18 +624,15 @@ static t_stack_version ts_parser__reduce(TSParser *self,
break;
i++;
next_slice_children = next_slice.subtrees;
ts_subtree_array_remove_trailing_extras(&next_slice_children,
&self->trailing_extras2);
if (ts_parser__select_children(self, (parent),
&next_slice_children))
ts_subtree_array_remove_trailing_extras(&next_slice_children, &self->trailing_extras2);
if (ts_parser__select_children(self, (parent), &next_slice_children))
{
ts_subtree_array_clear(
/*&self->tree_pool,*/ &self->trailing_extras);
ts_subtree_release(
/*&self->tree_pool,*/ (parent));
array_swap(&self->trailing_extras, &self->trailing_extras2);
parent = ts_subtree_new_node(symbol, &next_slice_children,
production_id, self->language);
parent = ts_subtree_new_node(symbol, &next_slice_children, production_id, self->language);
}
else
{
@ -705,8 +664,7 @@ static t_stack_version ts_parser__reduce(TSParser *self,
ts_stack_push(self->stack, slice_version, (parent), false, next_state);
for (j = 0; j < self->trailing_extras.len; j++)
{
ts_stack_push(self->stack, slice_version,
self->trailing_extras.buffer[j], false, next_state);
ts_stack_push(self->stack, slice_version, self->trailing_extras.buffer[j], false, next_state);
}
for (k = 0; k < slice_version; k++)
{
@ -723,8 +681,7 @@ static t_stack_version ts_parser__reduce(TSParser *self,
return ts_stack_version_count(self->stack) > initial_version_count ? initial_version_count : STACK_VERSION_NONE;
}
static void ts_parser__accept(TSParser *self, t_stack_version version,
t_subtree lookahead)
static void ts_parser__accept(TSParser *self, t_stack_version version, t_subtree lookahead)
{
t_u32 child_count;
const t_subtree *children;
@ -752,10 +709,8 @@ static void ts_parser__accept(TSParser *self, t_stack_version version,
children = ts_subtree_children(tree);
for (k = 0; k < child_count; k++)
children[k]->ref_count++;
vec_subtree_splice(&trees, vec_subtree_splice_args(j, 1,
child_count, children));
root = (ts_subtree_new_node(ts_subtree_symbol(tree), &trees,
tree->production_id, self->language));
vec_subtree_splice(&trees, vec_subtree_splice_args(j, 1, child_count, children));
root = (ts_subtree_new_node(ts_subtree_symbol(tree), &trees, tree->production_id, self->language));
ts_subtree_release(tree);
break;
}
@ -778,8 +733,7 @@ static void ts_parser__accept(TSParser *self, t_stack_version version,
ts_stack_halt(self->stack, version);
}
static bool ts_parser__do_all_potential_reductions(TSParser *self,
t_stack_version starting_version, TSSymbol lookahead_symbol)
static bool ts_parser__do_all_potential_reductions(TSParser *self, t_stack_version starting_version, TSSymbol lookahead_symbol)
{
t_u32 initial_version_count;
bool can_shift_lookahead_symbol;
@ -846,8 +800,7 @@ static bool ts_parser__do_all_potential_reductions(TSParser *self,
break;
case TSParseActionTypeReduce:
if (action.reduce.child_count > 0)
ts_reduce_action_set_add(&self->reduce_actions,
(ReduceAction){
ts_reduce_action_set_add(&self->reduce_actions, (ReduceAction){
.symbol = action.reduce.symbol,
.count = action.reduce.child_count,
.dynamic_precedence = action.reduce.dynamic_precedence,
@ -863,15 +816,12 @@ static bool ts_parser__do_all_potential_reductions(TSParser *self,
for (k = 0; k < self->reduce_actions.size; k++)
{
reduce_action = self->reduce_actions.contents[k];
reduction_version = ts_parser__reduce(self, version,
reduce_action.symbol, reduce_action.count,
reduce_action.dynamic_precedence,
reduce_action.production_id, true, false);
reduction_version = ts_parser__reduce(self, version, reduce_action.symbol, reduce_action.count,
reduce_action.dynamic_precedence, reduce_action.production_id, true, false);
}
if (has_shift_action)
can_shift_lookahead_symbol = true;
else if (reduction_version != STACK_VERSION_NONE
&& i < MAX_VERSION_COUNT)
else if (reduction_version != STACK_VERSION_NONE && i < MAX_VERSION_COUNT)
{
ts_stack_renumber_version(self->stack, reduction_version, version);
continue;
@ -886,8 +836,7 @@ static bool ts_parser__do_all_potential_reductions(TSParser *self,
return can_shift_lookahead_symbol;
}
static bool ts_parser__recover_to_state(TSParser *self, t_stack_version version,
t_u32 depth, TSStateId goal_state)
static bool ts_parser__recover_to_state(TSParser *self, t_stack_version version, t_u32 depth, TSStateId goal_state)
{
t_stack_slice_array pop;
t_stack_version previous_version;
@ -925,19 +874,16 @@ static bool ts_parser__recover_to_state(TSParser *self, t_stack_version version,
error_child_count = ts_subtree_child_count(error_tree);
if (error_child_count > 0)
{
vec_subtree_splice(&slice.subtrees, vec_subtree_splice_args(0,
0, error_child_count, ts_subtree_children(error_tree)));
vec_subtree_splice(&slice.subtrees, vec_subtree_splice_args(0, 0, error_child_count, ts_subtree_children(error_tree)));
for (j = 0; j < error_child_count; j++)
slice.subtrees.buffer[j]->ref_count++;
}
ts_subtree_array_delete(&error_trees);
}
ts_subtree_array_remove_trailing_extras(&slice.subtrees,
&self->trailing_extras);
ts_subtree_array_remove_trailing_extras(&slice.subtrees, &self->trailing_extras);
if (slice.subtrees.len > 0)
{
error = ts_subtree_new_error_node(&slice.subtrees, true,
self->language);
error = ts_subtree_new_error_node(&slice.subtrees, true, self->language);
ts_stack_push(self->stack, slice.version, error, false, goal_state);
}
else
@ -954,8 +900,7 @@ static bool ts_parser__recover_to_state(TSParser *self, t_stack_version version,
return previous_version != STACK_VERSION_NONE;
}
static void ts_parser__recover(TSParser *self, t_stack_version version,
t_subtree lookahead)
static void ts_parser__recover(TSParser *self, t_stack_version version, t_subtree lookahead)
{
Length position;
bool did_recover;
@ -981,8 +926,7 @@ static void ts_parser__recover(TSParser *self, t_stack_version version,
previous_version_count = ts_stack_version_count(self->stack);
position = ts_stack_position(self->stack, version);
summary = ts_stack_get_summary(self->stack, version);
node_count_since_error = ts_stack_node_count_since_error(self->stack,
version);
node_count_since_error = ts_stack_node_count_since_error(self->stack, version);
current_error_cost = ts_stack_error_cost(self->stack, version);
// When the parser is in the error state, there are two
// strategies for recovering with a given lookahead
@ -1021,9 +965,7 @@ static void ts_parser__recover(TSParser *self, t_stack_version version,
would_merge = false;
for (j = 0; j < previous_version_count; j++)
{
if (ts_stack_state(self->stack, j) == entry.state
&& ts_stack_position(self->stack,
j).bytes == position.bytes)
if (ts_stack_state(self->stack, j) == entry.state && ts_stack_position(self->stack, j).bytes == position.bytes)
{
would_merge = true;
break;
@ -1033,22 +975,17 @@ static void ts_parser__recover(TSParser *self, t_stack_version version,
continue;
// Do not recover if the result would clearly be
// worse than some existing stack version.
new_cost = current_error_cost + entry.depth
* ERROR_COST_PER_SKIPPED_TREE + (position.bytes
- entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR
+ (position.extent.row - entry.position.extent.row)
* ERROR_COST_PER_SKIPPED_LINE;
if (ts_parser__better_version_exists(self, version, false,
new_cost))
new_cost = current_error_cost + entry.depth * ERROR_COST_PER_SKIPPED_TREE +
(position.bytes - entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR +
(position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE;
if (ts_parser__better_version_exists(self, version, false, new_cost))
break;
// If the current lookahead token is valid in
// some previous state, recover to that state.
// Then stop looking for further recoveries.
if (ts_language_has_actions(self->language, entry.state,
ts_subtree_symbol(lookahead)))
if (ts_language_has_actions(self->language, entry.state, ts_subtree_symbol(lookahead)))
{
if (ts_parser__recover_to_state(self, version, depth,
entry.state))
if (ts_parser__recover_to_state(self, version, depth, entry.state))
{
did_recover = true;
break;
@ -1094,10 +1031,8 @@ static void ts_parser__recover(TSParser *self, t_stack_version version,
}
// Do not recover if the result would clearly be worse
// than some existing stack version.
new_cost = current_error_cost + ERROR_COST_PER_SKIPPED_TREE
+ ts_subtree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR
+ ts_subtree_total_size(lookahead).extent.row
* ERROR_COST_PER_SKIPPED_LINE;
new_cost = current_error_cost + ERROR_COST_PER_SKIPPED_TREE + ts_subtree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR +
ts_subtree_total_size(lookahead).extent.row * ERROR_COST_PER_SKIPPED_LINE;
if (ts_parser__better_version_exists(self, version, false, new_cost))
{
ts_stack_halt(self->stack, version);
@ -1107,10 +1042,8 @@ static void ts_parser__recover(TSParser *self, t_stack_version version,
// If the current lookahead token is an extra token,
// mark it as extra. This means it won't be counted in
// error cost calculations.
actions = ts_language_actions(self->language, 1,
ts_subtree_symbol(lookahead), &n);
if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n
- 1].shift.extra)
actions = ts_language_actions(self->language, 1, ts_subtree_symbol(lookahead), &n);
if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].shift.extra)
{
mutable_lookahead = ts_subtree_ensure_owner(lookahead);
ts_subtree_set_extra(&mutable_lookahead, true);
@ -1119,8 +1052,7 @@ static void ts_parser__recover(TSParser *self, t_stack_version version,
// Wrap the lookahead token in an ERROR.
children = vec_subtree_new(1, NULL);
vec_subtree_push(&children, lookahead);
error_repeat = ts_subtree_new_node(ts_builtin_sym_error_repeat, &children,
0, self->language);
error_repeat = ts_subtree_new_node(ts_builtin_sym_error_repeat, &children, 0, self->language);
// If other tokens have already been skipped, so there
// is already an ERROR at the top of the stack, then pop
// that ERROR off the stack and wrap the two ERRORs
@ -1139,26 +1071,20 @@ static void ts_parser__recover(TSParser *self, t_stack_version version,
for (i = 1; i < pop.size; i++)
ts_subtree_array_delete(
/*&self->tree_pool,*/ &pop.contents[i].subtrees);
while (ts_stack_version_count(self->stack) > pop.contents[0].version
+ 1)
ts_stack_remove_version(self->stack, pop.contents[0].version
+ 1);
while (ts_stack_version_count(self->stack) > pop.contents[0].version + 1)
ts_stack_remove_version(self->stack, pop.contents[0].version + 1);
}
ts_stack_renumber_version(self->stack, pop.contents[0].version,
version);
ts_stack_renumber_version(self->stack, pop.contents[0].version, version);
vec_subtree_push(&pop.contents[0].subtrees, (error_repeat));
error_repeat = ts_subtree_new_node(ts_builtin_sym_error_repeat,
&pop.contents[0].subtrees, 0, self->language);
error_repeat = ts_subtree_new_node(ts_builtin_sym_error_repeat, &pop.contents[0].subtrees, 0, self->language);
}
// Push the new ERROR onto the stack.
ts_stack_push(self->stack, version, (error_repeat), false, ERROR_STATE);
if (ts_subtree_has_external_tokens(lookahead))
ts_stack_set_last_external_token(self->stack, version,
ts_subtree_last_external_token(lookahead));
ts_stack_set_last_external_token(self->stack, version, ts_subtree_last_external_token(lookahead));
}
static void ts_parser__handle_error(TSParser *self, t_stack_version version,
t_subtree lookahead)
static void ts_parser__handle_error(TSParser *self, t_stack_version version, t_subtree lookahead)
{
TSSymbol missing_symbol;
bool did_insert_missing_token;
@ -1191,14 +1117,10 @@ static void ts_parser__handle_error(TSParser *self, t_stack_version version,
state = ts_stack_state(self->stack, v);
for (missing_symbol = 1; missing_symbol < (t_u16)self->language->token_count; missing_symbol++)
{
state_after_missing_symbol = ts_language_next_state(self->language,
state, missing_symbol);
if (state_after_missing_symbol == 0
|| state_after_missing_symbol == state)
state_after_missing_symbol = ts_language_next_state(self->language, state, missing_symbol);
if (state_after_missing_symbol == 0 || state_after_missing_symbol == state)
continue;
if (ts_language_has_reduce_action(self->language,
state_after_missing_symbol,
ts_subtree_leaf_symbol(lookahead)))
if (ts_language_has_reduce_action(self->language, state_after_missing_symbol, ts_subtree_leaf_symbol(lookahead)))
{
// In case the parser is currently
// outside of any included range, the
@ -1209,19 +1131,12 @@ static void ts_parser__handle_error(TSParser *self, t_stack_version version,
// range.
ts_lexer_reset(&self->lexer, position);
ts_lexer__mark_end((void *)&self->lexer);
padding = length_sub(self->lexer.token_end_position,
position);
lookahead_bytes = ts_subtree_total_bytes(lookahead)
+ ts_subtree_lookahead_bytes(lookahead);
version_with_missing_tree = ts_stack_copy_version(self->stack,
v);
missing_tree = ts_subtree_new_missing_leaf(missing_symbol,
padding, lookahead_bytes, self->language);
ts_stack_push(self->stack, version_with_missing_tree,
missing_tree, false, state_after_missing_symbol);
if (ts_parser__do_all_potential_reductions(self,
version_with_missing_tree,
ts_subtree_leaf_symbol(lookahead)))
padding = length_sub(self->lexer.token_end_position, position);
lookahead_bytes = ts_subtree_total_bytes(lookahead) + ts_subtree_lookahead_bytes(lookahead);
version_with_missing_tree = ts_stack_copy_version(self->stack, v);
missing_tree = ts_subtree_new_missing_leaf(missing_symbol, padding, lookahead_bytes, self->language);
ts_stack_push(self->stack, version_with_missing_tree, missing_tree, false, state_after_missing_symbol);
if (ts_parser__do_all_potential_reductions(self, version_with_missing_tree, ts_subtree_leaf_symbol(lookahead)))
{
did_insert_missing_token = true;
break;
@ -1247,8 +1162,7 @@ static void ts_parser__handle_error(TSParser *self, t_stack_version version,
ts_parser__recover(self, version, lookahead);
}
static bool ts_parser__advance(TSParser *self, t_stack_version version,
bool allow_node_reuse)
static bool ts_parser__advance(TSParser *self, t_stack_version version, bool allow_node_reuse)
{
TSStateId state;
t_subtree mutable_lookahead;
@ -1279,8 +1193,7 @@ static bool ts_parser__advance(TSParser *self, t_stack_version version,
return false;
if (lookahead)
{
ts_language_table_entry(self->language, state,
ts_subtree_symbol(lookahead), &table_entry);
ts_language_table_entry(self->language, state, ts_subtree_symbol(lookahead), &table_entry);
}
// When parsing a non-terminal extra, a null
// lookahead indicates the end of the rule. The
@ -1289,8 +1202,7 @@ static bool ts_parser__advance(TSParser *self, t_stack_version version,
// run again.
else
{
ts_language_table_entry(self->language, state,
ts_builtin_sym_end, &table_entry);
ts_language_table_entry(self->language, state, ts_builtin_sym_end, &table_entry);
}
}
// Process each parse action for the current
@ -1305,8 +1217,7 @@ static bool ts_parser__advance(TSParser *self, t_stack_version version,
action = table_entry.actions[i];
switch (action.type)
{
case TSParseActionTypeShift:
{
case TSParseActionTypeShift: {
if (action.shift.repetition)
break;
if (action.shift.extra)
@ -1319,35 +1230,28 @@ static bool ts_parser__advance(TSParser *self, t_stack_version version,
}
if (ts_subtree_child_count(lookahead) > 0)
{
next_state = ts_language_next_state(self->language, state,
ts_subtree_symbol(lookahead));
next_state = ts_language_next_state(self->language, state, ts_subtree_symbol(lookahead));
}
ts_parser__shift(self, version, next_state, lookahead,
action.shift.extra);
ts_parser__shift(self, version, next_state, lookahead, action.shift.extra);
return true;
}
case TSParseActionTypeReduce:
{
case TSParseActionTypeReduce: {
is_fragile = table_entry.action_count > 1;
end_of_non_terminal_extra = lookahead == NULL;
reduction_version = ts_parser__reduce(self, version,
action.reduce.symbol, action.reduce.child_count,
action.reduce.dynamic_precedence,
action.reduce.production_id, is_fragile,
end_of_non_terminal_extra);
reduction_version =
ts_parser__reduce(self, version, action.reduce.symbol, action.reduce.child_count, action.reduce.dynamic_precedence,
action.reduce.production_id, is_fragile, end_of_non_terminal_extra);
if (reduction_version != STACK_VERSION_NONE)
{
last_reduction_version = reduction_version;
}
break;
}
case TSParseActionTypeAccept:
{
case TSParseActionTypeAccept: {
ts_parser__accept(self, version, lookahead);
return true;
}
case TSParseActionTypeRecover:
{
case TSParseActionTypeRecover: {
ts_parser__recover(self, version, lookahead);
return true;
}
@ -1360,8 +1264,7 @@ static bool ts_parser__advance(TSParser *self, t_stack_version version,
// same lookahead symbol.
if (last_reduction_version != STACK_VERSION_NONE)
{
ts_stack_renumber_version(self->stack, last_reduction_version,
version);
ts_stack_renumber_version(self->stack, last_reduction_version, version);
state = ts_stack_state(self->stack, version);
// At the end of a non-terminal extra rule, the
// lexer will return a null subtree, because the
@ -1373,8 +1276,7 @@ static bool ts_parser__advance(TSParser *self, t_stack_version version,
if (!lookahead)
needs_lex = true;
else
ts_language_table_entry(self->language, state,
ts_subtree_leaf_symbol(lookahead), &table_entry);
ts_language_table_entry(self->language, state, ts_subtree_leaf_symbol(lookahead), &table_entry);
continue;
}
// A non-terminal extra rule was reduced and merged
@ -1390,16 +1292,13 @@ static bool ts_parser__advance(TSParser *self, t_stack_version version,
// state. If the current lookahead token is a
// keyword, then switch to treating it as the normal
// word token if that token is valid in this state.
if (ts_subtree_is_keyword(lookahead)
&& ts_subtree_symbol(lookahead) != self->language->keyword_capture_token)
if (ts_subtree_is_keyword(lookahead) && ts_subtree_symbol(lookahead) != self->language->keyword_capture_token)
{
ts_language_table_entry(self->language, state,
self->language->keyword_capture_token, &table_entry);
ts_language_table_entry(self->language, state, self->language->keyword_capture_token, &table_entry);
if (table_entry.action_count > 0)
{
mutable_lookahead = ts_subtree_ensure_owner(lookahead);
ts_subtree_set_symbol(&mutable_lookahead,
self->language->keyword_capture_token, self->language);
ts_subtree_set_symbol(&mutable_lookahead, self->language->keyword_capture_token, self->language);
lookahead = mutable_lookahead;
continue;
}
@ -1526,8 +1425,7 @@ static t_u32 ts_parser__condense_stack(TSParser *self)
{
if (ts_stack_is_paused(self->stack, i))
{
if (!has_unpaused_version
&& self->accept_count < MAX_VERSION_COUNT)
if (!has_unpaused_version && self->accept_count < MAX_VERSION_COUNT)
{
min_error_cost = ts_stack_error_cost(self->stack, i);
lookahead = ts_stack_resume(self->stack, i);
@ -1552,8 +1450,7 @@ static t_u32 ts_parser__condense_stack(TSParser *self)
static bool ts_parser_has_outstanding_parse(TSParser *self)
{
return (self->external_scanner_payload || ts_stack_state(self->stack,
0) != 1 || ts_stack_node_count_since_error(self->stack, 0) != 0);
return (self->external_scanner_payload || ts_stack_state(self->stack, 0) != 1 || ts_stack_node_count_since_error(self->stack, 0) != 0);
}
// Parser - Public
@ -1647,8 +1544,7 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input)
version = 0;
do
{
for (version = 0; version_count = ts_stack_version_count(self->stack),
version < version_count; version++)
for (version = 0; version_count = ts_stack_version_count(self->stack), version < version_count; version++)
{
allow_node_reuse = version_count == 1;
while (ts_stack_is_active(self->stack, version))
@ -1663,8 +1559,7 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input)
return NULL;
}
position = ts_stack_position(self->stack, version).bytes;
if (position > last_position || (version > 0
&& position == last_position))
if (position > last_position || (version > 0 && position == last_position))
{
last_position = position;
break;
@ -1681,8 +1576,7 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input)
// remove any extra references to subtrees within
// the finished tree, ensuring that these subtrees
// can be safely mutated in-place for rebalancing.
if (self->finished_tree
&& ts_subtree_error_cost(self->finished_tree) < min_error_cost)
if (self->finished_tree && ts_subtree_error_cost(self->finished_tree) < min_error_cost)
{
ts_stack_clear(self->stack);
break;
@ -1690,20 +1584,19 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input)
} while (version_count != 0);
if (self->finished_tree == NULL)
me_abort("self->finished_tree == NULL");
ts_subtree_balance(self->finished_tree, self->language);
result = ts_tree_new(self->finished_tree, self->language);
self->finished_tree = NULL;
ts_parser_reset(self);
return result;
}
TSTree *ts_parser_parse_string(TSParser *self, t_const_str string,
t_u32 length)
TSTree *ts_parser_parse_string(TSParser *self, t_const_str string, t_u32 length)
{
t_string_input input;
input = (t_string_input){(const t_u8 *)string, length};
return ts_parser_parse(self,
(TSInput){
return ts_parser_parse(self, (TSInput){
&input,
ts_string_input_read,
});

81
parser/src/subtree/subtree_balance.c Normal file
View file

@ -0,0 +1,81 @@
#include "me/vec/vec_subtree.h"
#include "parser/inner/ptypes.h"
#include "parser/subtree.h"
void ts_subtree__compress(t_subtree self, t_u32 count, const t_language *language, t_vec_subtree *stack)
{
t_u32 initial_stack_size = stack->len;
t_subtree tree = self;
TSSymbol symbol = tree->symbol;
for (t_u32 i = 0; i < count; i++)
{
if (tree->ref_count > 1 || tree->child_count < 2)
break;
t_subtree child = (ts_subtree_children(tree)[0]);
if (child->child_count < 2 || child->ref_count > 1 || child->symbol != symbol)
break;
t_subtree grandchild = (ts_subtree_children(child)[0]);
if (grandchild->child_count < 2 || grandchild->ref_count > 1 || grandchild->symbol != symbol)
break;
ts_subtree_children(tree)[0] = (grandchild);
ts_subtree_children(child)[0] = ts_subtree_children(grandchild)[grandchild->child_count - 1];
ts_subtree_children(grandchild)[grandchild->child_count - 1] = (child);
vec_subtree_push(stack, tree);
tree = grandchild;
}
while (stack->len > initial_stack_size)
{
vec_subtree_pop(stack, &tree);
t_subtree child = (ts_subtree_children(tree)[0]);
t_subtree grandchild = (ts_subtree_children(child)[child->child_count - 1]);
ts_subtree_summarize_children(grandchild, language);
ts_subtree_summarize_children(child, language);
ts_subtree_summarize_children(tree, language);
}
}
void ts_subtree_balance(t_subtree self, const t_language *language)
{
// return ;
t_vec_subtree tree_stack;
t_subtree tree;
tree_stack = vec_subtree_new(16, NULL);
if (ts_subtree_child_count(self) > 0 && self->ref_count == 1)
vec_subtree_push(&tree_stack, (self));
while (tree_stack.len > 0)
{
vec_subtree_pop(&tree_stack, &tree);
if (tree->repeat_depth > 0)
{
t_subtree child1 = ts_subtree_children(tree)[0];
t_subtree child2 = ts_subtree_children(tree)[tree->child_count - 1];
t_i64 repeat_delta = (t_i64)ts_subtree_repeat_depth(child1) - (t_i64)ts_subtree_repeat_depth(child2);
if (repeat_delta > 0)
{
t_u32 n = (t_u32)repeat_delta;
for (t_u32 i = n / 2; i > 0; i /= 2)
{
ts_subtree__compress(tree, i, language, &tree_stack);
n -= i;
}
}
}
for (t_u32 i = 0; i < tree->child_count; i++)
{
t_subtree child = ts_subtree_children(tree)[i];
if (ts_subtree_child_count(child) > 0 && child->ref_count == 1)
vec_subtree_push(&tree_stack, (child));
}
}
vec_subtree_free(tree_stack);
}

24
stdme/src/os/pipe.c
View file

@ -10,9 +10,13 @@
/* */
/* ************************************************************************** */
#include "fcntl.h"
#include "me/fs/fs.h"
#include "me/os/os.h"
#include "me/str/str.h"
#include "me/string/string.h"
#include "me/printf/printf.h"
#include "unistd.h"
static t_error _pipe_get_fd_slot(struct s_file_slot **read,
struct s_file_slot **write)
@ -28,6 +32,25 @@ static t_error _pipe_get_fd_slot(struct s_file_slot **read,
return (NO_ERROR);
}
static void _pipe_set_close_exec(t_fd *fd)
{
t_string s;
int new_fd;
int fd_perm;
s = string_new(16);
me_printf_str(&s, "/proc/self/fd/%i", fd->fd);
fd_perm = O_RDONLY;
if (fd->perms == FD_WRITE)
fd_perm = O_WRONLY;
new_fd = open(s.buf, fd_perm | O_CLOEXEC);
string_free(s);
if (new_fd == -1)
me_abort("pipe close_on_exec failed");
close(fd->fd);
fd->fd = new_fd;
}
t_error create_pipe(t_pipe *out)
{
int fd[2];
@ -52,5 +75,6 @@ t_error create_pipe(t_pipe *out)
write->slot.fd.name = str_clone("<pipe[write]>");
out->read = &read->slot.fd;
out->write = &write->slot.fd;
(_pipe_set_close_exec(out->read), _pipe_set_close_exec(out->write));
return (NO_ERROR);
}