raphael/minishell
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Removing more fat from the parser lib

Browse source
This commit is contained in:
Maix0 2024-08-19 14:19:12 +02:00
parent a6246a52a0
commit c1209452cd
11 changed files with 360 additions and 566 deletions
Download patch file Download diff file Expand all files Collapse all files

9
Makefile
View file

@ -6,7 +6,7 @@
# By: rparodi <rparodi@student.42.fr> +#+ +:+ +#+ #
# +#+#+#+#+#+ +#+ #
# Created: 2023/11/12 11:05:05 by rparodi #+# #+# #
# Updated: 2024/08/18 22:01:52 by maiboyer ### ########.fr #
# Updated: 2024/08/19 14:00:31 by maiboyer ### ########.fr #
# #
# **************************************************************************** #
@ -45,15 +45,16 @@ endif
# TODO: ADD THIS WHEN FINISHING THIS:
# CFLAGS_ADDITIONAL += -DNVALGRIND
CFLAGS_ADDITIONAL += -O0 -Wno-cpp -Wno-type-limits
# TODO: REMOVE THIS WHEN FINISHING THIS:
CFLAGS_ADDITIONAL += -O0
CFLAGS_ADDITIONAL += -Wno-cpp -Wno-type-limits
CFLAGS_ADDITIONAL += -gcolumn-info -g3
CFLAGS_ADDITIONAL += '-DERROR=((void)printf("ERROR HERE: " __FILE__ ":%d in %s\n", __LINE__, __func__), 1)'
CFLAGS_ADDITIONAL += '-Dinline=' -Wno-unused-function
#CFLAGS_ADDITIONAL += -O2
# CFLAGS_ADDITIONAL += -fuse-ld=gold -Wl,--print-symbol-counts -Wl,/tmp/symbols_count.log
#CFLAGS_ADDITIONAL += -fuse-ld=lld -Wl,--verbose -ffunction-sections -fdata-sections -Wl,--gc-sections -Wl,--print-gc-sections -Wl,-Map=output.map
CFLAGS_ADDITIONAL += -fuse-ld=lld -ffunction-sections -fdata-sections -Wl,--gc-sections -Wl,-O0
export CFLAGS_ADDITIONAL
export CC

4
parser/include/parser/array.h
View file

@ -181,7 +181,7 @@ static inline void _array__assign(Array *self, const Array *other, size_t elemen
{
_array__reserve(self, element_size, other->size);
self->size = other->size;
memcpy(self->contents, other->contents, self->size * element_size);
mem_copy(self->contents, other->contents, self->size * element_size);
}
/// This is not what you're looking for, see `array_swap`.
@ -226,7 +226,7 @@ static inline void _array__splice(Array *self, size_t element_size, t_u32 index,
{
if (elements)
{
memcpy((contents + index * element_size), elements, new_count * element_size);
mem_copy((contents + index * element_size), elements, new_count * element_size);
}
else
{

60
parser/include/parser/subtree.h
View file

@ -19,7 +19,6 @@ struct ExternalScannerState
{
union {
char *long_data;
char short_data[24];
};
t_u32 length;
};
@ -36,37 +35,6 @@ struct ExternalScannerState
// separately on the heap.
typedef struct ExternalScannerState ExternalScannerState;
// A compact representation of a subtree.
//
// This representation is used for small leaf nodes that are not
// errors, and were not created by an external scanner.
//
// The idea behind the layout of this struct is that the `is_inline`
// bit will fall exactly into the same location as the least significant
// bit of the pointer in `Subtree` or `MutableSubtree`, respectively.
// Because of alignment, for any valid pointer this will be 0, giving
// us the opportunity to make use of this bit to signify whether to use
// the pointer or the inline struct.
typedef struct SubtreeInlineData SubtreeInlineData_;
struct SubtreeInlineData
{
bool is_inline : 1;
bool visible : 1;
bool named : 1;
bool extra : 1;
bool has_changes : 1;
bool is_missing : 1;
bool is_keyword : 1;
t_u8 symbol;
t_u16 parse_state;
t_u8 padding_columns;
t_u8 padding_rows : 4;
t_u8 lookahead_bytes : 4;
t_u8 padding_bytes;
t_u8 size_bytes;
};
// A heap-allocated representation of a subtree.
//
// This representation is used for parent nodes, external tokens,
@ -74,7 +42,7 @@ struct SubtreeInlineData
// the inline representation.
typedef struct SubtreeHeapData
{
volatile t_u32 ref_count;
t_u32 ref_count;
Length padding;
Length size;
t_u32 lookahead_bytes;
@ -270,24 +238,11 @@ static inline t_u32 ts_subtree_repeat_depth(Subtree self)
return (self.ptr->repeat_depth);
}
static inline t_u32 ts_subtree_is_repetition(Subtree self)
{
return (!self.ptr->named && !self.ptr->visible && self.ptr->child_count != 0);
}
static inline t_u32 ts_subtree_visible_descendant_count(Subtree self)
{
return ((self.ptr->child_count == 0) ? 0 : self.ptr->visible_descendant_count);
}
static inline t_u32 ts_subtree_visible_child_count(Subtree self)
{
if (ts_subtree_child_count(self) > 0)
return (self.ptr->visible_child_count);
else
return 0;
}
static inline t_u32 ts_subtree_error_cost(Subtree self)
{
if (ts_subtree_missing(self))
@ -301,14 +256,6 @@ static inline t_i32 ts_subtree_dynamic_precedence(Subtree self)
return ((self.ptr->child_count == 0) ? 0 : self.ptr->dynamic_precedence);
}
static inline t_u16 ts_subtree_production_id(Subtree self)
{
if (ts_subtree_child_count(self) > 0)
return (self.ptr->production_id);
else
return (0);
}
static inline bool ts_subtree_fragile_left(Subtree self)
{
return (self.ptr->fragile_left);
@ -334,11 +281,6 @@ static inline bool ts_subtree_depends_on_column(Subtree self)
return (self.ptr->depends_on_column);
}
static inline bool ts_subtree_is_fragile(Subtree self)
{
return ((self.ptr->fragile_left || self.ptr->fragile_right));
}
static inline bool ts_subtree_is_error(Subtree self)
{
return (ts_subtree_symbol(self) == ts_builtin_sym_error);

8
parser/src/lexer.c
View file

@ -1,8 +1,8 @@
#include "parser/lexer.h"
#include "parser/length.h"
#include "parser/input.h"
#include "me/mem/mem.h"
#include "me/types.h"
#include "parser/input.h"
#include "parser/length.h"
#include <string.h>
#define LOG(...)
@ -427,7 +427,7 @@ bool ts_lexer_set_included_ranges(Lexer *self, const TSRange *ranges, t_u32 coun
size_t size = count * sizeof(TSRange);
self->included_ranges = mem_realloc(self->included_ranges, size);
memcpy(self->included_ranges, ranges, size);
mem_copy(self->included_ranges, ranges, size);
self->included_range_count = count;
ts_lexer_goto(self, self->current_position);
return true;
@ -438,5 +438,3 @@ TSRange *ts_lexer_included_ranges(const Lexer *self, t_u32 *count)
*count = self->included_range_count;
return self->included_ranges;
}
#undef LOG

611
parser/src/node.c
View file

@ -186,284 +186,270 @@ static inline TSNode ts_node__child(TSNode self, t_u32 child_index, bool include
return ts_node__null();
}
static bool ts_subtree_has_trailing_empty_descendant(Subtree self, Subtree other)
{
for (t_u32 i = ts_subtree_child_count(self) - 1; i + 1 > 0; i--)
{
Subtree child = ts_subtree_children(self)[i];
if (ts_subtree_total_bytes(child) > 0)
break;
if (child.ptr == other.ptr || ts_subtree_has_trailing_empty_descendant(child, other))
{
return true;
}
}
return false;
}
static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous)
{
Subtree self_subtree = ts_node__subtree(self);
bool self_is_empty = ts_subtree_total_bytes(self_subtree) == 0;
t_u32 target_end_byte = ts_node_end_byte(self);
TSNode node = ts_node_parent(self);
TSNode earlier_node = ts_node__null();
bool earlier_node_is_relevant = false;
while (!ts_node_is_null(node))
{
TSNode earlier_child = ts_node__null();
bool earlier_child_is_relevant = false;
bool found_child_containing_target = false;
TSNode child;
NodeChildIterator iterator = ts_node_iterate_children(&node);
while (ts_node_child_iterator_next(&iterator, &child))
{
if (child.id == self.id)
break;
if (iterator.position.bytes > target_end_byte)
{
found_child_containing_target = true;
break;
}
if (iterator.position.bytes == target_end_byte &&
(!self_is_empty || ts_subtree_has_trailing_empty_descendant(ts_node__subtree(child), self_subtree)))
{
found_child_containing_target = true;
break;
}
if (ts_node__is_relevant(child, include_anonymous))
{
earlier_child = child;
earlier_child_is_relevant = true;
}
else if (ts_node__relevant_child_count(child, include_anonymous) > 0)
{
earlier_child = child;
earlier_child_is_relevant = false;
}
}
if (found_child_containing_target)
{
if (!ts_node_is_null(earlier_child))
{
earlier_node = earlier_child;
earlier_node_is_relevant = earlier_child_is_relevant;
}
node = child;
}
else if (earlier_child_is_relevant)
{
return earlier_child;
}
else if (!ts_node_is_null(earlier_child))
{
node = earlier_child;
}
else if (earlier_node_is_relevant)
{
return earlier_node;
}
else
{
node = earlier_node;
earlier_node = ts_node__null();
earlier_node_is_relevant = false;
}
}
return ts_node__null();
}
static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous)
{
t_u32 target_end_byte = ts_node_end_byte(self);
TSNode node = ts_node_parent(self);
TSNode later_node = ts_node__null();
bool later_node_is_relevant = false;
while (!ts_node_is_null(node))
{
TSNode later_child = ts_node__null();
bool later_child_is_relevant = false;
TSNode child_containing_target = ts_node__null();
TSNode child;
NodeChildIterator iterator = ts_node_iterate_children(&node);
while (ts_node_child_iterator_next(&iterator, &child))
{
if (iterator.position.bytes < target_end_byte)
continue;
if (ts_node_start_byte(child) <= ts_node_start_byte(self))
{
if (ts_node__subtree(child).ptr != ts_node__subtree(self).ptr)
{
child_containing_target = child;
}
}
else if (ts_node__is_relevant(child, include_anonymous))
{
later_child = child;
later_child_is_relevant = true;
break;
}
else if (ts_node__relevant_child_count(child, include_anonymous) > 0)
{
later_child = child;
later_child_is_relevant = false;
break;
}
}
if (!ts_node_is_null(child_containing_target))
{
if (!ts_node_is_null(later_child))
{
later_node = later_child;
later_node_is_relevant = later_child_is_relevant;
}
node = child_containing_target;
}
else if (later_child_is_relevant)
{
return later_child;
}
else if (!ts_node_is_null(later_child))
{
node = later_child;
}
else if (later_node_is_relevant)
{
return later_node;
}
else
{
node = later_node;
}
}
return ts_node__null();
}
static inline TSNode ts_node__first_child_for_byte(TSNode self, t_u32 goal, bool include_anonymous)
{
TSNode node = self;
bool did_descend = true;
while (did_descend)
{
did_descend = false;
TSNode child;
NodeChildIterator iterator = ts_node_iterate_children(&node);
while (ts_node_child_iterator_next(&iterator, &child))
{
if (ts_node_end_byte(child) > goal)
{
if (ts_node__is_relevant(child, include_anonymous))
{
return child;
}
else if (ts_node_child_count(child) > 0)
{
did_descend = true;
node = child;
break;
}
}
}
}
return ts_node__null();
}
static inline TSNode ts_node__descendant_for_byte_range(TSNode self, t_u32 range_start, t_u32 range_end, bool include_anonymous)
{
TSNode node = self;
TSNode last_visible_node = self;
bool did_descend = true;
while (did_descend)
{
did_descend = false;
TSNode child;
NodeChildIterator iterator = ts_node_iterate_children(&node);
while (ts_node_child_iterator_next(&iterator, &child))
{
t_u32 node_end = iterator.position.bytes;
// The end of this node must extend far enough forward to touch
// the end of the range and exceed the start of the range.
if (node_end < range_end)
continue;
if (node_end <= range_start)
continue;
// The start of this node must extend far enough backward to
// touch the start of the range.
if (range_start < ts_node_start_byte(child))
break;
node = child;
if (ts_node__is_relevant(node, include_anonymous))
{
last_visible_node = node;
}
did_descend = true;
break;
}
}
return last_visible_node;
}
static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint range_start, TSPoint range_end, bool include_anonymous)
{
TSNode node = self;
TSNode last_visible_node = self;
bool did_descend = true;
while (did_descend)
{
did_descend = false;
TSNode child;
NodeChildIterator iterator = ts_node_iterate_children(&node);
while (ts_node_child_iterator_next(&iterator, &child))
{
TSPoint node_end = iterator.position.extent;
// The end of this node must extend far enough forward to touch
// the end of the range and exceed the start of the range.
if (point_lt(node_end, range_end))
continue;
if (point_lte(node_end, range_start))
continue;
// The start of this node must extend far enough backward to
// touch the start of the range.
if (point_lt(range_start, ts_node_start_point(child)))
break;
node = child;
if (ts_node__is_relevant(node, include_anonymous))
{
last_visible_node = node;
}
did_descend = true;
break;
}
}
return last_visible_node;
}
/* static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) */
/* { */
/* Subtree self_subtree = ts_node__subtree(self); */
/* bool self_is_empty = ts_subtree_total_bytes(self_subtree) == 0; */
/* t_u32 target_end_byte = ts_node_end_byte(self); */
/**/
/* TSNode node = ts_node_parent(self); */
/* TSNode earlier_node = ts_node__null(); */
/* bool earlier_node_is_relevant = false; */
/**/
/* while (!ts_node_is_null(node)) */
/* { */
/* TSNode earlier_child = ts_node__null(); */
/* bool earlier_child_is_relevant = false; */
/* bool found_child_containing_target = false; */
/**/
/* TSNode child; */
/* NodeChildIterator iterator = ts_node_iterate_children(&node); */
/* while (ts_node_child_iterator_next(&iterator, &child)) */
/* { */
/* if (child.id == self.id) */
/* break; */
/* if (iterator.position.bytes > target_end_byte) */
/* { */
/* found_child_containing_target = true; */
/* break; */
/* } */
/**/
/* if (iterator.position.bytes == target_end_byte && */
/* (!self_is_empty || ts_subtree_has_trailing_empty_descendant(ts_node__subtree(child), self_subtree))) */
/* { */
/* found_child_containing_target = true; */
/* break; */
/* } */
/**/
/* if (ts_node__is_relevant(child, include_anonymous)) */
/* { */
/* earlier_child = child; */
/* earlier_child_is_relevant = true; */
/* } */
/* else if (ts_node__relevant_child_count(child, include_anonymous) > 0) */
/* { */
/* earlier_child = child; */
/* earlier_child_is_relevant = false; */
/* } */
/* } */
/**/
/* if (found_child_containing_target) */
/* { */
/* if (!ts_node_is_null(earlier_child)) */
/* { */
/* earlier_node = earlier_child; */
/* earlier_node_is_relevant = earlier_child_is_relevant; */
/* } */
/* node = child; */
/* } */
/* else if (earlier_child_is_relevant) */
/* { */
/* return earlier_child; */
/* } */
/* else if (!ts_node_is_null(earlier_child)) */
/* { */
/* node = earlier_child; */
/* } */
/* else if (earlier_node_is_relevant) */
/* { */
/* return earlier_node; */
/* } */
/* else */
/* { */
/* node = earlier_node; */
/* earlier_node = ts_node__null(); */
/* earlier_node_is_relevant = false; */
/* } */
/* } */
/**/
/* return ts_node__null(); */
/* } */
/**/
/* static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) */
/* { */
/* t_u32 target_end_byte = ts_node_end_byte(self); */
/**/
/* TSNode node = ts_node_parent(self); */
/* TSNode later_node = ts_node__null(); */
/* bool later_node_is_relevant = false; */
/**/
/* while (!ts_node_is_null(node)) */
/* { */
/* TSNode later_child = ts_node__null(); */
/* bool later_child_is_relevant = false; */
/* TSNode child_containing_target = ts_node__null(); */
/**/
/* TSNode child; */
/* NodeChildIterator iterator = ts_node_iterate_children(&node); */
/* while (ts_node_child_iterator_next(&iterator, &child)) */
/* { */
/* if (iterator.position.bytes < target_end_byte) */
/* continue; */
/* if (ts_node_start_byte(child) <= ts_node_start_byte(self)) */
/* { */
/* if (ts_node__subtree(child).ptr != ts_node__subtree(self).ptr) */
/* { */
/* child_containing_target = child; */
/* } */
/* } */
/* else if (ts_node__is_relevant(child, include_anonymous)) */
/* { */
/* later_child = child; */
/* later_child_is_relevant = true; */
/* break; */
/* } */
/* else if (ts_node__relevant_child_count(child, include_anonymous) > 0) */
/* { */
/* later_child = child; */
/* later_child_is_relevant = false; */
/* break; */
/* } */
/* } */
/**/
/* if (!ts_node_is_null(child_containing_target)) */
/* { */
/* if (!ts_node_is_null(later_child)) */
/* { */
/* later_node = later_child; */
/* later_node_is_relevant = later_child_is_relevant; */
/* } */
/* node = child_containing_target; */
/* } */
/* else if (later_child_is_relevant) */
/* { */
/* return later_child; */
/* } */
/* else if (!ts_node_is_null(later_child)) */
/* { */
/* node = later_child; */
/* } */
/* else if (later_node_is_relevant) */
/* { */
/* return later_node; */
/* } */
/* else */
/* { */
/* node = later_node; */
/* } */
/* } */
/**/
/* return ts_node__null(); */
/* } */
/**/
/* static inline TSNode ts_node__first_child_for_byte(TSNode self, t_u32 goal, bool include_anonymous) */
/* { */
/* TSNode node = self; */
/* bool did_descend = true; */
/**/
/* while (did_descend) */
/* { */
/* did_descend = false; */
/**/
/* TSNode child; */
/* NodeChildIterator iterator = ts_node_iterate_children(&node); */
/* while (ts_node_child_iterator_next(&iterator, &child)) */
/* { */
/* if (ts_node_end_byte(child) > goal) */
/* { */
/* if (ts_node__is_relevant(child, include_anonymous)) */
/* { */
/* return child; */
/* } */
/* else if (ts_node_child_count(child) > 0) */
/* { */
/* did_descend = true; */
/* node = child; */
/* break; */
/* } */
/* } */
/* } */
/* } */
/**/
/* return ts_node__null(); */
/* } */
/**/
/* static inline TSNode ts_node__descendant_for_byte_range(TSNode self, t_u32 range_start, t_u32 range_end, bool include_anonymous) */
/* { */
/* TSNode node = self; */
/* TSNode last_visible_node = self; */
/**/
/* bool did_descend = true; */
/* while (did_descend) */
/* { */
/* did_descend = false; */
/**/
/* TSNode child; */
/* NodeChildIterator iterator = ts_node_iterate_children(&node); */
/* while (ts_node_child_iterator_next(&iterator, &child)) */
/* { */
/* t_u32 node_end = iterator.position.bytes; */
/**/
/* // The end of this node must extend far enough forward to touch */
/* // the end of the range and exceed the start of the range. */
/* if (node_end < range_end) */
/* continue; */
/* if (node_end <= range_start) */
/* continue; */
/**/
/* // The start of this node must extend far enough backward to */
/* // touch the start of the range. */
/* if (range_start < ts_node_start_byte(child)) */
/* break; */
/**/
/* node = child; */
/* if (ts_node__is_relevant(node, include_anonymous)) */
/* { */
/* last_visible_node = node; */
/* } */
/* did_descend = true; */
/* break; */
/* } */
/* } */
/**/
/* return last_visible_node; */
/* } */
/**/
/* static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint range_start, TSPoint range_end, bool include_anonymous) */
/* { */
/* TSNode node = self; */
/* TSNode last_visible_node = self; */
/**/
/* bool did_descend = true; */
/* while (did_descend) */
/* { */
/* did_descend = false; */
/**/
/* TSNode child; */
/* NodeChildIterator iterator = ts_node_iterate_children(&node); */
/* while (ts_node_child_iterator_next(&iterator, &child)) */
/* { */
/* TSPoint node_end = iterator.position.extent; */
/**/
/* // The end of this node must extend far enough forward to touch */
/* // the end of the range and exceed the start of the range. */
/* if (point_lt(node_end, range_end)) */
/* continue; */
/* if (point_lte(node_end, range_start)) */
/* continue; */
/**/
/* // The start of this node must extend far enough backward to */
/* // touch the start of the range. */
/* if (point_lt(range_start, ts_node_start_point(child))) */
/* break; */
/**/
/* node = child; */
/* if (ts_node__is_relevant(node, include_anonymous)) */
/* { */
/* last_visible_node = node; */
/* } */
/* did_descend = true; */
/* break; */
/* } */
/* } */
/**/
/* return last_visible_node; */
/* } */
// TSNode - public
@ -812,77 +798,6 @@ t_u32 ts_node_named_child_count(TSNode self)
}
}
TSNode ts_node_next_sibling(TSNode self)
{
return ts_node__next_sibling(self, true);
}
TSNode ts_node_next_named_sibling(TSNode self)
{
return ts_node__next_sibling(self, false);
}
TSNode ts_node_prev_sibling(TSNode self)
{
return ts_node__prev_sibling(self, true);
}
TSNode ts_node_prev_named_sibling(TSNode self)
{
return ts_node__prev_sibling(self, false);
}
TSNode ts_node_first_child_for_byte(TSNode self, t_u32 byte)
{
return ts_node__first_child_for_byte(self, byte, true);
}
TSNode ts_node_first_named_child_for_byte(TSNode self, t_u32 byte)
{
return ts_node__first_child_for_byte(self, byte, false);
}
TSNode ts_node_descendant_for_byte_range(TSNode self, t_u32 start, t_u32 end)
{
return ts_node__descendant_for_byte_range(self, start, end, true);
}
TSNode ts_node_named_descendant_for_byte_range(TSNode self, t_u32 start, t_u32 end)
{
return ts_node__descendant_for_byte_range(self, start, end, false);
}
TSNode ts_node_descendant_for_point_range(TSNode self, TSPoint start, TSPoint end)
{
return ts_node__descendant_for_point_range(self, start, end, true);
}
TSNode ts_node_named_descendant_for_point_range(TSNode self, TSPoint start, TSPoint end)
{
return ts_node__descendant_for_point_range(self, start, end, false);
}
void ts_node_edit(TSNode *self, const TSInputEdit *edit)
{
t_u32 start_byte = ts_node_start_byte(*self);
TSPoint start_point = ts_node_start_point(*self);
if (start_byte >= edit->old_end_byte)
{
start_byte = edit->new_end_byte + (start_byte - edit->old_end_byte);
start_point = point_add(edit->new_end_point, point_sub(start_point, edit->old_end_point));
}
else if (start_byte > edit->start_byte)
{
start_byte = edit->new_end_byte;
start_point = edit->new_end_point;
}
self->context[0] = start_byte;
self->context[1] = start_point.row;
self->context[2] = start_point.column;
}
TSSymbol ts_node_field_id_for_child(TSNode self, t_u32 child_index)
{
t_const_str name = ts_node_field_name_for_child(self, child_index);

96
parser/src/parser.c
View file

@ -20,14 +20,12 @@
#define LOG_TREE(...)
#define SYM_NAME(symbol) ts_language_symbol_name(self->language, symbol)
#define TREE_NAME(tree) SYM_NAME(ts_subtree_symbol(tree))
static const t_u32 MAX_VERSION_COUNT = 6;
static const t_u32 MAX_VERSION_COUNT_OVERFLOW = 4;
static const t_u32 MAX_SUMMARY_DEPTH = 16;
static const t_u32 MAX_SUMMARY_DEPTH = 1;
static const t_u32 MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE;
static const t_u32 OP_COUNT_PER_TIMEOUT_CHECK = 100;
typedef struct TokenCache
{
@ -340,34 +338,6 @@ static bool ts_parser__external_scanner_scan(TSParser *self, TSStateId external_
return self->language->external_scanner.scan(self->external_scanner_payload, &self->lexer.data, valid_external_tokens);
}
static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, Subtree tree, TableEntry *table_entry)
{
TSLexMode current_lex_mode = self->language->lex_modes[state];
TSSymbol leaf_symbol = ts_subtree_leaf_symbol(tree);
TSStateId leaf_state = ts_subtree_leaf_parse_state(tree);
TSLexMode leaf_lex_mode = self->language->lex_modes[leaf_state];
// At the end of a non-terminal extra node, the lexer normally returns
// NULL, which indicates that the parser should look for a reduce action
// at symbol `0`. Avoid reusing tokens in this situation to ensure that
// the same thing happens when incrementally reparsing.
if (current_lex_mode.lex_state == (t_u16)(-1))
return false;
// If the token was created in a state with the same set of lookaheads, it is reusable.
if (table_entry->action_count > 0 && memcmp(&leaf_lex_mode, &current_lex_mode, sizeof(TSLexMode)) == 0 &&
(leaf_symbol != self->language->keyword_capture_token || (!ts_subtree_is_keyword(tree) && ts_subtree_parse_state(tree) == state)))
return true;
// Empty tokens are not reusable in states with different lookaheads.
if (ts_subtree_size(tree).bytes == 0 && leaf_symbol != ts_builtin_sym_end)
return false;
// If the current state allows external tokens or other tokens that conflict with this
// token, this token is not reusable.
return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable;
}
static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state)
{
TSLexMode lex_mode = self->language->lex_modes[parse_state];
@ -532,39 +502,6 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId pa
return result;
}
static Subtree ts_parser__get_cached_token(TSParser *self, TSStateId state, size_t position, Subtree last_external_token,
TableEntry *table_entry)
{
TokenCache *cache = &self->token_cache;
if (cache->token.ptr && cache->byte_index == position &&
ts_subtree_external_scanner_state_eq(cache->last_external_token, last_external_token))
{
ts_language_table_entry(self->language, state, ts_subtree_symbol(cache->token), table_entry);
if (ts_parser__can_reuse_first_leaf(self, state, cache->token, table_entry))
{
ts_subtree_retain(cache->token);
return cache->token;
}
}
return NULL_SUBTREE;
}
static void ts_parser__set_cached_token(TSParser *self, t_u32 byte_index, Subtree last_external_token, Subtree token)
{
TokenCache *cache = &self->token_cache;
if (token.ptr)
ts_subtree_retain(token);
if (last_external_token.ptr)
ts_subtree_retain(last_external_token);
if (cache->token.ptr)
ts_subtree_release(/*&self->tree_pool, */ cache->token);
if (cache->last_external_token.ptr)
ts_subtree_release(/*&self->tree_pool, */ cache->last_external_token);
cache->token = token;
cache->byte_index = byte_index;
cache->last_external_token = last_external_token;
}
// Determine if a given tree should be replaced by an alternative tree.
//
// The decision is based on the trees' error costs (if any), their dynamic precedence,
@ -1249,20 +1186,11 @@ static bool ts_parser__advance(TSParser *self, StackVersion version, bool allow_
{
(void)(allow_node_reuse);
TSStateId state = ts_stack_state(self->stack, version);
t_u32 position = ts_stack_position(self->stack, version).bytes;
Subtree last_external_token = ts_stack_last_external_token(self->stack, version);
Subtree lookahead = NULL_SUBTREE;
TableEntry table_entry = {.action_count = 0};
// If no node from the previous syntax tree could be reused, then try to
// reuse the token previously returned by the lexer.
if (!lookahead.ptr)
{
lookahead = ts_parser__get_cached_token(self, state, position, last_external_token, &table_entry);
}
bool needs_lex = !lookahead.ptr;
bool needs_lex = true;
for (;;)
{
// Otherwise, re-run the lexer.
@ -1275,7 +1203,6 @@ static bool ts_parser__advance(TSParser *self, StackVersion version, bool allow_
if (lookahead.ptr)
{
ts_parser__set_cached_token(self, position, last_external_token, lookahead);
ts_language_table_entry(self->language, state, ts_subtree_symbol(lookahead), &table_entry);
}
@ -1287,22 +1214,6 @@ static bool ts_parser__advance(TSParser *self, StackVersion version, bool allow_
ts_language_table_entry(self->language, state, ts_builtin_sym_end, &table_entry);
}
}
// If a cancellation flag or a timeout was provided, then check every
// time a fixed number of parse actions has been processed.
if (++self->operation_count == OP_COUNT_PER_TIMEOUT_CHECK)
{
self->operation_count = 0;
}
if (self->operation_count == 0 && ((self->cancellation_flag && *self->cancellation_flag)))
{
if (lookahead.ptr)
{
ts_subtree_release(/*&self->tree_pool,*/ lookahead);
}
return false;
}
// Process each parse action for the current lookahead token in
// the current state. If there are multiple actions, then this is
// an ambiguous state. REDUCE actions always create a new stack
@ -1597,7 +1508,6 @@ TSParser *ts_parser_new(void)
self->operation_count = 0;
self->old_tree = NULL_SUBTREE;
self->included_range_difference_index = 0;
ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE);
return self;
}
@ -1618,7 +1528,6 @@ void ts_parser_delete(TSParser *self)
self->old_tree = NULL_SUBTREE;
}
ts_lexer_delete(&self->lexer);
ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE);
/* ts_subtree_pool_delete(&self->tree_pool); */
array_delete(&self->trailing_extras);
array_delete(&self->trailing_extras2);
@ -1658,7 +1567,6 @@ void ts_parser_reset(TSParser *self)
ts_lexer_reset(&self->lexer, length_zero());
ts_stack_clear(self->stack);
ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE);
if (self->finished_tree.ptr)
{
ts_subtree_release(/*&self->tree_pool,*/ self->finished_tree);

8
parser/src/scanner.c
View file

@ -118,11 +118,11 @@ static t_u32 serialize(Scanner *scanner, t_u8 *buffer)
buffer[size++] = (char)heredoc->started;
buffer[size++] = (char)heredoc->allows_indent;
memcpy(&buffer[size], &heredoc->delimiter.size, sizeof(t_u32));
mem_copy(&buffer[size], &heredoc->delimiter.size, sizeof(t_u32));
size += sizeof(t_u32);
if (heredoc->delimiter.size > 0)
{
memcpy(&buffer[size], heredoc->delimiter.contents, heredoc->delimiter.size);
mem_copy(&buffer[size], heredoc->delimiter.contents, heredoc->delimiter.size);
size += heredoc->delimiter.size;
}
}
@ -160,13 +160,13 @@ static void deserialize(Scanner *scanner, const t_u8 *buffer, t_u32 length)
heredoc->started = buffer[size++];
heredoc->allows_indent = buffer[size++];
memcpy(&heredoc->delimiter.size, &buffer[size], sizeof(t_u32));
mem_copy(&heredoc->delimiter.size, &buffer[size], sizeof(t_u32));
size += sizeof(t_u32);
array_reserve(&heredoc->delimiter, heredoc->delimiter.size);
if (heredoc->delimiter.size > 0)
{
memcpy(heredoc->delimiter.contents, &buffer[size], heredoc->delimiter.size);
mem_copy(heredoc->delimiter.contents, &buffer[size], heredoc->delimiter.size);
size += heredoc->delimiter.size;
}
}

38
parser/src/subtree.c
View file

@ -26,51 +26,31 @@ typedef struct
void ts_external_scanner_state_init(ExternalScannerState *self, const t_u8 *data, t_u32 length)
{
self->length = length;
if (length > sizeof(self->short_data))
{
self->long_data = mem_alloc(length);
memcpy(self->long_data, data, length);
}
else
{
memcpy(self->short_data, data, length);
}
mem_copy(self->long_data, data, length);
}
ExternalScannerState ts_external_scanner_state_copy(const ExternalScannerState *self)
{
ExternalScannerState result = *self;
if (self->length > sizeof(self->short_data))
{
result.long_data = mem_alloc(self->length);
memcpy(result.long_data, self->long_data, self->length);
}
mem_copy(result.long_data, self->long_data, self->length);
return result;
}
void ts_external_scanner_state_delete(ExternalScannerState *self)
{
if (self->length > sizeof(self->short_data))
{
mem_free(self->long_data);
}
}
const t_u8 *ts_external_scanner_state_data(const ExternalScannerState *self)
{
if (self->length > sizeof(self->short_data))
{
return (const t_u8 *)self->long_data;
}
else
{
return (const t_u8 *)self->short_data;
}
}
bool ts_external_scanner_state_eq(const ExternalScannerState *self, const t_u8 *buffer, t_u32 length)
{
return self->length == length && memcmp(ts_external_scanner_state_data(self), buffer, length) == 0;
return self->length == length && mem_compare(ts_external_scanner_state_data(self), buffer, length);
}
// SubtreeArray
@ -83,7 +63,7 @@ void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest)
if (self.capacity > 0)
{
dest->contents = mem_alloc_array(self.capacity, sizeof(Subtree));
memcpy(dest->contents, self.contents, self.size * sizeof(Subtree));
mem_copy(dest->contents, self.contents, self.size * sizeof(Subtree));
for (t_u32 i = 0; i < self.size; i++)
{
ts_subtree_retain(dest->contents[i]);
@ -159,12 +139,6 @@ void ts_subtree_array_reverse(SubtreeArray *self)
/**/
// Subtree
static inline bool ts_subtree_can_inline(Length padding, Length size, t_u32 lookahead_bytes)
{
return padding.bytes < TS_MAX_INLINE_TREE_LENGTH && padding.extent.row < 16 && padding.extent.column < TS_MAX_INLINE_TREE_LENGTH &&
size.extent.row == 0 && size.extent.column < TS_MAX_INLINE_TREE_LENGTH && lookahead_bytes < 16;
}
Subtree ts_subtree_new_leaf(TSSymbol symbol, Length padding, Length size, t_u32 lookahead_bytes, TSStateId parse_state,
bool has_external_tokens, bool depends_on_column, bool is_keyword, const TSLanguage *language)
// Subtree ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, t_u32 lookahead_bytes, TSStateId
@ -228,7 +202,7 @@ MutableSubtree ts_subtree_clone(Subtree self)
size_t alloc_size = ts_subtree_alloc_size(self.ptr->child_count);
Subtree *new_children = mem_alloc(alloc_size);
Subtree *old_children = ts_subtree_children(self);
memcpy(new_children, old_children, alloc_size);
mem_copy(new_children, old_children, alloc_size);
SubtreeHeapData *result = (SubtreeHeapData *)&new_children[self.ptr->child_count];
if (self.ptr->child_count > 0)
{
@ -994,7 +968,7 @@ void ts_subtree_print_dot_graph(Subtree self, const TSLanguage *language, FILE *
const ExternalScannerState *ts_subtree_external_scanner_state(Subtree self)
{
static const ExternalScannerState empty_state = {{.short_data = {0}}, .length = 0};
static const ExternalScannerState empty_state = {{NULL}, .length = 0};
if (self.ptr && self.ptr->has_external_tokens && self.ptr->child_count == 0)
{
return &self.ptr->external_scanner_state;

4
parser/src/tree.c
View file

@ -14,7 +14,7 @@ TSTree *ts_tree_new(Subtree root, const TSLanguage *language, const TSRange *inc
result->root = root;
result->language = ts_language_copy(language);
result->included_ranges = mem_alloc_array(included_range_count, sizeof(TSRange));
memcpy(result->included_ranges, included_ranges, included_range_count * sizeof(TSRange));
mem_copy(result->included_ranges, included_ranges, included_range_count * sizeof(TSRange));
result->included_range_count = included_range_count;
return result;
}
@ -103,6 +103,6 @@ TSRange *ts_tree_included_ranges(const TSTree *self, t_u32 *length)
{
*length = self->included_range_count;
TSRange *ranges = mem_alloc_array(self->included_range_count, sizeof(TSRange));
memcpy(ranges, self->included_ranges, self->included_range_count * sizeof(TSRange));
mem_copy(ranges, self->included_ranges, self->included_range_count * sizeof(TSRange));
return ranges;
}

56
symdif.py Executable file
View file

@ -0,0 +1,56 @@
#!/usr/bin/env python3
import subprocess
import os
archive_files = [
"libaq.a",
"libast.a",
"libexec.a",
"libgmr.a",
"libline.a",
"libme.a",
"libparser.a",
"libsh.a",
]
build_dir = "./build/"
binary_file = "./minishell"
dump_archive = subprocess.run(
["/usr/bin/env", "readelf", "--symbols", "--wide"]
+ [f"{build_dir}/{name}" for name in archive_files],
text=True,
capture_output=True,
)
dump_binary = subprocess.run(
["/usr/bin/env", "readelf", "--symbols", "--wide"] + [binary_file],
text=True,
capture_output=True,
)
if dump_archive.returncode != 0 or dump_binary.returncode != 0:
print("Error when getting the symbols out of the objects files or binary (or both)")
print(f"{dump_archive.stderr}")
print(f"{dump_binary.stderr}")
exit(1)
symbols_archive = set()
symbols_binary = set()
for line in dump_archive.stdout.split('\n'):
words = line.split()
if (len (words) >= 8 and words[3] == 'FUNC'):
symbols_archive.add(words[7])
for line in dump_binary.stdout.split('\n'):
words = line.split()
if (len (words) >= 8 and words[3] == 'FUNC'):
symbols_binary.add(words[7])
diff = list(symbols_archive - symbols_binary)
diff.sort()
for sym in diff:
print(f"{sym}")