raphael/minishell
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Removing more fat from the parser lib

Browse source
This commit is contained in:
Maix0 2024-08-19 14:19:12 +02:00
parent a6246a52a0
commit c1209452cd
11 changed files with 360 additions and 566 deletions
Download patch file Download diff file Expand all files Collapse all files

9
Makefile
View file

@ -6,7 +6,7 @@
# By: rparodi <rparodi@student.42.fr> +#+ +:+ +#+ # # By: rparodi <rparodi@student.42.fr> +#+ +:+ +#+ #
# +#+#+#+#+#+ +#+ # # +#+#+#+#+#+ +#+ #
# Created: 2023/11/12 11:05:05 by rparodi #+# #+# # # Created: 2023/11/12 11:05:05 by rparodi #+# #+# #
# Updated: 2024/08/18 22:01:52 by maiboyer ### ########.fr # # Updated: 2024/08/19 14:00:31 by maiboyer ### ########.fr #
# # # #
# **************************************************************************** # # **************************************************************************** #
@ -45,15 +45,16 @@ endif
# TODO: ADD THIS WHEN FINISHING THIS: # TODO: ADD THIS WHEN FINISHING THIS:
# CFLAGS_ADDITIONAL += -DNVALGRIND # CFLAGS_ADDITIONAL += -DNVALGRIND
CFLAGS_ADDITIONAL += -O0 -Wno-cpp -Wno-type-limits
# TODO: REMOVE THIS WHEN FINISHING THIS: # TODO: REMOVE THIS WHEN FINISHING THIS:
CFLAGS_ADDITIONAL += -O0
CFLAGS_ADDITIONAL += -Wno-cpp -Wno-type-limits
CFLAGS_ADDITIONAL += -gcolumn-info -g3 CFLAGS_ADDITIONAL += -gcolumn-info -g3
CFLAGS_ADDITIONAL += '-DERROR=((void)printf("ERROR HERE: " __FILE__ ":%d in %s\n", __LINE__, __func__), 1)' CFLAGS_ADDITIONAL += '-DERROR=((void)printf("ERROR HERE: " __FILE__ ":%d in %s\n", __LINE__, __func__), 1)'
CFLAGS_ADDITIONAL += '-Dinline=' -Wno-unused-function
#CFLAGS_ADDITIONAL += -O2 #CFLAGS_ADDITIONAL += -O2
# CFLAGS_ADDITIONAL += -fuse-ld=gold -Wl,--print-symbol-counts -Wl,/tmp/symbols_count.log # CFLAGS_ADDITIONAL += -fuse-ld=gold -Wl,--print-symbol-counts -Wl,/tmp/symbols_count.log
#CFLAGS_ADDITIONAL += -fuse-ld=lld -Wl,--verbose -ffunction-sections -fdata-sections -Wl,--gc-sections -Wl,--print-gc-sections -Wl,-Map=output.map CFLAGS_ADDITIONAL += -fuse-ld=lld -ffunction-sections -fdata-sections -Wl,--gc-sections -Wl,-O0
export CFLAGS_ADDITIONAL export CFLAGS_ADDITIONAL
export CC export CC

4
parser/include/parser/array.h
View file

@ -181,7 +181,7 @@ static inline void _array__assign(Array *self, const Array *other, size_t elemen
{ {
_array__reserve(self, element_size, other->size); _array__reserve(self, element_size, other->size);
self->size = other->size; self->size = other->size;
memcpy(self->contents, other->contents, self->size * element_size); mem_copy(self->contents, other->contents, self->size * element_size);
} }
/// This is not what you're looking for, see `array_swap`. /// This is not what you're looking for, see `array_swap`.
@ -226,7 +226,7 @@ static inline void _array__splice(Array *self, size_t element_size, t_u32 index,
{ {
if (elements) if (elements)
{ {
memcpy((contents + index * element_size), elements, new_count * element_size); mem_copy((contents + index * element_size), elements, new_count * element_size);
} }
else else
{ {

78
parser/include/parser/subtree.h
View file

@ -19,7 +19,6 @@ struct ExternalScannerState
{ {
union { union {
char *long_data; char *long_data;
char short_data[24];
}; };
t_u32 length; t_u32 length;
}; };
@ -36,37 +35,6 @@ struct ExternalScannerState
// separately on the heap. // separately on the heap.
typedef struct ExternalScannerState ExternalScannerState; typedef struct ExternalScannerState ExternalScannerState;
// A compact representation of a subtree.
//
// This representation is used for small leaf nodes that are not
// errors, and were not created by an external scanner.
//
// The idea behind the layout of this struct is that the `is_inline`
// bit will fall exactly into the same location as the least significant
// bit of the pointer in `Subtree` or `MutableSubtree`, respectively.
// Because of alignment, for any valid pointer this will be 0, giving
// us the opportunity to make use of this bit to signify whether to use
// the pointer or the inline struct.
typedef struct SubtreeInlineData SubtreeInlineData_;
struct SubtreeInlineData
{
bool is_inline : 1;
bool visible : 1;
bool named : 1;
bool extra : 1;
bool has_changes : 1;
bool is_missing : 1;
bool is_keyword : 1;
t_u8 symbol;
t_u16 parse_state;
t_u8 padding_columns;
t_u8 padding_rows : 4;
t_u8 lookahead_bytes : 4;
t_u8 padding_bytes;
t_u8 size_bytes;
};
// A heap-allocated representation of a subtree. // A heap-allocated representation of a subtree.
// //
// This representation is used for parent nodes, external tokens, // This representation is used for parent nodes, external tokens,
@ -74,14 +42,14 @@ struct SubtreeInlineData
// the inline representation. // the inline representation.
typedef struct SubtreeHeapData typedef struct SubtreeHeapData
{ {
volatile t_u32 ref_count; t_u32 ref_count;
Length padding; Length padding;
Length size; Length size;
t_u32 lookahead_bytes; t_u32 lookahead_bytes;
t_u32 error_cost; t_u32 error_cost;
t_u32 child_count; t_u32 child_count;
TSSymbol symbol; TSSymbol symbol;
TSStateId parse_state; TSStateId parse_state;
bool visible : 1; bool visible : 1;
bool named : 1; bool named : 1;
@ -161,12 +129,12 @@ Subtree ts_subtree_new_missing_leaf(/*SubtreePool *,*/ TSSymbol, Length, t_u
MutableSubtree ts_subtree_make_mut(/*SubtreePool *,*/ Subtree); MutableSubtree ts_subtree_make_mut(/*SubtreePool *,*/ Subtree);
void ts_subtree_retain(Subtree); void ts_subtree_retain(Subtree);
void ts_subtree_release(/*SubtreePool *,*/ Subtree); void ts_subtree_release(/*SubtreePool *,*/ Subtree);
int ts_subtree_compare(Subtree, Subtree/*, SubtreePool **/); int ts_subtree_compare(Subtree, Subtree /*, SubtreePool **/);
void ts_subtree_set_symbol(MutableSubtree *, TSSymbol, const TSLanguage *); void ts_subtree_set_symbol(MutableSubtree *, TSSymbol, const TSLanguage *);
void ts_subtree_summarize(MutableSubtree, const Subtree *, t_u32, const TSLanguage *); void ts_subtree_summarize(MutableSubtree, const Subtree *, t_u32, const TSLanguage *);
void ts_subtree_summarize_children(MutableSubtree, const TSLanguage *); void ts_subtree_summarize_children(MutableSubtree, const TSLanguage *);
void ts_subtree_balance(Subtree, /*SubtreePool *,*/ const TSLanguage *); void ts_subtree_balance(Subtree, /*SubtreePool *,*/ const TSLanguage *);
Subtree ts_subtree_edit(Subtree, const TSInputEdit *edit/*, SubtreePool **/); Subtree ts_subtree_edit(Subtree, const TSInputEdit *edit /*, SubtreePool **/);
char *ts_subtree_string(Subtree, TSSymbol, bool, const TSLanguage *, bool include_all); char *ts_subtree_string(Subtree, TSSymbol, bool, const TSLanguage *, bool include_all);
void ts_subtree_print_dot_graph(Subtree, const TSLanguage *, FILE *); void ts_subtree_print_dot_graph(Subtree, const TSLanguage *, FILE *);
Subtree ts_subtree_last_external_token(Subtree); Subtree ts_subtree_last_external_token(Subtree);
@ -270,24 +238,11 @@ static inline t_u32 ts_subtree_repeat_depth(Subtree self)
return (self.ptr->repeat_depth); return (self.ptr->repeat_depth);
} }
static inline t_u32 ts_subtree_is_repetition(Subtree self)
{
return (!self.ptr->named && !self.ptr->visible && self.ptr->child_count != 0);
}
static inline t_u32 ts_subtree_visible_descendant_count(Subtree self) static inline t_u32 ts_subtree_visible_descendant_count(Subtree self)
{ {
return ((self.ptr->child_count == 0) ? 0 : self.ptr->visible_descendant_count); return ((self.ptr->child_count == 0) ? 0 : self.ptr->visible_descendant_count);
} }
static inline t_u32 ts_subtree_visible_child_count(Subtree self)
{
if (ts_subtree_child_count(self) > 0)
return (self.ptr->visible_child_count);
else
return 0;
}
static inline t_u32 ts_subtree_error_cost(Subtree self) static inline t_u32 ts_subtree_error_cost(Subtree self)
{ {
if (ts_subtree_missing(self)) if (ts_subtree_missing(self))
@ -301,14 +256,6 @@ static inline t_i32 ts_subtree_dynamic_precedence(Subtree self)
return ((self.ptr->child_count == 0) ? 0 : self.ptr->dynamic_precedence); return ((self.ptr->child_count == 0) ? 0 : self.ptr->dynamic_precedence);
} }
static inline t_u16 ts_subtree_production_id(Subtree self)
{
if (ts_subtree_child_count(self) > 0)
return (self.ptr->production_id);
else
return (0);
}
static inline bool ts_subtree_fragile_left(Subtree self) static inline bool ts_subtree_fragile_left(Subtree self)
{ {
return (self.ptr->fragile_left); return (self.ptr->fragile_left);
@ -334,11 +281,6 @@ static inline bool ts_subtree_depends_on_column(Subtree self)
return (self.ptr->depends_on_column); return (self.ptr->depends_on_column);
} }
static inline bool ts_subtree_is_fragile(Subtree self)
{
return ((self.ptr->fragile_left || self.ptr->fragile_right));
}
static inline bool ts_subtree_is_error(Subtree self) static inline bool ts_subtree_is_error(Subtree self)
{ {
return (ts_subtree_symbol(self) == ts_builtin_sym_error); return (ts_subtree_symbol(self) == ts_builtin_sym_error);

8
parser/src/lexer.c
View file

@ -1,8 +1,8 @@
#include "parser/lexer.h" #include "parser/lexer.h"
#include "parser/length.h"
#include "parser/input.h"
#include "me/mem/mem.h" #include "me/mem/mem.h"
#include "me/types.h" #include "me/types.h"
#include "parser/input.h"
#include "parser/length.h"
#include <string.h> #include <string.h>
#define LOG(...) #define LOG(...)
@ -427,7 +427,7 @@ bool ts_lexer_set_included_ranges(Lexer *self, const TSRange *ranges, t_u32 coun
size_t size = count * sizeof(TSRange); size_t size = count * sizeof(TSRange);
self->included_ranges = mem_realloc(self->included_ranges, size); self->included_ranges = mem_realloc(self->included_ranges, size);
memcpy(self->included_ranges, ranges, size); mem_copy(self->included_ranges, ranges, size);
self->included_range_count = count; self->included_range_count = count;
ts_lexer_goto(self, self->current_position); ts_lexer_goto(self, self->current_position);
return true; return true;
@ -438,5 +438,3 @@ TSRange *ts_lexer_included_ranges(const Lexer *self, t_u32 *count)
*count = self->included_range_count; *count = self->included_range_count;
return self->included_ranges; return self->included_ranges;
} }
#undef LOG

611
parser/src/node.c
View file

@ -186,284 +186,270 @@ static inline TSNode ts_node__child(TSNode self, t_u32 child_index, bool include
return ts_node__null(); return ts_node__null();
} }
static bool ts_subtree_has_trailing_empty_descendant(Subtree self, Subtree other)
{
for (t_u32 i = ts_subtree_child_count(self) - 1; i + 1 > 0; i--)
{
Subtree child = ts_subtree_children(self)[i];
if (ts_subtree_total_bytes(child) > 0)
break;
if (child.ptr == other.ptr || ts_subtree_has_trailing_empty_descendant(child, other))
{
return true;
}
}
return false;
}
static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) /* static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) */
{ /* { */
Subtree self_subtree = ts_node__subtree(self); /* Subtree self_subtree = ts_node__subtree(self); */
bool self_is_empty = ts_subtree_total_bytes(self_subtree) == 0; /* bool self_is_empty = ts_subtree_total_bytes(self_subtree) == 0; */
t_u32 target_end_byte = ts_node_end_byte(self); /* t_u32 target_end_byte = ts_node_end_byte(self); */
/**/
TSNode node = ts_node_parent(self); /* TSNode node = ts_node_parent(self); */
TSNode earlier_node = ts_node__null(); /* TSNode earlier_node = ts_node__null(); */
bool earlier_node_is_relevant = false; /* bool earlier_node_is_relevant = false; */
/**/
while (!ts_node_is_null(node)) /* while (!ts_node_is_null(node)) */
{ /* { */
TSNode earlier_child = ts_node__null(); /* TSNode earlier_child = ts_node__null(); */
bool earlier_child_is_relevant = false; /* bool earlier_child_is_relevant = false; */
bool found_child_containing_target = false; /* bool found_child_containing_target = false; */
/**/
TSNode child; /* TSNode child; */
NodeChildIterator iterator = ts_node_iterate_children(&node); /* NodeChildIterator iterator = ts_node_iterate_children(&node); */
while (ts_node_child_iterator_next(&iterator, &child)) /* while (ts_node_child_iterator_next(&iterator, &child)) */
{ /* { */
if (child.id == self.id) /* if (child.id == self.id) */
break; /* break; */
if (iterator.position.bytes > target_end_byte) /* if (iterator.position.bytes > target_end_byte) */
{ /* { */
found_child_containing_target = true; /* found_child_containing_target = true; */
break; /* break; */
} /* } */
/**/
if (iterator.position.bytes == target_end_byte && /* if (iterator.position.bytes == target_end_byte && */
(!self_is_empty || ts_subtree_has_trailing_empty_descendant(ts_node__subtree(child), self_subtree))) /* (!self_is_empty || ts_subtree_has_trailing_empty_descendant(ts_node__subtree(child), self_subtree))) */
{ /* { */
found_child_containing_target = true; /* found_child_containing_target = true; */
break; /* break; */
} /* } */
/**/
if (ts_node__is_relevant(child, include_anonymous)) /* if (ts_node__is_relevant(child, include_anonymous)) */
{ /* { */
earlier_child = child; /* earlier_child = child; */
earlier_child_is_relevant = true; /* earlier_child_is_relevant = true; */
} /* } */
else if (ts_node__relevant_child_count(child, include_anonymous) > 0) /* else if (ts_node__relevant_child_count(child, include_anonymous) > 0) */
{ /* { */
earlier_child = child; /* earlier_child = child; */
earlier_child_is_relevant = false; /* earlier_child_is_relevant = false; */
} /* } */
} /* } */
/**/
if (found_child_containing_target) /* if (found_child_containing_target) */
{ /* { */
if (!ts_node_is_null(earlier_child)) /* if (!ts_node_is_null(earlier_child)) */
{ /* { */
earlier_node = earlier_child; /* earlier_node = earlier_child; */
earlier_node_is_relevant = earlier_child_is_relevant; /* earlier_node_is_relevant = earlier_child_is_relevant; */
} /* } */
node = child; /* node = child; */
} /* } */
else if (earlier_child_is_relevant) /* else if (earlier_child_is_relevant) */
{ /* { */
return earlier_child; /* return earlier_child; */
} /* } */
else if (!ts_node_is_null(earlier_child)) /* else if (!ts_node_is_null(earlier_child)) */
{ /* { */
node = earlier_child; /* node = earlier_child; */
} /* } */
else if (earlier_node_is_relevant) /* else if (earlier_node_is_relevant) */
{ /* { */
return earlier_node; /* return earlier_node; */
} /* } */
else /* else */
{ /* { */
node = earlier_node; /* node = earlier_node; */
earlier_node = ts_node__null(); /* earlier_node = ts_node__null(); */
earlier_node_is_relevant = false; /* earlier_node_is_relevant = false; */
} /* } */
} /* } */
/**/
return ts_node__null(); /* return ts_node__null(); */
} /* } */
/**/
static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) /* static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) */
{ /* { */
t_u32 target_end_byte = ts_node_end_byte(self); /* t_u32 target_end_byte = ts_node_end_byte(self); */
/**/
TSNode node = ts_node_parent(self); /* TSNode node = ts_node_parent(self); */
TSNode later_node = ts_node__null(); /* TSNode later_node = ts_node__null(); */
bool later_node_is_relevant = false; /* bool later_node_is_relevant = false; */
/**/
while (!ts_node_is_null(node)) /* while (!ts_node_is_null(node)) */
{ /* { */
TSNode later_child = ts_node__null(); /* TSNode later_child = ts_node__null(); */
bool later_child_is_relevant = false; /* bool later_child_is_relevant = false; */
TSNode child_containing_target = ts_node__null(); /* TSNode child_containing_target = ts_node__null(); */
/**/
TSNode child; /* TSNode child; */
NodeChildIterator iterator = ts_node_iterate_children(&node); /* NodeChildIterator iterator = ts_node_iterate_children(&node); */
while (ts_node_child_iterator_next(&iterator, &child)) /* while (ts_node_child_iterator_next(&iterator, &child)) */
{ /* { */
if (iterator.position.bytes < target_end_byte) /* if (iterator.position.bytes < target_end_byte) */
continue; /* continue; */
if (ts_node_start_byte(child) <= ts_node_start_byte(self)) /* if (ts_node_start_byte(child) <= ts_node_start_byte(self)) */
{ /* { */
if (ts_node__subtree(child).ptr != ts_node__subtree(self).ptr) /* if (ts_node__subtree(child).ptr != ts_node__subtree(self).ptr) */
{ /* { */
child_containing_target = child; /* child_containing_target = child; */
} /* } */
} /* } */
else if (ts_node__is_relevant(child, include_anonymous)) /* else if (ts_node__is_relevant(child, include_anonymous)) */
{ /* { */
later_child = child; /* later_child = child; */
later_child_is_relevant = true; /* later_child_is_relevant = true; */
break; /* break; */
} /* } */
else if (ts_node__relevant_child_count(child, include_anonymous) > 0) /* else if (ts_node__relevant_child_count(child, include_anonymous) > 0) */
{ /* { */
later_child = child; /* later_child = child; */
later_child_is_relevant = false; /* later_child_is_relevant = false; */
break; /* break; */
} /* } */
} /* } */
/**/
if (!ts_node_is_null(child_containing_target)) /* if (!ts_node_is_null(child_containing_target)) */
{ /* { */
if (!ts_node_is_null(later_child)) /* if (!ts_node_is_null(later_child)) */
{ /* { */
later_node = later_child; /* later_node = later_child; */
later_node_is_relevant = later_child_is_relevant; /* later_node_is_relevant = later_child_is_relevant; */
} /* } */
node = child_containing_target; /* node = child_containing_target; */
} /* } */
else if (later_child_is_relevant) /* else if (later_child_is_relevant) */
{ /* { */
return later_child; /* return later_child; */
} /* } */
else if (!ts_node_is_null(later_child)) /* else if (!ts_node_is_null(later_child)) */
{ /* { */
node = later_child; /* node = later_child; */
} /* } */
else if (later_node_is_relevant) /* else if (later_node_is_relevant) */
{ /* { */
return later_node; /* return later_node; */
} /* } */
else /* else */
{ /* { */
node = later_node; /* node = later_node; */
} /* } */
} /* } */
/**/
return ts_node__null(); /* return ts_node__null(); */
} /* } */
/**/
static inline TSNode ts_node__first_child_for_byte(TSNode self, t_u32 goal, bool include_anonymous) /* static inline TSNode ts_node__first_child_for_byte(TSNode self, t_u32 goal, bool include_anonymous) */
{ /* { */
TSNode node = self; /* TSNode node = self; */
bool did_descend = true; /* bool did_descend = true; */
/**/
while (did_descend) /* while (did_descend) */
{ /* { */
did_descend = false; /* did_descend = false; */
/**/
TSNode child; /* TSNode child; */
NodeChildIterator iterator = ts_node_iterate_children(&node); /* NodeChildIterator iterator = ts_node_iterate_children(&node); */
while (ts_node_child_iterator_next(&iterator, &child)) /* while (ts_node_child_iterator_next(&iterator, &child)) */
{ /* { */
if (ts_node_end_byte(child) > goal) /* if (ts_node_end_byte(child) > goal) */
{ /* { */
if (ts_node__is_relevant(child, include_anonymous)) /* if (ts_node__is_relevant(child, include_anonymous)) */
{ /* { */
return child; /* return child; */
} /* } */
else if (ts_node_child_count(child) > 0) /* else if (ts_node_child_count(child) > 0) */
{ /* { */
did_descend = true; /* did_descend = true; */
node = child; /* node = child; */
break; /* break; */
} /* } */
} /* } */
} /* } */
} /* } */
/**/
return ts_node__null(); /* return ts_node__null(); */
} /* } */
/**/
static inline TSNode ts_node__descendant_for_byte_range(TSNode self, t_u32 range_start, t_u32 range_end, bool include_anonymous) /* static inline TSNode ts_node__descendant_for_byte_range(TSNode self, t_u32 range_start, t_u32 range_end, bool include_anonymous) */
{ /* { */
TSNode node = self; /* TSNode node = self; */
TSNode last_visible_node = self; /* TSNode last_visible_node = self; */
/**/
bool did_descend = true; /* bool did_descend = true; */
while (did_descend) /* while (did_descend) */
{ /* { */
did_descend = false; /* did_descend = false; */
/**/
TSNode child; /* TSNode child; */
NodeChildIterator iterator = ts_node_iterate_children(&node); /* NodeChildIterator iterator = ts_node_iterate_children(&node); */
while (ts_node_child_iterator_next(&iterator, &child)) /* while (ts_node_child_iterator_next(&iterator, &child)) */
{ /* { */
t_u32 node_end = iterator.position.bytes; /* t_u32 node_end = iterator.position.bytes; */
/**/
// The end of this node must extend far enough forward to touch /* // The end of this node must extend far enough forward to touch */
// the end of the range and exceed the start of the range. /* // the end of the range and exceed the start of the range. */
if (node_end < range_end) /* if (node_end < range_end) */
continue; /* continue; */
if (node_end <= range_start) /* if (node_end <= range_start) */
continue; /* continue; */
/**/
// The start of this node must extend far enough backward to /* // The start of this node must extend far enough backward to */
// touch the start of the range. /* // touch the start of the range. */
if (range_start < ts_node_start_byte(child)) /* if (range_start < ts_node_start_byte(child)) */
break; /* break; */
/**/
node = child; /* node = child; */
if (ts_node__is_relevant(node, include_anonymous)) /* if (ts_node__is_relevant(node, include_anonymous)) */
{ /* { */
last_visible_node = node; /* last_visible_node = node; */
} /* } */
did_descend = true; /* did_descend = true; */
break; /* break; */
} /* } */
} /* } */
/**/
return last_visible_node; /* return last_visible_node; */
} /* } */
/**/
static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint range_start, TSPoint range_end, bool include_anonymous) /* static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint range_start, TSPoint range_end, bool include_anonymous) */
{ /* { */
TSNode node = self; /* TSNode node = self; */
TSNode last_visible_node = self; /* TSNode last_visible_node = self; */
/**/
bool did_descend = true; /* bool did_descend = true; */
while (did_descend) /* while (did_descend) */
{ /* { */
did_descend = false; /* did_descend = false; */
/**/
TSNode child; /* TSNode child; */
NodeChildIterator iterator = ts_node_iterate_children(&node); /* NodeChildIterator iterator = ts_node_iterate_children(&node); */
while (ts_node_child_iterator_next(&iterator, &child)) /* while (ts_node_child_iterator_next(&iterator, &child)) */
{ /* { */
TSPoint node_end = iterator.position.extent; /* TSPoint node_end = iterator.position.extent; */
/**/
// The end of this node must extend far enough forward to touch /* // The end of this node must extend far enough forward to touch */
// the end of the range and exceed the start of the range. /* // the end of the range and exceed the start of the range. */
if (point_lt(node_end, range_end)) /* if (point_lt(node_end, range_end)) */
continue; /* continue; */
if (point_lte(node_end, range_start)) /* if (point_lte(node_end, range_start)) */
continue; /* continue; */
/**/
// The start of this node must extend far enough backward to /* // The start of this node must extend far enough backward to */
// touch the start of the range. /* // touch the start of the range. */
if (point_lt(range_start, ts_node_start_point(child))) /* if (point_lt(range_start, ts_node_start_point(child))) */
break; /* break; */
/**/
node = child; /* node = child; */
if (ts_node__is_relevant(node, include_anonymous)) /* if (ts_node__is_relevant(node, include_anonymous)) */
{ /* { */
last_visible_node = node; /* last_visible_node = node; */
} /* } */
did_descend = true; /* did_descend = true; */
break; /* break; */
} /* } */
} /* } */
/**/
return last_visible_node; /* return last_visible_node; */
} /* } */
// TSNode - public // TSNode - public
@ -812,77 +798,6 @@ t_u32 ts_node_named_child_count(TSNode self)
} }
} }
TSNode ts_node_next_sibling(TSNode self)
{
return ts_node__next_sibling(self, true);
}
TSNode ts_node_next_named_sibling(TSNode self)
{
return ts_node__next_sibling(self, false);
}
TSNode ts_node_prev_sibling(TSNode self)
{
return ts_node__prev_sibling(self, true);
}
TSNode ts_node_prev_named_sibling(TSNode self)
{
return ts_node__prev_sibling(self, false);
}
TSNode ts_node_first_child_for_byte(TSNode self, t_u32 byte)
{
return ts_node__first_child_for_byte(self, byte, true);
}
TSNode ts_node_first_named_child_for_byte(TSNode self, t_u32 byte)
{
return ts_node__first_child_for_byte(self, byte, false);
}
TSNode ts_node_descendant_for_byte_range(TSNode self, t_u32 start, t_u32 end)
{
return ts_node__descendant_for_byte_range(self, start, end, true);
}
TSNode ts_node_named_descendant_for_byte_range(TSNode self, t_u32 start, t_u32 end)
{
return ts_node__descendant_for_byte_range(self, start, end, false);
}
TSNode ts_node_descendant_for_point_range(TSNode self, TSPoint start, TSPoint end)
{
return ts_node__descendant_for_point_range(self, start, end, true);
}
TSNode ts_node_named_descendant_for_point_range(TSNode self, TSPoint start, TSPoint end)
{
return ts_node__descendant_for_point_range(self, start, end, false);
}
void ts_node_edit(TSNode *self, const TSInputEdit *edit)
{
t_u32 start_byte = ts_node_start_byte(*self);
TSPoint start_point = ts_node_start_point(*self);
if (start_byte >= edit->old_end_byte)
{
start_byte = edit->new_end_byte + (start_byte - edit->old_end_byte);
start_point = point_add(edit->new_end_point, point_sub(start_point, edit->old_end_point));
}
else if (start_byte > edit->start_byte)
{
start_byte = edit->new_end_byte;
start_point = edit->new_end_point;
}
self->context[0] = start_byte;
self->context[1] = start_point.row;
self->context[2] = start_point.column;
}
TSSymbol ts_node_field_id_for_child(TSNode self, t_u32 child_index) TSSymbol ts_node_field_id_for_child(TSNode self, t_u32 child_index)
{ {
t_const_str name = ts_node_field_name_for_child(self, child_index); t_const_str name = ts_node_field_name_for_child(self, child_index);

100
parser/src/parser.c
View file

@ -20,14 +20,12 @@
#define LOG_TREE(...) #define LOG_TREE(...)
#define SYM_NAME(symbol) ts_language_symbol_name(self->language, symbol) #define SYM_NAME(symbol) ts_language_symbol_name(self->language, symbol)
#define TREE_NAME(tree) SYM_NAME(ts_subtree_symbol(tree)) #define TREE_NAME(tree) SYM_NAME(ts_subtree_symbol(tree))
static const t_u32 MAX_VERSION_COUNT = 6; static const t_u32 MAX_VERSION_COUNT = 6;
static const t_u32 MAX_VERSION_COUNT_OVERFLOW = 4; static const t_u32 MAX_VERSION_COUNT_OVERFLOW = 4;
static const t_u32 MAX_SUMMARY_DEPTH = 16; static const t_u32 MAX_SUMMARY_DEPTH = 1;
static const t_u32 MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE; static const t_u32 MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE;
static const t_u32 OP_COUNT_PER_TIMEOUT_CHECK = 100;
typedef struct TokenCache typedef struct TokenCache
{ {
@ -38,8 +36,8 @@ typedef struct TokenCache
struct TSParser struct TSParser
{ {
Lexer lexer; Lexer lexer;
Stack *stack; Stack *stack;
/* SubtreePool tree_pool; */ /* SubtreePool tree_pool; */
const TSLanguage *language; const TSLanguage *language;
ReduceActionSet reduce_actions; ReduceActionSet reduce_actions;
@ -340,34 +338,6 @@ static bool ts_parser__external_scanner_scan(TSParser *self, TSStateId external_
return self->language->external_scanner.scan(self->external_scanner_payload, &self->lexer.data, valid_external_tokens); return self->language->external_scanner.scan(self->external_scanner_payload, &self->lexer.data, valid_external_tokens);
} }
static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, Subtree tree, TableEntry *table_entry)
{
TSLexMode current_lex_mode = self->language->lex_modes[state];
TSSymbol leaf_symbol = ts_subtree_leaf_symbol(tree);
TSStateId leaf_state = ts_subtree_leaf_parse_state(tree);
TSLexMode leaf_lex_mode = self->language->lex_modes[leaf_state];
// At the end of a non-terminal extra node, the lexer normally returns
// NULL, which indicates that the parser should look for a reduce action
// at symbol `0`. Avoid reusing tokens in this situation to ensure that
// the same thing happens when incrementally reparsing.
if (current_lex_mode.lex_state == (t_u16)(-1))
return false;
// If the token was created in a state with the same set of lookaheads, it is reusable.
if (table_entry->action_count > 0 && memcmp(&leaf_lex_mode, &current_lex_mode, sizeof(TSLexMode)) == 0 &&
(leaf_symbol != self->language->keyword_capture_token || (!ts_subtree_is_keyword(tree) && ts_subtree_parse_state(tree) == state)))
return true;
// Empty tokens are not reusable in states with different lookaheads.
if (ts_subtree_size(tree).bytes == 0 && leaf_symbol != ts_builtin_sym_end)
return false;
// If the current state allows external tokens or other tokens that conflict with this
// token, this token is not reusable.
return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable;
}
static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state) static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state)
{ {
TSLexMode lex_mode = self->language->lex_modes[parse_state]; TSLexMode lex_mode = self->language->lex_modes[parse_state];
@ -532,39 +502,6 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId pa
return result; return result;
} }
static Subtree ts_parser__get_cached_token(TSParser *self, TSStateId state, size_t position, Subtree last_external_token,
TableEntry *table_entry)
{
TokenCache *cache = &self->token_cache;
if (cache->token.ptr && cache->byte_index == position &&
ts_subtree_external_scanner_state_eq(cache->last_external_token, last_external_token))
{
ts_language_table_entry(self->language, state, ts_subtree_symbol(cache->token), table_entry);
if (ts_parser__can_reuse_first_leaf(self, state, cache->token, table_entry))
{
ts_subtree_retain(cache->token);
return cache->token;
}
}
return NULL_SUBTREE;
}
static void ts_parser__set_cached_token(TSParser *self, t_u32 byte_index, Subtree last_external_token, Subtree token)
{
TokenCache *cache = &self->token_cache;
if (token.ptr)
ts_subtree_retain(token);
if (last_external_token.ptr)
ts_subtree_retain(last_external_token);
if (cache->token.ptr)
ts_subtree_release(/*&self->tree_pool, */ cache->token);
if (cache->last_external_token.ptr)
ts_subtree_release(/*&self->tree_pool, */ cache->last_external_token);
cache->token = token;
cache->byte_index = byte_index;
cache->last_external_token = last_external_token;
}
// Determine if a given tree should be replaced by an alternative tree. // Determine if a given tree should be replaced by an alternative tree.
// //
// The decision is based on the trees' error costs (if any), their dynamic precedence, // The decision is based on the trees' error costs (if any), their dynamic precedence,
@ -1249,20 +1186,11 @@ static bool ts_parser__advance(TSParser *self, StackVersion version, bool allow_
{ {
(void)(allow_node_reuse); (void)(allow_node_reuse);
TSStateId state = ts_stack_state(self->stack, version); TSStateId state = ts_stack_state(self->stack, version);
t_u32 position = ts_stack_position(self->stack, version).bytes;
Subtree last_external_token = ts_stack_last_external_token(self->stack, version);
Subtree lookahead = NULL_SUBTREE; Subtree lookahead = NULL_SUBTREE;
TableEntry table_entry = {.action_count = 0}; TableEntry table_entry = {.action_count = 0};
// If no node from the previous syntax tree could be reused, then try to bool needs_lex = true;
// reuse the token previously returned by the lexer.
if (!lookahead.ptr)
{
lookahead = ts_parser__get_cached_token(self, state, position, last_external_token, &table_entry);
}
bool needs_lex = !lookahead.ptr;
for (;;) for (;;)
{ {
// Otherwise, re-run the lexer. // Otherwise, re-run the lexer.
@ -1275,7 +1203,6 @@ static bool ts_parser__advance(TSParser *self, StackVersion version, bool allow_
if (lookahead.ptr) if (lookahead.ptr)
{ {
ts_parser__set_cached_token(self, position, last_external_token, lookahead);
ts_language_table_entry(self->language, state, ts_subtree_symbol(lookahead), &table_entry); ts_language_table_entry(self->language, state, ts_subtree_symbol(lookahead), &table_entry);
} }
@ -1287,22 +1214,6 @@ static bool ts_parser__advance(TSParser *self, StackVersion version, bool allow_
ts_language_table_entry(self->language, state, ts_builtin_sym_end, &table_entry); ts_language_table_entry(self->language, state, ts_builtin_sym_end, &table_entry);
} }
} }
// If a cancellation flag or a timeout was provided, then check every
// time a fixed number of parse actions has been processed.
if (++self->operation_count == OP_COUNT_PER_TIMEOUT_CHECK)
{
self->operation_count = 0;
}
if (self->operation_count == 0 && ((self->cancellation_flag && *self->cancellation_flag)))
{
if (lookahead.ptr)
{
ts_subtree_release(/*&self->tree_pool,*/ lookahead);
}
return false;
}
// Process each parse action for the current lookahead token in // Process each parse action for the current lookahead token in
// the current state. If there are multiple actions, then this is // the current state. If there are multiple actions, then this is
// an ambiguous state. REDUCE actions always create a new stack // an ambiguous state. REDUCE actions always create a new stack
@ -1597,7 +1508,6 @@ TSParser *ts_parser_new(void)
self->operation_count = 0; self->operation_count = 0;
self->old_tree = NULL_SUBTREE; self->old_tree = NULL_SUBTREE;
self->included_range_difference_index = 0; self->included_range_difference_index = 0;
ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE);
return self; return self;
} }
@ -1618,7 +1528,6 @@ void ts_parser_delete(TSParser *self)
self->old_tree = NULL_SUBTREE; self->old_tree = NULL_SUBTREE;
} }
ts_lexer_delete(&self->lexer); ts_lexer_delete(&self->lexer);
ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE);
/* ts_subtree_pool_delete(&self->tree_pool); */ /* ts_subtree_pool_delete(&self->tree_pool); */
array_delete(&self->trailing_extras); array_delete(&self->trailing_extras);
array_delete(&self->trailing_extras2); array_delete(&self->trailing_extras2);
@ -1658,7 +1567,6 @@ void ts_parser_reset(TSParser *self)
ts_lexer_reset(&self->lexer, length_zero()); ts_lexer_reset(&self->lexer, length_zero());
ts_stack_clear(self->stack); ts_stack_clear(self->stack);
ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE);
if (self->finished_tree.ptr) if (self->finished_tree.ptr)
{ {
ts_subtree_release(/*&self->tree_pool,*/ self->finished_tree); ts_subtree_release(/*&self->tree_pool,*/ self->finished_tree);

8
parser/src/scanner.c
View file

@ -118,11 +118,11 @@ static t_u32 serialize(Scanner *scanner, t_u8 *buffer)
buffer[size++] = (char)heredoc->started; buffer[size++] = (char)heredoc->started;
buffer[size++] = (char)heredoc->allows_indent; buffer[size++] = (char)heredoc->allows_indent;
memcpy(&buffer[size], &heredoc->delimiter.size, sizeof(t_u32)); mem_copy(&buffer[size], &heredoc->delimiter.size, sizeof(t_u32));
size += sizeof(t_u32); size += sizeof(t_u32);
if (heredoc->delimiter.size > 0) if (heredoc->delimiter.size > 0)
{ {
memcpy(&buffer[size], heredoc->delimiter.contents, heredoc->delimiter.size); mem_copy(&buffer[size], heredoc->delimiter.contents, heredoc->delimiter.size);
size += heredoc->delimiter.size; size += heredoc->delimiter.size;
} }
} }
@ -160,13 +160,13 @@ static void deserialize(Scanner *scanner, const t_u8 *buffer, t_u32 length)
heredoc->started = buffer[size++]; heredoc->started = buffer[size++];
heredoc->allows_indent = buffer[size++]; heredoc->allows_indent = buffer[size++];
memcpy(&heredoc->delimiter.size, &buffer[size], sizeof(t_u32)); mem_copy(&heredoc->delimiter.size, &buffer[size], sizeof(t_u32));
size += sizeof(t_u32); size += sizeof(t_u32);
array_reserve(&heredoc->delimiter, heredoc->delimiter.size); array_reserve(&heredoc->delimiter, heredoc->delimiter.size);
if (heredoc->delimiter.size > 0) if (heredoc->delimiter.size > 0)
{ {
memcpy(heredoc->delimiter.contents, &buffer[size], heredoc->delimiter.size); mem_copy(heredoc->delimiter.contents, &buffer[size], heredoc->delimiter.size);
size += heredoc->delimiter.size; size += heredoc->delimiter.size;
} }
} }

46
parser/src/subtree.c
View file

@ -26,51 +26,31 @@ typedef struct
void ts_external_scanner_state_init(ExternalScannerState *self, const t_u8 *data, t_u32 length) void ts_external_scanner_state_init(ExternalScannerState *self, const t_u8 *data, t_u32 length)
{ {
self->length = length; self->length = length;
if (length > sizeof(self->short_data)) self->long_data = mem_alloc(length);
{ mem_copy(self->long_data, data, length);
self->long_data = mem_alloc(length);
memcpy(self->long_data, data, length);
}
else
{
memcpy(self->short_data, data, length);
}
} }
ExternalScannerState ts_external_scanner_state_copy(const ExternalScannerState *self) ExternalScannerState ts_external_scanner_state_copy(const ExternalScannerState *self)
{ {
ExternalScannerState result = *self; ExternalScannerState result = *self;
if (self->length > sizeof(self->short_data)) result.long_data = mem_alloc(self->length);
{ mem_copy(result.long_data, self->long_data, self->length);
result.long_data = mem_alloc(self->length);
memcpy(result.long_data, self->long_data, self->length);
}
return result; return result;
} }
void ts_external_scanner_state_delete(ExternalScannerState *self) void ts_external_scanner_state_delete(ExternalScannerState *self)
{ {
if (self->length > sizeof(self->short_data)) mem_free(self->long_data);
{
mem_free(self->long_data);
}
} }
const t_u8 *ts_external_scanner_state_data(const ExternalScannerState *self) const t_u8 *ts_external_scanner_state_data(const ExternalScannerState *self)
{ {
if (self->length > sizeof(self->short_data)) return (const t_u8 *)self->long_data;
{
return (const t_u8 *)self->long_data;
}
else
{
return (const t_u8 *)self->short_data;
}
} }
bool ts_external_scanner_state_eq(const ExternalScannerState *self, const t_u8 *buffer, t_u32 length) bool ts_external_scanner_state_eq(const ExternalScannerState *self, const t_u8 *buffer, t_u32 length)
{ {
return self->length == length && memcmp(ts_external_scanner_state_data(self), buffer, length) == 0; return self->length == length && mem_compare(ts_external_scanner_state_data(self), buffer, length);
} }
// SubtreeArray // SubtreeArray
@ -83,7 +63,7 @@ void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest)
if (self.capacity > 0) if (self.capacity > 0)
{ {
dest->contents = mem_alloc_array(self.capacity, sizeof(Subtree)); dest->contents = mem_alloc_array(self.capacity, sizeof(Subtree));
memcpy(dest->contents, self.contents, self.size * sizeof(Subtree)); mem_copy(dest->contents, self.contents, self.size * sizeof(Subtree));
for (t_u32 i = 0; i < self.size; i++) for (t_u32 i = 0; i < self.size; i++)
{ {
ts_subtree_retain(dest->contents[i]); ts_subtree_retain(dest->contents[i]);
@ -159,12 +139,6 @@ void ts_subtree_array_reverse(SubtreeArray *self)
/**/ /**/
// Subtree // Subtree
static inline bool ts_subtree_can_inline(Length padding, Length size, t_u32 lookahead_bytes)
{
return padding.bytes < TS_MAX_INLINE_TREE_LENGTH && padding.extent.row < 16 && padding.extent.column < TS_MAX_INLINE_TREE_LENGTH &&
size.extent.row == 0 && size.extent.column < TS_MAX_INLINE_TREE_LENGTH && lookahead_bytes < 16;
}
Subtree ts_subtree_new_leaf(TSSymbol symbol, Length padding, Length size, t_u32 lookahead_bytes, TSStateId parse_state, Subtree ts_subtree_new_leaf(TSSymbol symbol, Length padding, Length size, t_u32 lookahead_bytes, TSStateId parse_state,
bool has_external_tokens, bool depends_on_column, bool is_keyword, const TSLanguage *language) bool has_external_tokens, bool depends_on_column, bool is_keyword, const TSLanguage *language)
// Subtree ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, t_u32 lookahead_bytes, TSStateId // Subtree ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, t_u32 lookahead_bytes, TSStateId
@ -228,7 +202,7 @@ MutableSubtree ts_subtree_clone(Subtree self)
size_t alloc_size = ts_subtree_alloc_size(self.ptr->child_count); size_t alloc_size = ts_subtree_alloc_size(self.ptr->child_count);
Subtree *new_children = mem_alloc(alloc_size); Subtree *new_children = mem_alloc(alloc_size);
Subtree *old_children = ts_subtree_children(self); Subtree *old_children = ts_subtree_children(self);
memcpy(new_children, old_children, alloc_size); mem_copy(new_children, old_children, alloc_size);
SubtreeHeapData *result = (SubtreeHeapData *)&new_children[self.ptr->child_count]; SubtreeHeapData *result = (SubtreeHeapData *)&new_children[self.ptr->child_count];
if (self.ptr->child_count > 0) if (self.ptr->child_count > 0)
{ {
@ -994,7 +968,7 @@ void ts_subtree_print_dot_graph(Subtree self, const TSLanguage *language, FILE *
const ExternalScannerState *ts_subtree_external_scanner_state(Subtree self) const ExternalScannerState *ts_subtree_external_scanner_state(Subtree self)
{ {
static const ExternalScannerState empty_state = {{.short_data = {0}}, .length = 0}; static const ExternalScannerState empty_state = {{NULL}, .length = 0};
if (self.ptr && self.ptr->has_external_tokens && self.ptr->child_count == 0) if (self.ptr && self.ptr->has_external_tokens && self.ptr->child_count == 0)
{ {
return &self.ptr->external_scanner_state; return &self.ptr->external_scanner_state;

4
parser/src/tree.c
View file

@ -14,7 +14,7 @@ TSTree *ts_tree_new(Subtree root, const TSLanguage *language, const TSRange *inc
result->root = root; result->root = root;
result->language = ts_language_copy(language); result->language = ts_language_copy(language);
result->included_ranges = mem_alloc_array(included_range_count, sizeof(TSRange)); result->included_ranges = mem_alloc_array(included_range_count, sizeof(TSRange));
memcpy(result->included_ranges, included_ranges, included_range_count * sizeof(TSRange)); mem_copy(result->included_ranges, included_ranges, included_range_count * sizeof(TSRange));
result->included_range_count = included_range_count; result->included_range_count = included_range_count;
return result; return result;
} }
@ -103,6 +103,6 @@ TSRange *ts_tree_included_ranges(const TSTree *self, t_u32 *length)
{ {
*length = self->included_range_count; *length = self->included_range_count;
TSRange *ranges = mem_alloc_array(self->included_range_count, sizeof(TSRange)); TSRange *ranges = mem_alloc_array(self->included_range_count, sizeof(TSRange));
memcpy(ranges, self->included_ranges, self->included_range_count * sizeof(TSRange)); mem_copy(ranges, self->included_ranges, self->included_range_count * sizeof(TSRange));
return ranges; return ranges;
} }

56
symdif.py Executable file
View file

@ -0,0 +1,56 @@
#!/usr/bin/env python3
import subprocess
import os
archive_files = [
"libaq.a",
"libast.a",
"libexec.a",
"libgmr.a",
"libline.a",
"libme.a",
"libparser.a",
"libsh.a",
]
build_dir = "./build/"
binary_file = "./minishell"
dump_archive = subprocess.run(
["/usr/bin/env", "readelf", "--symbols", "--wide"]
+ [f"{build_dir}/{name}" for name in archive_files],
text=True,
capture_output=True,
)
dump_binary = subprocess.run(
["/usr/bin/env", "readelf", "--symbols", "--wide"] + [binary_file],
text=True,
capture_output=True,
)
if dump_archive.returncode != 0 or dump_binary.returncode != 0:
print("Error when getting the symbols out of the objects files or binary (or both)")
print(f"{dump_archive.stderr}")
print(f"{dump_binary.stderr}")
exit(1)
symbols_archive = set()
symbols_binary = set()
for line in dump_archive.stdout.split('\n'):
words = line.split()
if (len (words) >= 8 and words[3] == 'FUNC'):
symbols_archive.add(words[7])
for line in dump_binary.stdout.split('\n'):
words = line.split()
if (len (words) >= 8 and words[3] == 'FUNC'):
symbols_binary.add(words[7])
diff = list(symbols_archive - symbols_binary)
diff.sort()
for sym in diff:
print(f"{sym}")