raphael/minishell
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

working parser tm

Browse source
This commit is contained in:
Maieul BOYER 2024-06-30 18:32:35 +02:00
parent 8d192a0abc
commit 2d88b6af25
No known key found for this signature in database
51 changed files with 33 additions and 31734 deletions
Download patch file Download diff file Expand all files Collapse all files

4
parser/Grammar.mk
View file

@ -6,7 +6,7 @@
# By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ # # By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ #
# +#+#+#+#+#+ +#+ # # +#+#+#+#+#+ +#+ #
# Created: 2023/11/03 13:20:01 by maiboyer #+# #+# # # Created: 2023/11/03 13:20:01 by maiboyer #+# #+# #
# Updated: 2024/06/30 17:56:39 by maiboyer ### ########.fr # # Updated: 2024/06/30 18:15:08 by maiboyer ### ########.fr #
# # # #
# **************************************************************************** # # **************************************************************************** #
@ -27,7 +27,7 @@ CFLAGS = -Wall -Wextra -Werror -MMD -I./includes -I../includes -I../output/inc
include ./Filelist.mk include ./Filelist.mk
SRC_FILES += #SRC_FILES = parser
#scanner #scanner
#parser #parser

364
parser/gen/funcs.h
View file

@ -1,364 +0,0 @@
CaptureListPool capture_list_pool_new(void);
CaptureList *capture_list_pool_get_mut(CaptureListPool *self, uint16_t id);
CaptureQuantifiers capture_quantifiers_new(void);
Iterator iterator_new(TreeCursor *cursor, const Subtree *tree, const TSLanguage *language);
IteratorComparison iterator_compare(const Iterator *old_iter, const Iterator *new_iter);
Length iterator_end_position(Iterator *self);
Length iterator_start_position(Iterator *self);
QueryStep query_step__new(TSSymbol symbol, uint16_t depth, bool is_immediate);
StackNode *stack_node_new(StackNode *previous_node, Subtree subtree, bool is_pending, TSStateId state, StackNodeArray *pool);
StackSliceArray stack__iter(Stack *self, StackVersion version, StackCallback callback, void *payload, int goal_subtree_count);
Stream stream_new(const char *string, uint32_t length);
SymbolTable symbol_table_new(void);
TSQuantifier capture_quantifier_for_id(const CaptureQuantifiers *self, uint16_t id);
TSQuantifier quantifier_add(TSQuantifier left, TSQuantifier right);
TSQuantifier quantifier_join(TSQuantifier left, TSQuantifier right);
TSQuantifier quantifier_mul(TSQuantifier left, TSQuantifier right);
bool capture_list_pool_is_empty(const CaptureListPool *self);
bool iterator_descend(Iterator *self, uint32_t goal_position);
bool iterator_done(Iterator *self);
bool iterator_tree_is_visible(const Iterator *self);
bool stack__subtree_is_equivalent(Subtree left, Subtree right);
bool stream_advance(Stream *self);
bool stream_is_ident_start(Stream *self);
const CaptureList *capture_list_pool_get(const CaptureListPool *self, uint16_t id);
const Length LENGTH_MAX;
const Length LENGTH_UNDEFINED;
const TSQueryError PARENT_DONE;
const TSRange DEFAULT_RANGE;
const TSSymbol WILDCARD_SYMBOL;
const char *symbol_table_name_for_id(const SymbolTable *self, uint16_t id, uint32_t *length);
const char *const ROOT_FIELD;
const int32_t BYTE_ORDER_MARK;
const int32_t TS_DECODE_ERROR;
const uint16_t NONE;
const uint16_t PATTERN_DONE_MARKER;
const unsigned MAX_COST_DIFFERENCE;
const unsigned MAX_SUMMARY_DEPTH;
const unsigned MAX_VERSION_COUNT;
const unsigned MAX_VERSION_COUNT_OVERFLOW;
const unsigned OP_COUNT_PER_TIMEOUT_CHECK;
int symbol_table_id_for_name(const SymbolTable *self, const char *name, uint32_t length);
uint16_t capture_list_pool_acquire(CaptureListPool *self);
uint16_t symbol_table_insert_name(SymbolTable *self, const char *name, uint32_t length);
uint32_t stack__subtree_node_count(Subtree subtree);
uint32_t stream_offset(Stream *self);
unsigned analysis_state__recursion_depth(const AnalysisState *self);
void capture_list_pool_delete(CaptureListPool *self);
void capture_list_pool_release(CaptureListPool *self, uint16_t id);
void capture_list_pool_reset(CaptureListPool *self);
void capture_quantifiers_add_all(CaptureQuantifiers *self, CaptureQuantifiers *quantifiers);
void capture_quantifiers_add_for_id(CaptureQuantifiers *self, uint16_t id, TSQuantifier quantifier);
void capture_quantifiers_clear(CaptureQuantifiers *self);
void capture_quantifiers_delete(CaptureQuantifiers *self);
void capture_quantifiers_join_all(CaptureQuantifiers *self, CaptureQuantifiers *quantifiers);
void capture_quantifiers_mul(CaptureQuantifiers *self, TSQuantifier quantifier);
void capture_quantifiers_replace(CaptureQuantifiers *self, CaptureQuantifiers *quantifiers);
void iterator_advance(Iterator *self);
void iterator_ascend(Iterator *self);
void iterator_get_visible_state(const Iterator *self, Subtree *tree, TSSymbol *alias_symbol, uint32_t *start_byte);
void query_step__add_capture(QueryStep *self, uint16_t capture_id);
void query_step__remove_capture(QueryStep *self, uint16_t capture_id);
void stack_head_delete(StackHead *self, StackNodeArray *pool, SubtreePool *subtree_pool);
void stack_node_add_link(StackNode *self, StackLink link, SubtreePool *subtree_pool);
void stack_node_release(StackNode *self, StackNodeArray *pool, SubtreePool *subtree_pool);
void stack_node_retain(StackNode *self);
void stream_reset(Stream *self, const char *input);
void stream_scan_identifier(Stream *stream);
void stream_skip_whitespace(Stream *self);
void symbol_table_delete(SymbolTable *self);
void *ts_calloc_default(size_t count, size_t size);
uint32_t ts_decode_ascii(const uint8_t *string, uint32_t length, int32_t *code_point);
int _ts_dup(int file_descriptor);
ExternalScannerState ts_external_scanner_state_copy(const ExternalScannerState *self);
const char *ts_external_scanner_state_data(const ExternalScannerState *self);
void ts_external_scanner_state_delete(ExternalScannerState *self);
bool ts_external_scanner_state_eq(const ExternalScannerState *self, const char *buffer, unsigned length);
void ts_external_scanner_state_init(ExternalScannerState *self, const char *data, unsigned length);
const TSLanguage *ts_language_copy(const TSLanguage *self);
void ts_language_delete(const TSLanguage *self);
uint32_t ts_language_field_count(const TSLanguage *self);
TSFieldId ts_language_field_id_for_name(const TSLanguage *self, const char *name, uint32_t name_length);
const char *ts_language_field_name_for_id(const TSLanguage *self, TSFieldId id);
TSStateId ts_language_next_state(const TSLanguage *self, TSStateId state, TSSymbol symbol);
TSSymbol ts_language_public_symbol(const TSLanguage *self, TSSymbol symbol);
uint32_t ts_language_state_count(const TSLanguage *self);
uint32_t ts_language_symbol_count(const TSLanguage *self);
TSSymbol ts_language_symbol_for_name(const TSLanguage *self, const char *string, uint32_t length, bool is_named);
TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *self, TSSymbol symbol);
const char *ts_language_symbol_name(const TSLanguage *self, TSSymbol symbol);
TSSymbolType ts_language_symbol_type(const TSLanguage *self, TSSymbol symbol);
void ts_language_table_entry(const TSLanguage *self, TSStateId state, TSSymbol symbol, TableEntry *result);
uint32_t ts_language_version(const TSLanguage *self);
void ts_lexer__advance(TSLexer *_self, bool skip);
void ts_lexer__clear_chunk(Lexer *self);
void ts_lexer__do_advance(Lexer *self, bool skip);
bool ts_lexer__eof(const TSLexer *_self);
void ts_lexer__get_chunk(Lexer *self);
uint32_t ts_lexer__get_column(TSLexer *_self);
void ts_lexer__get_lookahead(Lexer *self);
bool ts_lexer__is_at_included_range_start(const TSLexer *_self);
void ts_lexer__mark_end(TSLexer *_self);
void ts_lexer_advance_to_end(Lexer *self);
void ts_lexer_delete(Lexer *self);
void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte);
void ts_lexer_goto(Lexer *self, Length position);
TSRange *ts_lexer_included_ranges(const Lexer *self, uint32_t *count);
void ts_lexer_init(Lexer *self);
void ts_lexer_mark_end(Lexer *self);
void ts_lexer_reset(Lexer *self, Length position);
bool ts_lexer_set_included_ranges(Lexer *self, const TSRange *ranges, uint32_t count);
void ts_lexer_set_input(Lexer *self, TSInput input);
void ts_lexer_start(Lexer *self);
TSSymbol ts_lookahead_iterator_current_symbol(const TSLookaheadIterator *self);
const char *ts_lookahead_iterator_current_symbol_name(const TSLookaheadIterator *self);
void ts_lookahead_iterator_delete(TSLookaheadIterator *self);
const TSLanguage *ts_lookahead_iterator_language(const TSLookaheadIterator *self);
TSLookaheadIterator *ts_lookahead_iterator_new(const TSLanguage *self, TSStateId state);
bool ts_lookahead_iterator_next(TSLookaheadIterator *self);
bool ts_lookahead_iterator_reset(TSLookaheadIterator *self, const TSLanguage *language, TSStateId state);
bool ts_lookahead_iterator_reset_state(TSLookaheadIterator *self, TSStateId state);
void *ts_malloc_default(size_t size);
TSNode ts_node_child(TSNode self, uint32_t child_index);
TSNode ts_node_child_by_field_id(TSNode self, TSFieldId field_id);
TSNode ts_node_child_by_field_name(TSNode self, const char *name, uint32_t name_length);
TSNode ts_node_child_containing_descendant(TSNode self, TSNode subnode);
uint32_t ts_node_child_count(TSNode self);
uint32_t ts_node_descendant_count(TSNode self);
TSNode ts_node_descendant_for_byte_range(TSNode self, uint32_t start, uint32_t end);
TSNode ts_node_descendant_for_point_range(TSNode self, TSPoint start, TSPoint end);
void ts_node_edit(TSNode *self, const TSInputEdit *edit);
uint32_t ts_node_end_byte(TSNode self);
TSPoint ts_node_end_point(TSNode self);
bool ts_node_eq(TSNode self, TSNode other);
const char *ts_node_field_name_for_child(TSNode self, uint32_t child_index);
TSNode ts_node_first_child_for_byte(TSNode self, uint32_t byte);
TSNode ts_node_first_named_child_for_byte(TSNode self, uint32_t byte);
TSSymbol ts_node_grammar_symbol(TSNode self);
const char *ts_node_grammar_type(TSNode self);
bool ts_node_has_changes(TSNode self);
bool ts_node_has_error(TSNode self);
bool ts_node_is_error(TSNode self);
bool ts_node_is_extra(TSNode self);
bool ts_node_is_missing(TSNode self);
bool ts_node_is_named(TSNode self);
bool ts_node_is_null(TSNode self);
const TSLanguage *ts_node_language(TSNode self);
TSNode ts_node_named_child(TSNode self, uint32_t child_index);
uint32_t ts_node_named_child_count(TSNode self);
TSNode ts_node_named_descendant_for_byte_range(TSNode self, uint32_t start, uint32_t end);
TSNode ts_node_named_descendant_for_point_range(TSNode self, TSPoint start, TSPoint end);
TSNode ts_node_new(const TSTree *tree, const Subtree *subtree, Length position, TSSymbol alias);
TSNode ts_node_next_named_sibling(TSNode self);
TSStateId ts_node_next_parse_state(TSNode self);
TSNode ts_node_next_sibling(TSNode self);
TSNode ts_node_parent(TSNode self);
TSStateId ts_node_parse_state(TSNode self);
TSNode ts_node_prev_named_sibling(TSNode self);
TSNode ts_node_prev_sibling(TSNode self);
uint32_t ts_node_start_byte(TSNode self);
TSPoint ts_node_start_point(TSNode self);
char *ts_node_string(TSNode self);
TSSymbol ts_node_symbol(TSNode self);
const char *ts_node_type(TSNode self);
void ts_parser__accept(TSParser *self, StackVersion version, Subtree lookahead);
bool ts_parser__advance(TSParser *self, StackVersion version, bool allow_node_reuse);
bool ts_parser__better_version_exists(TSParser *self, StackVersion version, bool is_in_error, unsigned cost);
void ts_parser__breakdown_lookahead(TSParser *self, Subtree *lookahead, TSStateId state, ReusableNode *reusable_node);
bool ts_parser__breakdown_top_of_stack(TSParser *self, StackVersion version);
bool ts_parser__call_keyword_lex_fn(TSParser *self, TSLexMode lex_mode);
bool ts_parser__call_main_lex_fn(TSParser *self, TSLexMode lex_mode);
bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, Subtree tree, TableEntry *table_entry);
ErrorComparison ts_parser__compare_versions(TSParser *self, ErrorStatus a, ErrorStatus b);
unsigned ts_parser__condense_stack(TSParser *self);
bool ts_parser__do_all_potential_reductions(TSParser *self, StackVersion starting_version, TSSymbol lookahead_symbol);
void ts_parser__external_scanner_create(TSParser *self);
void ts_parser__external_scanner_deserialize(TSParser *self, Subtree external_token);
void ts_parser__external_scanner_destroy(TSParser *self);
bool ts_parser__external_scanner_scan(TSParser *self, TSStateId external_lex_state);
unsigned ts_parser__external_scanner_serialize(TSParser *self);
Subtree ts_parser__get_cached_token(TSParser *self, TSStateId state, size_t position, Subtree last_external_token, TableEntry *table_entry);
void ts_parser__handle_error(TSParser *self, StackVersion version, Subtree lookahead);
bool ts_parser__has_included_range_difference(const TSParser *self, uint32_t start_position, uint32_t end_position);
Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId parse_state);
void ts_parser__log(TSParser *self);
void ts_parser__recover(TSParser *self, StackVersion version, Subtree lookahead);
bool ts_parser__recover_to_state(TSParser *self, StackVersion version, unsigned depth, TSStateId goal_state);
StackVersion ts_parser__reduce(TSParser *self, StackVersion version, TSSymbol symbol, uint32_t count, int dynamic_precedence, uint16_t production_id, bool is_fragile, bool end_of_non_terminal_extra);
Subtree ts_parser__reuse_node(TSParser *self, StackVersion version, TSStateId *state, uint32_t position, Subtree last_external_token, TableEntry *table_entry);
bool ts_parser__select_children(TSParser *self, Subtree left, const SubtreeArray *children);
bool ts_parser__select_tree(TSParser *self, Subtree left, Subtree right);
void ts_parser__set_cached_token(TSParser *self, uint32_t byte_index, Subtree last_external_token, Subtree token);
void ts_parser__shift(TSParser *self, StackVersion version, TSStateId state, Subtree lookahead, bool extra);
ErrorStatus ts_parser__version_status(TSParser *self, StackVersion version);
const size_t *ts_parser_cancellation_flag(const TSParser *self);
void ts_parser_delete(TSParser *self);
bool ts_parser_has_outstanding_parse(TSParser *self);
const TSRange *ts_parser_included_ranges(const TSParser *self, uint32_t *count);
const TSLanguage *ts_parser_language(const TSParser *self);
TSLogger ts_parser_logger(const TSParser *self);
TSParser *ts_parser_new(void);
TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input);
TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, const char *string, uint32_t length);
TSTree *ts_parser_parse_string_encoding(TSParser *self, const TSTree *old_tree, const char *string, uint32_t length, TSInputEncoding encoding);
void ts_parser_print_dot_graphs(TSParser *self, int fd);
void ts_parser_reset(TSParser *self);
void ts_parser_set_cancellation_flag(TSParser *self, const size_t *flag);
bool ts_parser_set_included_ranges(TSParser *self, const TSRange *ranges, uint32_t count);
bool ts_parser_set_language(TSParser *self, const TSLanguage *language);
void ts_parser_set_logger(TSParser *self, TSLogger logger);
void ts_parser_set_timeout_micros(TSParser *self, uint64_t timeout_micros);
void ts_parser_set_wasm_store(TSParser *self, TSWasmStore *store);
TSWasmStore *ts_parser_take_wasm_store(TSParser *self);
uint64_t ts_parser_timeout_micros(const TSParser *self);
void ts_query__add_negated_fields(TSQuery *self, uint16_t step_index, TSFieldId *field_ids, uint16_t field_count);
bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset);
TSQueryError ts_query__parse_pattern(TSQuery *self, Stream *stream, uint32_t depth, bool is_immediate, CaptureQuantifiers *capture_quantifiers);
TSQueryError ts_query__parse_predicate(TSQuery *self, Stream *stream);
TSQueryError ts_query__parse_string_literal(TSQuery *self, Stream *stream);
void ts_query__perform_analysis(TSQuery *self, const AnalysisSubgraphArray *subgraphs, QueryAnalysis *analysis);
bool ts_query__step_is_fallible(const TSQuery *self, uint16_t step_index);
uint32_t ts_query_capture_count(const TSQuery *self);
const char *ts_query_capture_name_for_id(const TSQuery *self, uint32_t index, uint32_t *length);
TSQuantifier ts_query_capture_quantifier_for_id(const TSQuery *self, uint32_t pattern_index, uint32_t capture_index);
void ts_query_cursor__add_state(TSQueryCursor *self, const PatternEntry *pattern);
void ts_query_cursor__capture(TSQueryCursor *self, QueryState *state, QueryStep *step, TSNode node);
void ts_query_cursor__compare_captures(TSQueryCursor *self, QueryState *left_state, QueryState *right_state, bool *left_contains_right, bool *right_contains_left);
int ts_query_cursor__compare_nodes(TSNode left, TSNode right);
QueryState *ts_query_cursor__copy_state(TSQueryCursor *self, QueryState **state_ref);
bool ts_query_cursor__first_in_progress_capture(TSQueryCursor *self, uint32_t *state_index, uint32_t *byte_offset, uint32_t *pattern_index, bool *root_pattern_guaranteed);
CaptureList *ts_query_cursor__prepare_to_capture(TSQueryCursor *self, QueryState *state, unsigned state_index_to_preserve);
void ts_query_cursor_delete(TSQueryCursor *self);
bool ts_query_cursor_did_exceed_match_limit(const TSQueryCursor *self);
void ts_query_cursor_exec(TSQueryCursor *self, const TSQuery *query, TSNode node);
uint32_t ts_query_cursor_match_limit(const TSQueryCursor *self);
TSQueryCursor *ts_query_cursor_new(void);
bool ts_query_cursor_next_capture(TSQueryCursor *self, TSQueryMatch *match, uint32_t *capture_index);
bool ts_query_cursor_next_match(TSQueryCursor *self, TSQueryMatch *match);
void ts_query_cursor_remove_match(TSQueryCursor *self, uint32_t match_id);
void ts_query_cursor_set_byte_range(TSQueryCursor *self, uint32_t start_byte, uint32_t end_byte);
void ts_query_cursor_set_match_limit(TSQueryCursor *self, uint32_t limit);
void ts_query_cursor_set_max_start_depth(TSQueryCursor *self, uint32_t max_start_depth);
void ts_query_cursor_set_point_range(TSQueryCursor *self, TSPoint start_point, TSPoint end_point);
void ts_query_delete(TSQuery *self);
void ts_query_disable_capture(TSQuery *self, const char *name, uint32_t length);
void ts_query_disable_pattern(TSQuery *self, uint32_t pattern_index);
bool ts_query_is_pattern_guaranteed_at_step(const TSQuery *self, uint32_t byte_offset);
bool ts_query_is_pattern_non_local(const TSQuery *self, uint32_t pattern_index);
bool ts_query_is_pattern_rooted(const TSQuery *self, uint32_t pattern_index);
TSQuery *ts_query_new(const TSLanguage *language, const char *source, uint32_t source_len, uint32_t *error_offset, TSQueryError *error_type);
uint32_t ts_query_pattern_count(const TSQuery *self);
const TSQueryPredicateStep *ts_query_predicates_for_pattern(const TSQuery *self, uint32_t pattern_index, uint32_t *step_count);
uint32_t ts_query_start_byte_for_pattern(const TSQuery *self, uint32_t pattern_index);
uint32_t ts_query_string_count(const TSQuery *self);
const char *ts_query_string_value_for_id(const TSQuery *self, uint32_t index, uint32_t *length);
void ts_range_array_add(TSRangeArray *self, Length start, Length end);
void ts_range_array_get_changed_ranges(const TSRange *old_ranges, unsigned old_range_count, const TSRange *new_ranges, unsigned new_range_count, TSRangeArray *differences);
bool ts_range_array_intersects(const TSRangeArray *self, unsigned start_index, uint32_t start_byte, uint32_t end_byte);
void *ts_realloc_default(void *buffer, size_t size);
void ts_set_allocator(void *(*new_malloc)(void), void *(*new_calloc)(void), void *(*new_realloc)(void), void (*new_free)(void));
void ts_stack__add_slice(Stack *self, StackVersion original_version, StackNode *node, SubtreeArray *subtrees);
StackVersion ts_stack__add_version(Stack *self, StackVersion original_version, StackNode *node);
bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version2);
void ts_stack_clear(Stack *self);
StackVersion ts_stack_copy_version(Stack *self, StackVersion version);
void ts_stack_delete(Stack *self);
int ts_stack_dynamic_precedence(Stack *self, StackVersion version);
unsigned ts_stack_error_cost(const Stack *self, StackVersion version);
StackSummary *ts_stack_get_summary(Stack *self, StackVersion version);
void ts_stack_halt(Stack *self, StackVersion version);
bool ts_stack_has_advanced_since_error(const Stack *self, StackVersion version);
bool ts_stack_is_active(const Stack *self, StackVersion version);
bool ts_stack_is_halted(const Stack *self, StackVersion version);
bool ts_stack_is_paused(const Stack *self, StackVersion version);
Subtree ts_stack_last_external_token(const Stack *self, StackVersion version);
bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2);
Stack *ts_stack_new(SubtreePool *subtree_pool);
unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version);
void ts_stack_pause(Stack *self, StackVersion version, Subtree lookahead);
StackSliceArray ts_stack_pop_all(Stack *self, StackVersion version);
StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t count);
SubtreeArray ts_stack_pop_error(Stack *self, StackVersion version);
StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version);
Length ts_stack_position(const Stack *self, StackVersion version);
bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f);
void ts_stack_push(Stack *self, StackVersion version, Subtree subtree, bool pending, TSStateId state);
void ts_stack_record_summary(Stack *self, StackVersion version, unsigned max_depth);
void ts_stack_remove_version(Stack *self, StackVersion version);
void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2);
Subtree ts_stack_resume(Stack *self, StackVersion version);
void ts_stack_set_last_external_token(Stack *self, StackVersion version, Subtree token);
TSStateId ts_stack_state(const Stack *self, StackVersion version);
void ts_stack_swap_versions(Stack *self, StackVersion v1, StackVersion v2);
uint32_t ts_stack_version_count(const Stack *self);
const char *ts_string_input_read(void *_self, uint32_t byte, TSPoint point, uint32_t *length);
void ts_subtree__compress(MutableSubtree self, unsigned count, const TSLanguage *language, MutableSubtreeArray *stack);
void ts_subtree__print_dot_graph(const Subtree *self, uint32_t start_offset, const TSLanguage *language, TSSymbol alias_symbol, FILE *f);
size_t ts_subtree__write_char_to_string(char *str, size_t n, int32_t chr);
size_t ts_subtree__write_to_string(Subtree self, char *string, size_t limit, const TSLanguage *language, bool include_all, TSSymbol alias_symbol, bool alias_is_named, const char *field_name);
void ts_subtree_array_clear(SubtreePool *pool, SubtreeArray *self);
void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest);
void ts_subtree_array_delete(SubtreePool *pool, SubtreeArray *self);
void ts_subtree_array_remove_trailing_extras(SubtreeArray *self, SubtreeArray *destination);
void ts_subtree_array_reverse(SubtreeArray *self);
void ts_subtree_balance(Subtree self, SubtreePool *pool, const TSLanguage *language);
MutableSubtree ts_subtree_clone(Subtree self);
int ts_subtree_compare(Subtree left, Subtree right, SubtreePool *pool);
Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit, SubtreePool *pool);
const ExternalScannerState *ts_subtree_external_scanner_state(Subtree self);
bool ts_subtree_external_scanner_state_eq(Subtree self, Subtree other);
unsigned ts_subtree_get_changed_ranges(const Subtree *old_tree, const Subtree *new_tree, TreeCursor *cursor1, TreeCursor *cursor2, const TSLanguage *language, const TSRangeArray *included_range_differences, TSRange **ranges);
bool ts_subtree_has_trailing_empty_descendant(Subtree self, Subtree other);
Subtree ts_subtree_last_external_token(Subtree tree);
MutableSubtree ts_subtree_make_mut(SubtreePool *pool, Subtree self);
Subtree ts_subtree_new_error(SubtreePool *pool, int32_t lookahead_char, Length padding, Length size, uint32_t bytes_scanned, TSStateId parse_state, const TSLanguage *language);
Subtree ts_subtree_new_error_node(SubtreeArray *children, bool extra, const TSLanguage *language);
Subtree ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Length size, uint32_t lookahead_bytes, TSStateId parse_state, bool has_external_tokens, bool depends_on_column, bool is_keyword, const TSLanguage *language);
Subtree ts_subtree_new_missing_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, uint32_t lookahead_bytes, const TSLanguage *language);
MutableSubtree ts_subtree_new_node(TSSymbol symbol, SubtreeArray *children, unsigned production_id, const TSLanguage *language);
SubtreeHeapData *ts_subtree_pool_allocate(SubtreePool *self);
void ts_subtree_pool_delete(SubtreePool *self);
void ts_subtree_pool_free(SubtreePool *self, SubtreeHeapData *tree);
SubtreePool ts_subtree_pool_new(uint32_t capacity);
void ts_subtree_print_dot_graph(Subtree self, const TSLanguage *language, FILE *f);
void ts_subtree_release(SubtreePool *pool, Subtree self);
void ts_subtree_retain(Subtree self);
void ts_subtree_set_symbol(MutableSubtree *self, TSSymbol symbol, const TSLanguage *language);
char *ts_subtree_string(Subtree self, TSSymbol alias_symbol, bool alias_is_named, const TSLanguage *language, bool include_all);
void ts_subtree_summarize_children(MutableSubtree self, const TSLanguage *language);
TSTree *ts_tree_copy(const TSTree *self);
TSTreeCursor ts_tree_cursor_copy(const TSTreeCursor *_cursor);
uint32_t ts_tree_cursor_current_depth(const TSTreeCursor *_self);
uint32_t ts_tree_cursor_current_descendant_index(const TSTreeCursor *_self);
TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *_self);
const char *ts_tree_cursor_current_field_name(const TSTreeCursor *_self);
TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self);
void ts_tree_cursor_current_status(const TSTreeCursor *_self, TSFieldId *field_id, bool *has_later_siblings, bool *has_later_named_siblings, bool *can_have_later_siblings_with_this_field, TSSymbol *supertypes, unsigned *supertype_count);
void ts_tree_cursor_delete(TSTreeCursor *_self);
void ts_tree_cursor_goto_descendant(TSTreeCursor *_self, uint32_t goal_descendant_index);
bool ts_tree_cursor_goto_first_child(TSTreeCursor *self);
int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *self, uint32_t goal_byte);
int64_t ts_tree_cursor_goto_first_child_for_point(TSTreeCursor *self, TSPoint goal_point);
TreeCursorStep ts_tree_cursor_goto_first_child_internal(TSTreeCursor *_self);
bool ts_tree_cursor_goto_last_child(TSTreeCursor *self);
TreeCursorStep ts_tree_cursor_goto_last_child_internal(TSTreeCursor *_self);
bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self);
TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *_self);
bool ts_tree_cursor_goto_parent(TSTreeCursor *_self);
bool ts_tree_cursor_goto_previous_sibling(TSTreeCursor *self);
TreeCursorStep ts_tree_cursor_goto_previous_sibling_internal(TSTreeCursor *_self);
TreeCursorStep ts_tree_cursor_goto_sibling_internal(TSTreeCursor *_self, bool (*advance)(CursorChildIterator *, TreeCursorEntry *, bool *));
void ts_tree_cursor_init(TreeCursor *self, TSNode node);
TSTreeCursor ts_tree_cursor_new(TSNode node);
TSNode ts_tree_cursor_parent_node(const TSTreeCursor *_self);
void ts_tree_cursor_reset(TSTreeCursor *_self, TSNode node);
void ts_tree_cursor_reset_to(TSTreeCursor *_dst, const TSTreeCursor *_src);
void ts_tree_delete(TSTree *self);
void ts_tree_edit(TSTree *self, const TSInputEdit *edit);
TSRange *ts_tree_get_changed_ranges(const TSTree *old_tree, const TSTree *new_tree, uint32_t *length);
TSRange *ts_tree_included_ranges(const TSTree *self, uint32_t *length);
const TSLanguage *ts_tree_language(const TSTree *self);
TSTree *ts_tree_new(Subtree root, const TSLanguage *language, const TSRange *included_ranges, unsigned included_range_count);
void ts_tree_print_dot_graph(const TSTree *self, int file_descriptor);
TSNode ts_tree_root_node(const TSTree *self);
TSNode ts_tree_root_node_with_offset(const TSTree *self, uint32_t offset_bytes, TSPoint offset_extent);

935
parser/gen/types.h
View file

@ -1,935 +0,0 @@
#include <stdint.h>
#include <stdio.h>
typedef uint16_t TSStateId;
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
typedef struct TSParser TSParser;
typedef struct TSTree TSTree;
typedef struct TSQuery TSQuery;
typedef struct TSQueryCursor TSQueryCursor;
typedef struct TSLookaheadIterator TSLookaheadIterator;
typedef enum TSInputEncoding
{
TSInputEncodingUTF8,
TSInputEncodingUTF16,
} TSInputEncoding;
typedef enum TSSymbolType
{
TSSymbolTypeRegular,
TSSymbolTypeAnonymous,
TSSymbolTypeAuxiliary,
} TSSymbolType;
typedef struct TSPoint
{
uint32_t row;
uint32_t column;
} TSPoint;
typedef struct TSRange
{
TSPoint start_point;
TSPoint end_point;
uint32_t start_byte;
uint32_t end_byte;
} TSRange;
typedef struct TSInput
{
void *payload;
const char *(*read)(void *payload, uint32_t byte_index, TSPoint position, uint32_t *bytes_read);
TSInputEncoding encoding;
} TSInput;
typedef enum TSLogType
{
TSLogTypeParse,
TSLogTypeLex,
} TSLogType;
typedef struct TSLogger
{
void *payload;
void (*log)(void *payload, TSLogType log_type, const char *buffer);
} TSLogger;
typedef struct TSInputEdit
{
uint32_t start_byte;
uint32_t old_end_byte;
uint32_t new_end_byte;
TSPoint start_point;
TSPoint old_end_point;
TSPoint new_end_point;
} TSInputEdit;
typedef struct TSNode
{
uint32_t context[4];
const void *id;
const TSTree *tree;
} TSNode;
typedef struct TSTreeCursor
{
const void *tree;
const void *id;
uint32_t context[3];
} TSTreeCursor;
typedef struct TSQueryCapture
{
TSNode node;
uint32_t index;
} TSQueryCapture;
typedef enum TSQuantifier
{
TSQuantifierZero = 0,
TSQuantifierZeroOrOne,
TSQuantifierZeroOrMore,
TSQuantifierOne,
TSQuantifierOneOrMore,
} TSQuantifier;
typedef struct TSQueryMatch
{
uint32_t id;
uint16_t pattern_index;
uint16_t capture_count;
const TSQueryCapture *captures;
} TSQueryMatch;
typedef enum TSQueryPredicateStepType
{
TSQueryPredicateStepTypeDone,
TSQueryPredicateStepTypeCapture,
TSQueryPredicateStepTypeString,
} TSQueryPredicateStepType;
typedef struct TSQueryPredicateStep
{
TSQueryPredicateStepType type;
uint32_t value_id;
} TSQueryPredicateStep;
typedef enum TSQueryError
{
TSQueryErrorNone = 0,
TSQueryErrorSyntax,
TSQueryErrorNodeType,
TSQueryErrorField,
TSQueryErrorCapture,
TSQueryErrorStructure,
TSQueryErrorLanguage,
} TSQueryError;
typedef struct wasm_engine_t TSWasmEngine;
typedef struct TSWasmStore TSWasmStore;
typedef enum
{
TSWasmErrorKindNone = 0,
TSWasmErrorKindParse,
TSWasmErrorKindCompile,
TSWasmErrorKindInstantiate,
TSWasmErrorKindAllocate,
} TSWasmErrorKind;
typedef struct
{
TSWasmErrorKind kind;
char *message;
} TSWasmError;
typedef struct
{
void *contents;
uint32_t size;
uint32_t capacity;
} Array;
typedef struct
{
uint32_t bytes;
TSPoint extent;
} Length;
typedef struct
{
TSFieldId field_id;
uint8_t child_index;
_Bool inherited;
} TSFieldMapEntry;
typedef struct
{
uint16_t index;
uint16_t length;
} TSFieldMapSlice;
typedef struct
{
_Bool visible;
_Bool named;
_Bool supertype;
} TSSymbolMetadata;
typedef struct TSLexer TSLexer;
struct TSLexer
{
int32_t lookahead;
TSSymbol result_symbol;
void (*advance)(TSLexer *, _Bool);
void (*mark_end)(TSLexer *);
uint32_t (*get_column)(TSLexer *);
_Bool (*is_at_included_range_start)(const TSLexer *);
_Bool (*eof)(const TSLexer *);
};
typedef enum
{
TSParseActionTypeShift,
TSParseActionTypeReduce,
TSParseActionTypeAccept,
TSParseActionTypeRecover,
} TSParseActionType;
typedef union {
struct
{
uint8_t type;
TSStateId state;
_Bool extra;
_Bool repetition;
} shift;
struct
{
uint8_t type;
uint8_t child_count;
TSSymbol symbol;
int16_t dynamic_precedence;
uint16_t production_id;
} reduce;
uint8_t type;
} TSParseAction;
typedef struct
{
uint16_t lex_state;
uint16_t external_lex_state;
} TSLexMode;
typedef union {
TSParseAction action;
struct
{
uint8_t count;
_Bool reusable;
} entry;
} TSParseActionEntry;
typedef struct
{
int32_t start;
int32_t end;
} TSCharacterRange;
struct TSLanguage
{
uint32_t version;
uint32_t symbol_count;
uint32_t alias_count;
uint32_t token_count;
uint32_t external_token_count;
uint32_t state_count;
uint32_t large_state_count;
uint32_t production_id_count;
uint32_t field_count;
uint16_t max_alias_sequence_length;
const uint16_t *parse_table;
const uint16_t *small_parse_table;
const uint32_t *small_parse_table_map;
const TSParseActionEntry *parse_actions;
const char *const *symbol_names;
const char *const *field_names;
const TSFieldMapSlice *field_map_slices;
const TSFieldMapEntry *field_map_entries;
const TSSymbolMetadata *symbol_metadata;
const TSSymbol *public_symbol_map;
const uint16_t *alias_map;
const TSSymbol *alias_sequences;
const TSLexMode *lex_modes;
_Bool (*lex_fn)(TSLexer *, TSStateId);
_Bool (*keyword_lex_fn)(TSLexer *, TSStateId);
TSSymbol keyword_capture_token;
struct
{
const _Bool *states;
const TSSymbol *symbol_map;
void *(*create)(void);
void (*destroy)(void *);
_Bool (*scan)(void *, TSLexer *, const _Bool *symbol_whitelist);
unsigned (*serialize)(void *, char *);
void (*deserialize)(void *, const char *, unsigned);
} external_scanner;
const TSStateId *primary_state_ids;
};
typedef struct
{
union {
char *long_data;
char short_data[24];
};
uint32_t length;
} ExternalScannerState;
typedef struct SubtreeInlineData SubtreeInlineData;
struct SubtreeInlineData
{
_Bool is_inline : 1;
_Bool visible : 1;
_Bool named : 1;
_Bool extra : 1;
_Bool has_changes : 1;
_Bool is_missing : 1;
_Bool is_keyword : 1;
uint8_t symbol;
uint16_t parse_state;
uint8_t padding_columns;
uint8_t padding_rows : 4;
uint8_t lookahead_bytes : 4;
uint8_t padding_bytes;
uint8_t size_bytes;
};
typedef struct
{
volatile uint32_t ref_count;
Length padding;
Length size;
uint32_t lookahead_bytes;
uint32_t error_cost;
uint32_t child_count;
TSSymbol symbol;
TSStateId parse_state;
_Bool visible : 1;
_Bool named : 1;
_Bool extra : 1;
_Bool fragile_left : 1;
_Bool fragile_right : 1;
_Bool has_changes : 1;
_Bool has_external_tokens : 1;
_Bool has_external_scanner_state_change : 1;
_Bool depends_on_column : 1;
_Bool is_missing : 1;
_Bool is_keyword : 1;
union {
struct
{
uint32_t visible_child_count;
uint32_t named_child_count;
uint32_t visible_descendant_count;
int32_t dynamic_precedence;
uint16_t repeat_depth;
uint16_t production_id;
struct
{
TSSymbol symbol;
TSStateId parse_state;
} first_leaf;
};
ExternalScannerState external_scanner_state;
int32_t lookahead_char;
};
} SubtreeHeapData;
typedef union {
SubtreeInlineData data;
const SubtreeHeapData *ptr;
} Subtree;
typedef union {
SubtreeInlineData data;
SubtreeHeapData *ptr;
} MutableSubtree;
typedef struct
{
Subtree *contents;
uint32_t size;
uint32_t capacity;
} SubtreeArray;
typedef struct
{
MutableSubtree *contents;
uint32_t size;
uint32_t capacity;
} MutableSubtreeArray;
typedef struct
{
MutableSubtreeArray free_trees;
MutableSubtreeArray tree_stack;
} SubtreePool;
typedef struct
{
const Subtree *subtree;
Length position;
uint32_t child_index;
uint32_t structural_child_index;
uint32_t descendant_index;
} TreeCursorEntry;
typedef struct
{
const TSTree *tree;
struct
{
TreeCursorEntry *contents;
uint32_t size;
uint32_t capacity;
} stack;
TSSymbol root_alias_symbol;
} TreeCursor;
typedef enum
{
TreeCursorStepNone,
TreeCursorStepHidden,
TreeCursorStepVisible,
} TreeCursorStep;
typedef struct
{
TSRange *contents;
uint32_t size;
uint32_t capacity;
} TSRangeArray;
typedef struct
{
const TSParseAction *actions;
uint32_t action_count;
_Bool is_reusable;
} TableEntry;
typedef struct
{
const TSLanguage *language;
const uint16_t *data;
const uint16_t *group_end;
TSStateId state;
uint16_t table_value;
uint16_t section_index;
uint16_t group_count;
_Bool is_small_state;
const TSParseAction *actions;
TSSymbol symbol;
TSStateId next_state;
uint16_t action_count;
} LookaheadIterator;
typedef struct
{
TreeCursor cursor;
const TSLanguage *language;
unsigned visible_depth;
_Bool in_padding;
} Iterator;
typedef enum
{
IteratorDiffers,
IteratorMayDiffer,
IteratorMatches,
} IteratorComparison;
typedef struct
{
TSLexer data;
Length current_position;
Length token_start_position;
Length token_end_position;
TSRange *included_ranges;
const char *chunk;
TSInput input;
TSLogger logger;
uint32_t included_range_count;
uint32_t current_included_range_index;
uint32_t chunk_start;
uint32_t chunk_size;
uint32_t lookahead_size;
_Bool did_get_column;
char debug_buffer[1024];
} Lexer;
typedef struct
{
const Subtree *child;
const Subtree *parent;
Length position;
TSSymbol alias_symbol;
} ParentCacheEntry;
struct TSTree
{
Subtree root;
const TSLanguage *language;
TSRange *included_ranges;
unsigned included_range_count;
};
typedef struct
{
Subtree parent;
const TSTree *tree;
Length position;
uint32_t child_index;
uint32_t structural_child_index;
const TSSymbol *alias_sequence;
} NodeChildIterator;
typedef uint64_t TSDuration;
typedef uint64_t TSClock;
typedef struct
{
uint32_t count;
TSSymbol symbol;
int dynamic_precedence;
unsigned short production_id;
} ReduceAction;
typedef struct
{
ReduceAction *contents;
uint32_t size;
uint32_t capacity;
} ReduceActionSet;
typedef struct
{
Subtree tree;
uint32_t child_index;
uint32_t byte_offset;
} StackEntry;
typedef struct
{
struct
{
StackEntry *contents;
uint32_t size;
uint32_t capacity;
} stack;
Subtree last_external_token;
} ReusableNode;
typedef struct Stack Stack;
typedef unsigned StackVersion;
typedef struct
{
SubtreeArray subtrees;
StackVersion version;
} StackSlice;
typedef struct
{
StackSlice *contents;
uint32_t size;
uint32_t capacity;
} StackSliceArray;
typedef struct
{
Length position;
unsigned depth;
TSStateId state;
} StackSummaryEntry;
typedef struct
{
StackSummaryEntry *contents;
uint32_t size;
uint32_t capacity;
} StackSummary;
typedef void (*StackIterateCallback)(void *, TSStateId, uint32_t);
typedef int __gwchar_t;
typedef struct
{
long int quot;
long int rem;
} imaxdiv_t;
struct tm
{
int tm_sec;
int tm_min;
int tm_hour;
int tm_mday;
int tm_mon;
int tm_year;
int tm_wday;
int tm_yday;
int tm_isdst;
long int tm_gmtoff;
const char *tm_zone;
};
typedef struct
{
Subtree token;
Subtree last_external_token;
uint32_t byte_index;
} TokenCache;
struct TSParser
{
Lexer lexer;
Stack *stack;
SubtreePool tree_pool;
const TSLanguage *language;
ReduceActionSet reduce_actions;
Subtree finished_tree;
SubtreeArray trailing_extras;
SubtreeArray trailing_extras2;
SubtreeArray scratch_trees;
TokenCache token_cache;
ReusableNode reusable_node;
void *external_scanner_payload;
FILE *dot_graph_file;
TSClock end_clock;
TSDuration timeout_duration;
unsigned accept_count;
unsigned operation_count;
const volatile size_t *cancellation_flag;
Subtree old_tree;
TSRangeArray included_range_differences;
unsigned included_range_difference_index;
_Bool has_scanner_error;
};
typedef struct
{
unsigned cost;
unsigned node_count;
int dynamic_precedence;
_Bool is_in_error;
} ErrorStatus;
typedef enum
{
ErrorComparisonTakeLeft,
ErrorComparisonPreferLeft,
ErrorComparisonNone,
ErrorComparisonPreferRight,
ErrorComparisonTakeRight,
} ErrorComparison;
typedef struct
{
const char *string;
uint32_t length;
} TSStringInput;
typedef struct
{
const char *input;
const char *start;
const char *end;
int32_t next;
uint8_t next_size;
} Stream;
typedef struct
{
TSSymbol symbol;
TSSymbol supertype_symbol;
TSFieldId field;
uint16_t capture_ids[3];
uint16_t depth;
uint16_t alternative_index;
uint16_t negated_field_list_id;
_Bool is_named : 1;
_Bool is_immediate : 1;
_Bool is_last_child : 1;
_Bool is_pass_through : 1;
_Bool is_dead_end : 1;
_Bool alternative_is_immediate : 1;
_Bool contains_captures : 1;
_Bool root_pattern_guaranteed : 1;
_Bool parent_pattern_guaranteed : 1;
} QueryStep;
typedef struct
{
uint32_t offset;
uint32_t length;
} Slice;
typedef struct
{
struct
{
char *contents;
uint32_t size;
uint32_t capacity;
} characters;
struct
{
Slice *contents;
uint32_t size;
uint32_t capacity;
} slices;
} SymbolTable;
typedef struct
{
uint8_t *contents;
uint32_t size;
uint32_t capacity;
} CaptureQuantifiers;
typedef struct
{
uint16_t step_index;
uint16_t pattern_index;
_Bool is_rooted;
} PatternEntry;
typedef struct
{
Slice steps;
Slice predicate_steps;
uint32_t start_byte;
_Bool is_non_local;
} QueryPattern;
typedef struct
{
uint32_t byte_offset;
uint16_t step_index;
} StepOffset;
typedef struct
{
uint32_t id;
uint32_t capture_list_id;
uint16_t start_depth;
uint16_t step_index;
uint16_t pattern_index;
uint16_t consumed_capture_count : 12;
_Bool seeking_immediate_match : 1;
_Bool has_in_progress_alternatives : 1;
_Bool dead : 1;
_Bool needs_parent : 1;
} QueryState;
typedef struct
{
TSQueryCapture *contents;
uint32_t size;
uint32_t capacity;
} CaptureList;
typedef struct
{
struct
{
CaptureList *contents;
uint32_t size;
uint32_t capacity;
} list;
CaptureList empty_list;
uint32_t max_capture_list_count;
uint32_t free_capture_list_count;
} CaptureListPool;
typedef struct
{
TSStateId parse_state;
TSSymbol parent_symbol;
uint16_t child_index;
TSFieldId field_id : 15;
_Bool done : 1;
} AnalysisStateEntry;
typedef struct
{
AnalysisStateEntry stack[8];
uint16_t depth;
uint16_t step_index;
TSSymbol root_symbol;
} AnalysisState;
typedef struct
{
AnalysisState **contents;
uint32_t size;
uint32_t capacity;
} AnalysisStateSet;
typedef struct
{
AnalysisStateSet states;
AnalysisStateSet next_states;
AnalysisStateSet deeper_states;
AnalysisStateSet state_pool;
struct
{
uint16_t *contents;
uint32_t size;
uint32_t capacity;
} final_step_indices;
struct
{
TSSymbol *contents;
uint32_t size;
uint32_t capacity;
} finished_parent_symbols;
_Bool did_abort;
} QueryAnalysis;
typedef struct
{
TSStateId state;
uint16_t production_id;
uint8_t child_index : 7;
_Bool done : 1;
} AnalysisSubgraphNode;
typedef struct
{
TSSymbol symbol;
struct
{
TSStateId *contents;
uint32_t size;
uint32_t capacity;
} start_states;
struct
{
AnalysisSubgraphNode *contents;
uint32_t size;
uint32_t capacity;
} nodes;
} AnalysisSubgraph;
typedef struct
{
AnalysisSubgraph *contents;
uint32_t size;
uint32_t capacity;
} AnalysisSubgraphArray;
typedef struct
{
TSStateId *contents;
} StatePredecessorMap;
struct TSQuery
{
SymbolTable captures;
SymbolTable predicate_values;
struct
{
CaptureQuantifiers *contents;
uint32_t size;
uint32_t capacity;
} capture_quantifiers;
struct
{
QueryStep *contents;
uint32_t size;
uint32_t capacity;
} steps;
struct
{
PatternEntry *contents;
uint32_t size;
uint32_t capacity;
} pattern_map;
struct
{
TSQueryPredicateStep *contents;
uint32_t size;
uint32_t capacity;
} predicate_steps;
struct
{
QueryPattern *contents;
uint32_t size;
uint32_t capacity;
} patterns;
struct
{
StepOffset *contents;
uint32_t size;
uint32_t capacity;
} step_offsets;
struct
{
TSFieldId *contents;
uint32_t size;
uint32_t capacity;
} negated_fields;
struct
{
char *contents;
uint32_t size;
uint32_t capacity;
} string_buffer;
struct
{
TSSymbol *contents;
uint32_t size;
uint32_t capacity;
} repeat_symbols_with_rootless_patterns;
const TSLanguage *language;
uint16_t wildcard_root_pattern_count;
};
struct TSQueryCursor
{
const TSQuery *query;
TSTreeCursor cursor;
struct
{
QueryState *contents;
uint32_t size;
uint32_t capacity;
} states;
struct
{
QueryState *contents;
uint32_t size;
uint32_t capacity;
} finished_states;
CaptureListPool capture_list_pool;
uint32_t depth;
uint32_t max_start_depth;
uint32_t start_byte;
uint32_t end_byte;
TSPoint start_point;
TSPoint end_point;
uint32_t next_state_id;
_Bool on_visible_node;
_Bool ascending;
_Bool halted;
_Bool did_exceed_match_limit;
};
typedef struct StackNode StackNode;
typedef struct
{
StackNode *node;
Subtree subtree;
_Bool is_pending;
} StackLink;
struct StackNode
{
TSStateId state;
Length position;
StackLink links[8];
short unsigned int link_count;
uint32_t ref_count;
unsigned error_cost;
unsigned node_count;
int dynamic_precedence;
};
typedef struct
{
StackNode *node;
SubtreeArray subtrees;
uint32_t subtree_count;
_Bool is_pending;
} StackIterator;
typedef struct
{
StackNode **contents;
uint32_t size;
uint32_t capacity;
} StackNodeArray;
typedef enum
{
StackStatusActive,
StackStatusPaused,
StackStatusHalted,
} StackStatus;
typedef struct
{
StackNode *node;
StackSummary *summary;
unsigned node_count_at_last_error;
Subtree last_external_token;
Subtree lookahead_when_paused;
StackStatus status;
} StackHead;
struct Stack
{
struct
{
StackHead *contents;
uint32_t size;
uint32_t capacity;
} heads;
StackSliceArray slices;
struct
{
StackIterator *contents;
uint32_t size;
uint32_t capacity;
} iterators;
StackNodeArray node_pool;
StackNode *base_node;
SubtreePool *subtree_pool;
};
typedef unsigned StackAction;
enum
{
StackActionNone,
StackActionStop = 1,
StackActionPop = 2,
};
typedef StackAction (*StackCallback)(void *, const StackIterator *);
typedef struct
{
StackSummary *summary;
unsigned max_depth;
} SummarizeStackSession;
typedef struct
{
Length start;
Length old_end;
Length new_end;
} Edit;
typedef struct
{
Subtree parent;
const TSTree *tree;
Length position;
uint32_t child_index;
uint32_t structural_child_index;
uint32_t descendant_index;
const TSSymbol *alias_sequence;
} CursorChildIterator;

2
parser/nnsrc/lib.c
View file

@ -13,5 +13,5 @@
#include "./tree_cursor.c" #include "./tree_cursor.c"
#include "./wasm_store.c" #include "./wasm_store.c"
#include "./create_language.c" //#include "./create_language.c"
#include "./scanner.c" #include "./scanner.c"

1
parser/nnsrc/scanner.c
View file

@ -250,6 +250,7 @@ static inline bool scan_bare_dollar(TSLexer *lexer)
while (iswspace(lexer->lookahead) && lexer->lookahead != '\n' && !lexer->eof(lexer)) while (iswspace(lexer->lookahead) && lexer->lookahead != '\n' && !lexer->eof(lexer))
skip(lexer); skip(lexer);
if (lexer->lookahead == '$') if (lexer->lookahead == '$')
{ {
advance(lexer); advance(lexer);

54
parser/nnsrc/unicode.h
View file

@ -1,50 +1,46 @@
#ifndef TREE_SITTER_UNICODE_H_ #ifndef TREE_SITTER_UNICODE_H_
#define TREE_SITTER_UNICODE_H_ #define TREE_SITTER_UNICODE_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <limits.h> #include <limits.h>
#include <stdint.h> #include <stdint.h>
#include <stdio.h>
#define U_EXPORT #define U_EXPORT
#define U_EXPORT2 #define U_EXPORT2
#include "unicode/utf8.h"
#include "unicode/utf16.h" #include "unicode/utf16.h"
#include "unicode/utf8.h"
static const int32_t TS_DECODE_ERROR = U_SENTINEL; static const int32_t TS_DECODE_ERROR = U_SENTINEL;
// These functions read one unicode code point from the given string, // These functions read one unicode code point from the given string,
// returning the number of bytes consumed. // returning the number of bytes consumed.
typedef uint32_t (*UnicodeDecodeFunction)( typedef uint32_t (*UnicodeDecodeFunction)(const uint8_t *string, uint32_t length, int32_t *code_point);
const uint8_t *string,
uint32_t length,
int32_t *code_point
);
static inline uint32_t ts_decode_utf8( static inline uint32_t ts_decode_ascii(const uint8_t *string, uint32_t length, int32_t *code_point)
const uint8_t *string, {
uint32_t length, (void)(length);
int32_t *code_point *code_point = 0;
) { *(uint8_t *)code_point = *string;
uint32_t i = 0; return (1);
U8_NEXT(string, i, length, *code_point);
return i;
} }
static inline uint32_t ts_decode_utf16( static inline uint32_t ts_decode_utf8(const uint8_t *string, uint32_t length, int32_t *code_point)
const uint8_t *string, {
uint32_t length, return (ts_decode_ascii(string, length, code_point));
int32_t *code_point /*
) { uint32_t i = 0;
uint32_t i = 0; U8_NEXT(string, i, length, *code_point);
U16_NEXT(((uint16_t *)string), i, length, *code_point); printf("[UTF8]read %i bytes\n", i);
return i * 2; return i;
*/
} }
#ifdef __cplusplus static inline uint32_t ts_decode_utf16(const uint8_t *string, uint32_t length, int32_t *code_point)
{
uint32_t i = 0;
U16_NEXT(((uint16_t *)string), i, length, *code_point);
printf("[UTF16]read %i bytes\n", i);
return i * 2;
} }
#endif
#endif // TREE_SITTER_UNICODE_H_ #endif // TREE_SITTER_UNICODE_H_

48
parser/nsrc/alloc.c
View file

@ -1,48 +0,0 @@
#include "alloc.h"
#include "./api.h"
#include <stdlib.h>
static void *ts_malloc_default(size_t size) {
void *result = malloc(size);
if (size > 0 && !result) {
fprintf(stderr, "tree-sitter failed to allocate %zu bytes", size);
abort();
}
return result;
}
static void *ts_calloc_default(size_t count, size_t size) {
void *result = calloc(count, size);
if (count > 0 && !result) {
fprintf(stderr, "tree-sitter failed to allocate %zu bytes", count * size);
abort();
}
return result;
}
static void *ts_realloc_default(void *buffer, size_t size) {
void *result = realloc(buffer, size);
if (size > 0 && !result) {
fprintf(stderr, "tree-sitter failed to reallocate %zu bytes", size);
abort();
}
return result;
}
// Allow clients to override allocation functions dynamically
TS_PUBLIC void *(*ts_current_malloc)(size_t) = ts_malloc_default;
TS_PUBLIC void *(*ts_current_calloc)(size_t, size_t) = ts_calloc_default;
TS_PUBLIC void *(*ts_current_realloc)(void *, size_t) = ts_realloc_default;
TS_PUBLIC void (*ts_current_free)(void *) = free;
void ts_set_allocator(
void *(*new_malloc)(size_t size),
void *(*new_calloc)(size_t count, size_t size),
void *(*new_realloc)(void *ptr, size_t size),
void (*new_free)(void *ptr)
) {
ts_current_malloc = new_malloc ? new_malloc : ts_malloc_default;
ts_current_calloc = new_calloc ? new_calloc : ts_calloc_default;
ts_current_realloc = new_realloc ? new_realloc : ts_realloc_default;
ts_current_free = new_free ? new_free : free;
}

41
parser/nsrc/alloc.h
View file

@ -1,41 +0,0 @@
#ifndef TREE_SITTER_ALLOC_H_
#define TREE_SITTER_ALLOC_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#if defined(TREE_SITTER_HIDDEN_SYMBOLS) || defined(_WIN32)
#define TS_PUBLIC
#else
#define TS_PUBLIC __attribute__((visibility("default")))
#endif
TS_PUBLIC extern void *(*ts_current_malloc)(size_t);
TS_PUBLIC extern void *(*ts_current_calloc)(size_t, size_t);
TS_PUBLIC extern void *(*ts_current_realloc)(void *, size_t);
TS_PUBLIC extern void (*ts_current_free)(void *);
// Allow clients to override allocation functions
#ifndef ts_malloc
#define ts_malloc ts_current_malloc
#endif
#ifndef ts_calloc
#define ts_calloc ts_current_calloc
#endif
#ifndef ts_realloc
#define ts_realloc ts_current_realloc
#endif
#ifndef ts_free
#define ts_free ts_current_free
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ALLOC_H_

1207
parser/nsrc/api.h
View file

File diff suppressed because it is too large Load diff

293
parser/nsrc/array.h
View file

@ -1,293 +0,0 @@
#ifndef TREE_SITTER_ARRAY_H_
#define TREE_SITTER_ARRAY_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include "./alloc.h"
#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#ifdef _MSC_VER
# pragma warning(disable : 4101)
#elif defined(__GNUC__) || defined(__clang__)
# pragma GCC diagnostic push
# pragma GCC diagnostic ignored "-Wunused-variable"
#endif
#define Array(T) \
struct \
{ \
T *contents; \
uint32_t size; \
uint32_t capacity; \
}
/// Initialize an array.
#define array_init(self) ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
/// Create an empty array.
#define array_new() \
{ \
NULL, 0, 0 \
}
/// Get a pointer to the element at a given `index` in the array.
#define array_get(self, _index) (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
/// Get a pointer to the first element in the array.
#define array_front(self) array_get(self, 0)
/// Get a pointer to the last element in the array.
#define array_back(self) array_get(self, (self)->size - 1)
/// Clear the array, setting its size to zero. Note that this does not free any
/// memory allocated for the array's contents.
#define array_clear(self) ((self)->size = 0)
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
/// less than the array's current capacity, this function has no effect.
#define array_reserve(self, new_capacity) _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
/// Free any memory allocated for this array. Note that this does not free any
/// memory allocated for the array's contents.
#define array_delete(self) _array__delete((Array *)(self))
/// Push a new `element` onto the end of the array.
#define array_push(self, element) (_array__grow((Array *)(self), 1, array_elem_size(self)), (self)->contents[(self)->size++] = (element))
/// Increase the array's size by `count` elements.
/// New elements are zero-initialized.
#define array_grow_by(self, count) \
do \
{ \
if ((count) == 0) \
break; \
_array__grow((Array *)(self), count, array_elem_size(self)); \
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
(self)->size += (count); \
} while (0)
/// Append all elements from one array to the end of another.
#define array_push_all(self, other) array_extend((self), (other)->size, (other)->contents)
/// Append `count` elements to the end of the array, reading their values from the
/// `contents` pointer.
#define array_extend(self, count, contents) _array__splice((Array *)(self), array_elem_size(self), (self)->size, 0, count, contents)
/// Remove `old_count` elements from the array starting at the given `index`. At
/// the same index, insert `new_count` new elements, reading their values from the
/// `new_contents` pointer.
#define array_splice(self, _index, old_count, new_count, new_contents) \
_array__splice((Array *)(self), array_elem_size(self), _index, old_count, new_count, new_contents)
/// Insert one `element` into the array at the given `index`.
#define array_insert(self, _index, element) _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
/// Remove one element from the array at the given `index`.
#define array_erase(self, _index) _array__erase((Array *)(self), array_elem_size(self), _index)
/// Pop the last element off the array, returning the element by value.
#define array_pop(self) ((self)->contents[--(self)->size])
/// Assign the contents of one array to another, reallocating if necessary.
#define array_assign(self, other) _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
/// Swap one array with another
#define array_swap(self, other) _array__swap((Array *)(self), (Array *)(other))
/// Get the size of the array contents
#define array_elem_size(self) (sizeof *(self)->contents)
/// Search a sorted array for a given `needle` value, using the given `compare`
/// callback to determine the order.
///
/// If an existing element is found to be equal to `needle`, then the `index`
/// out-parameter is set to the existing value's index, and the `exists`
/// out-parameter is set to true. Otherwise, `index` is set to an index where
/// `needle` should be inserted in order to preserve the sorting, and `exists`
/// is set to false.
#define array_search_sorted_with(self, compare, needle, _index, _exists) _array__search_sorted(self, 0, compare, , needle, _index, _exists)
/// Search a sorted array for a given `needle` value, using integer comparisons
/// of a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_with`.
#define array_search_sorted_by(self, field, needle, _index, _exists) \
_array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
/// Insert a given `value` into a sorted array, using the given `compare`
/// callback to determine the order.
#define array_insert_sorted_with(self, compare, value) \
do \
{ \
unsigned _index, _exists; \
array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
if (!_exists) \
array_insert(self, _index, value); \
} while (0)
/// Insert a given `value` into a sorted array, using integer comparisons of
/// a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_by`.
#define array_insert_sorted_by(self, field, value) \
do \
{ \
unsigned _index, _exists; \
array_search_sorted_by(self, field, (value)field, &_index, &_exists); \
if (!_exists) \
array_insert(self, _index, value); \
} while (0)
// Private
typedef Array(void) Array;
/// This is not what you're looking for, see `array_delete`.
static inline void _array__delete(Array *self)
{
if (self->contents)
{
ts_free(self->contents);
self->contents = NULL;
self->size = 0;
self->capacity = 0;
}
}
/// This is not what you're looking for, see `array_erase`.
static inline void _array__erase(Array *self, size_t element_size, uint32_t index)
{
assert(index < self->size);
char *contents = (char *)self->contents;
memmove(contents + index * element_size, contents + (index + 1) * element_size, (self->size - index - 1) * element_size);
self->size--;
}
/// This is not what you're looking for, see `array_reserve`.
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity)
{
if (new_capacity > self->capacity)
{
if (self->contents)
{
self->contents = ts_realloc(self->contents, new_capacity * element_size);
}
else
{
self->contents = ts_malloc(new_capacity * element_size);
}
self->capacity = new_capacity;
}
}
/// This is not what you're looking for, see `array_assign`.
static inline void _array__assign(Array *self, const Array *other, size_t element_size)
{
_array__reserve(self, element_size, other->size);
self->size = other->size;
memcpy(self->contents, other->contents, self->size * element_size);
}
/// This is not what you're looking for, see `array_swap`.
static inline void _array__swap(Array *self, Array *other)
{
Array swap = *other;
*other = *self;
*self = swap;
}
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
static inline void _array__grow(Array *self, uint32_t count, size_t element_size)
{
uint32_t new_size = self->size + count;
if (new_size > self->capacity)
{
uint32_t new_capacity = self->capacity * 2;
if (new_capacity < 8)
new_capacity = 8;
if (new_capacity < new_size)
new_capacity = new_size;
_array__reserve(self, element_size, new_capacity);
}
}
/// This is not what you're looking for, see `array_splice`.
static inline void _array__splice(Array *self, size_t element_size, uint32_t index, uint32_t old_count, uint32_t new_count,
const void *elements)
{
uint32_t new_size = self->size + new_count - old_count;
uint32_t old_end = index + old_count;
uint32_t new_end = index + new_count;
assert(old_end <= self->size);
_array__reserve(self, element_size, new_size);
char *contents = (char *)self->contents;
if (self->size > old_end)
{
memmove(contents + new_end * element_size, contents + old_end * element_size, (self->size - old_end) * element_size);
}
if (new_count > 0)
{
if (elements)
{
memcpy((contents + index * element_size), elements, new_count * element_size);
}
else
{
memset((contents + index * element_size), 0, new_count * element_size);
}
}
self->size += new_count - old_count;
}
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
do \
{ \
*(_index) = start; \
*(_exists) = false; \
uint32_t size = (self)->size - *(_index); \
if (size == 0) \
break; \
int comparison; \
while (size > 1) \
{ \
uint32_t half_size = size / 2; \
uint32_t mid_index = *(_index) + half_size; \
comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
if (comparison <= 0) \
*(_index) = mid_index; \
size -= half_size; \
} \
comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
if (comparison == 0) \
*(_exists) = true; \
else if (comparison < 0) \
*(_index) += 1; \
} while (0)
/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
/// parameter by reference in order to work with the generic sorting function above.
#define _compare_int(a, b) ((int)*(a) - (int)(b))
#ifdef _MSC_VER
# pragma warning(default : 4101)
#elif defined(__GNUC__) || defined(__clang__)
# pragma GCC diagnostic pop
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ARRAY_H_

35
parser/nsrc/atomic.h
View file

@ -1,35 +0,0 @@
#ifndef TREE_SITTER_ATOMIC_H_
#define TREE_SITTER_ATOMIC_H_
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
static inline size_t atomic_load(const volatile size_t *p)
{
#ifdef __ATOMIC_RELAXED
return __atomic_load_n(p, __ATOMIC_RELAXED);
#else
return __sync_fetch_and_add((volatile size_t *)p, 0);
#endif
}
static inline uint32_t atomic_inc(volatile uint32_t *p)
{
#ifdef __ATOMIC_RELAXED
return __atomic_add_fetch(p, 1U, __ATOMIC_SEQ_CST);
#else
return __sync_add_and_fetch(p, 1U);
#endif
}
static inline uint32_t atomic_dec(volatile uint32_t *p)
{
#ifdef __ATOMIC_RELAXED
return __atomic_sub_fetch(p, 1U, __ATOMIC_SEQ_CST);
#else
return __sync_sub_and_fetch(p, 1U);
#endif
}
#endif // TREE_SITTER_ATOMIC_H_

41
parser/nsrc/clock.h
View file

@ -1,41 +0,0 @@
#ifndef TREE_SITTER_CLOCK_H_
#define TREE_SITTER_CLOCK_H_
#include <stdbool.h>
#include <stdint.h>
typedef uint64_t TSDuration;
#include <time.h>
typedef uint64_t TSClock;
static inline TSDuration duration_from_micros(uint64_t micros) {
return micros * (uint64_t)CLOCKS_PER_SEC / 1000000;
}
static inline uint64_t duration_to_micros(TSDuration self) {
return self * 1000000 / (uint64_t)CLOCKS_PER_SEC;
}
static inline TSClock clock_null(void) {
return 0;
}
static inline TSClock clock_now(void) {
return (uint64_t)clock();
}
static inline TSClock clock_after(TSClock base, TSDuration duration) {
return base + duration;
}
static inline bool clock_is_null(TSClock self) {
return !self;
}
static inline bool clock_is_gt(TSClock self, TSClock other) {
return self > other;
}
#endif // TREE_SITTER_CLOCK_H_

99
parser/nsrc/create_language.c
View file

@ -1,99 +0,0 @@
/* ************************************************************************** */
/* */
/* ::: :::::::: */
/* create_language.c :+: :+: :+: */
/* +:+ +:+ +:+ */
/* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */
/* +#+#+#+#+#+ +#+ */
/* Created: 2024/04/25 16:13:52 by maiboyer #+# #+# */
/* Updated: 2024/06/24 00:35:41 by maiboyer ### ########.fr */
/* */
/* ************************************************************************** */
#include "../static/headers/constants.h"
#include "../static/headers/symbols.h"
#include "./parser.h"
bool lex_keywords_main(TSLexer *lexer, TSStateId state);
bool lex_normal_main(TSLexer *lexer, TSStateId state);
bool tree_sitter_sh_external_scanner_scan(void *ctx, TSLexer *lexer, const bool *ret);
void *create_external_scanner_states(void);
void *create_field_names(void);
void *create_symbols_names(void);
void *create_field_map_entries(void);
void *create_field_map_slices(void);
void *create_lex_modes(void);
void *create_parse_actions_entries(void);
void *create_primary_state_ids(void);
void *create_alias_sequences(void);
void *create_external_scanner_symbol_map(void);
void *create_non_terminal_alias_map(void);
void *create_unique_symbols_map(void);
void *create_symbols_metadata(void);
void *create_parse_table(void);
void *create_small_parse_table(void);
void *create_small_parse_table_map(void);
uint32_t tree_sitter_sh_external_scanner_serialize(void *ctx, char *s);
void tree_sitter_sh_external_scanner_deserialize(void *ctx, const char *s, uint32_t val);
void tree_sitter_sh_external_scanner_destroy(void *ctx);
void *tree_sitter_sh_external_scanner_create(void);
static struct ExternalScannerDefinition init_scanner(void)
{
return ((struct ExternalScannerDefinition){
create_external_scanner_states(),
create_external_scanner_symbol_map(),
tree_sitter_sh_external_scanner_create,
tree_sitter_sh_external_scanner_destroy,
tree_sitter_sh_external_scanner_scan,
tree_sitter_sh_external_scanner_serialize,
tree_sitter_sh_external_scanner_deserialize,
});
}
static void init_language(TSLanguage *language)
{
language->parse_table = create_parse_table();
language->small_parse_table = create_small_parse_table();
language->small_parse_table_map = create_small_parse_table_map();
language->parse_actions = create_parse_actions_entries();
language->symbol_names = create_symbols_names();
language->field_names = create_field_names();
language->field_map_slices = create_field_map_slices();
language->field_map_entries = create_field_map_entries();
language->symbol_metadata = create_symbols_metadata();
language->public_symbol_map = create_unique_symbols_map();
language->alias_map = create_non_terminal_alias_map();
language->alias_sequences = create_alias_sequences();
language->lex_modes = create_lex_modes();
language->primary_state_ids = create_primary_state_ids();
language->lex_fn = lex_normal_main;
language->keyword_lex_fn = lex_keywords_main;
language->keyword_capture_token = sym_word;
language->external_scanner = init_scanner();
}
const TSLanguage *tree_sitter_bash(void)
{
static bool init = false;
static TSLanguage language = {
.version = LANGUAGE_VERSION,
.symbol_count = SYMBOL_COUNT,
.alias_count = ALIAS_COUNT,
.token_count = TOKEN_COUNT,
.external_token_count = EXTERNAL_TOKEN_COUNT,
.state_count = STATE_COUNT,
.large_state_count = LARGE_STATE_COUNT,
.production_id_count = PRODUCTION_ID_COUNT,
.field_count = FIELD_COUNT,
.max_alias_sequence_length = MAX_ALIAS_SEQUENCE_LENGTH,
};
if (!init)
{
init_language(&language);
init = true;
}
return ((TSLanguage *)&language);
}

11
parser/nsrc/error_costs.h
View file

@ -1,11 +0,0 @@
#ifndef TREE_SITTER_ERROR_COSTS_H_
#define TREE_SITTER_ERROR_COSTS_H_
#define ERROR_STATE 0
#define ERROR_COST_PER_RECOVERY 500
#define ERROR_COST_PER_MISSING_TREE 110
#define ERROR_COST_PER_SKIPPED_TREE 100
#define ERROR_COST_PER_SKIPPED_LINE 30
#define ERROR_COST_PER_SKIPPED_CHAR 1
#endif

547
parser/nsrc/get_changed_ranges.c
View file

@ -1,547 +0,0 @@
#include "./get_changed_ranges.h"
#include "./error_costs.h"
#include "./language.h"
#include "./subtree.h"
#include "./tree_cursor.h"
#include <assert.h>
// #define DEBUG_GET_CHANGED_RANGES
static void ts_range_array_add(TSRangeArray *self, Length start, Length end)
{
if (self->size > 0)
{
TSRange *last_range = array_back(self);
if (start.bytes <= last_range->end_byte)
{
last_range->end_byte = end.bytes;
last_range->end_point = end.extent;
return;
}
}
if (start.bytes < end.bytes)
{
TSRange range = {start.extent, end.extent, start.bytes, end.bytes};
array_push(self, range);
}
}
bool ts_range_array_intersects(const TSRangeArray *self, unsigned start_index, uint32_t start_byte, uint32_t end_byte)
{
for (unsigned i = start_index; i < self->size; i++)
{
TSRange *range = &self->contents[i];
if (range->end_byte > start_byte)
{
if (range->start_byte >= end_byte)
break;
return true;
}
}
return false;
}
void ts_range_array_get_changed_ranges(const TSRange *old_ranges, unsigned old_range_count, const TSRange *new_ranges,
unsigned new_range_count, TSRangeArray *differences)
{
unsigned new_index = 0;
unsigned old_index = 0;
Length current_position = length_zero();
bool in_old_range = false;
bool in_new_range = false;
while (old_index < old_range_count || new_index < new_range_count)
{
const TSRange *old_range = &old_ranges[old_index];
const TSRange *new_range = &new_ranges[new_index];
Length next_old_position;
if (in_old_range)
{
next_old_position = (Length){old_range->end_byte, old_range->end_point};
}
else if (old_index < old_range_count)
{
next_old_position = (Length){old_range->start_byte, old_range->start_point};
}
else
{
next_old_position = LENGTH_MAX;
}
Length next_new_position;
if (in_new_range)
{
next_new_position = (Length){new_range->end_byte, new_range->end_point};
}
else if (new_index < new_range_count)
{
next_new_position = (Length){new_range->start_byte, new_range->start_point};
}
else
{
next_new_position = LENGTH_MAX;
}
if (next_old_position.bytes < next_new_position.bytes)
{
if (in_old_range != in_new_range)
{
ts_range_array_add(differences, current_position, next_old_position);
}
if (in_old_range)
old_index++;
current_position = next_old_position;
in_old_range = !in_old_range;
}
else if (next_new_position.bytes < next_old_position.bytes)
{
if (in_old_range != in_new_range)
{
ts_range_array_add(differences, current_position, next_new_position);
}
if (in_new_range)
new_index++;
current_position = next_new_position;
in_new_range = !in_new_range;
}
else
{
if (in_old_range != in_new_range)
{
ts_range_array_add(differences, current_position, next_new_position);
}
if (in_old_range)
old_index++;
if (in_new_range)
new_index++;
in_old_range = !in_old_range;
in_new_range = !in_new_range;
current_position = next_new_position;
}
}
}
typedef struct Iterator
{
TreeCursor cursor;
const TSLanguage *language;
unsigned visible_depth;
bool in_padding;
} Iterator;
static Iterator iterator_new(TreeCursor *cursor, const Subtree *tree, const TSLanguage *language)
{
array_clear(&cursor->stack);
array_push(&cursor->stack, ((TreeCursorEntry){
.subtree = tree,
.position = length_zero(),
.child_index = 0,
.structural_child_index = 0,
}));
return (Iterator){
.cursor = *cursor,
.language = language,
.visible_depth = 1,
.in_padding = false,
};
}
static bool iterator_done(Iterator *self)
{
return self->cursor.stack.size == 0;
}
static Length iterator_start_position(Iterator *self)
{
TreeCursorEntry entry = *array_back(&self->cursor.stack);
if (self->in_padding)
{
return entry.position;
}
else
{
return length_add(entry.position, ts_subtree_padding(*entry.subtree));
}
}
static Length iterator_end_position(Iterator *self)
{
TreeCursorEntry entry = *array_back(&self->cursor.stack);
Length result = length_add(entry.position, ts_subtree_padding(*entry.subtree));
if (self->in_padding)
{
return result;
}
else
{
return length_add(result, ts_subtree_size(*entry.subtree));
}
}
static bool iterator_tree_is_visible(const Iterator *self)
{
TreeCursorEntry entry = *array_back(&self->cursor.stack);
if (ts_subtree_visible(*entry.subtree))
return true;
if (self->cursor.stack.size > 1)
{
Subtree parent = *self->cursor.stack.contents[self->cursor.stack.size - 2].subtree;
return ts_language_alias_at(self->language, parent.ptr->inner.non_terminal.production_id, entry.structural_child_index) != 0;
}
return false;
}
static void iterator_get_visible_state(const Iterator *self, Subtree *tree, TSSymbol *alias_symbol, uint32_t *start_byte)
{
uint32_t i = self->cursor.stack.size - 1;
if (self->in_padding)
{
if (i == 0)
return;
i--;
}
for (; i + 1 > 0; i--)
{
TreeCursorEntry entry = self->cursor.stack.contents[i];
if (i > 0)
{
const Subtree *parent = self->cursor.stack.contents[i - 1].subtree;
*alias_symbol = ts_language_alias_at(self->language, parent->ptr->inner.non_terminal.production_id, entry.structural_child_index);
}
if (ts_subtree_visible(*entry.subtree) || *alias_symbol)
{
*tree = *entry.subtree;
*start_byte = entry.position.bytes;
break;
}
}
}
static void iterator_ascend(Iterator *self)
{
if (iterator_done(self))
return;
if (iterator_tree_is_visible(self) && !self->in_padding)
self->visible_depth--;
if (array_back(&self->cursor.stack)->child_index > 0)
self->in_padding = false;
self->cursor.stack.size--;
}
static bool iterator_descend(Iterator *self, uint32_t goal_position)
{
if (self->in_padding)
return false;
bool did_descend = false;
do
{
did_descend = false;
TreeCursorEntry entry = *array_back(&self->cursor.stack);
Length position = entry.position;
uint32_t structural_child_index = 0;
for (uint32_t i = 0, n = ts_subtree_child_count(*entry.subtree); i < n; i++)
{
const Subtree *child = &ts_subtree_children(*entry.subtree)[i];
Length child_left = length_add(position, ts_subtree_padding(*child));
Length child_right = length_add(child_left, ts_subtree_size(*child));
if (child_right.bytes > goal_position)
{
array_push(&self->cursor.stack, ((TreeCursorEntry){
.subtree = child,
.position = position,
.child_index = i,
.structural_child_index = structural_child_index,
}));
if (iterator_tree_is_visible(self))
{
if (child_left.bytes > goal_position)
{
self->in_padding = true;
}
else
{
self->visible_depth++;
}
return true;
}
did_descend = true;
break;
}
position = child_right;
if (!ts_subtree_extra(*child))
structural_child_index++;
}
} while (did_descend);
return false;
}
static void iterator_advance(Iterator *self)
{
if (self->in_padding)
{
self->in_padding = false;
if (iterator_tree_is_visible(self))
{
self->visible_depth++;
}
else
{
iterator_descend(self, 0);
}
return;
}
for (;;)
{
if (iterator_tree_is_visible(self))
self->visible_depth--;
TreeCursorEntry entry = array_pop(&self->cursor.stack);
if (iterator_done(self))
return;
const Subtree *parent = array_back(&self->cursor.stack)->subtree;
uint32_t child_index = entry.child_index + 1;
if (ts_subtree_child_count(*parent) > child_index)
{
Length position = length_add(entry.position, ts_subtree_total_size(*entry.subtree));
uint32_t structural_child_index = entry.structural_child_index;
if (!ts_subtree_extra(*entry.subtree))
structural_child_index++;
const Subtree *next_child = &ts_subtree_children(*parent)[child_index];
array_push(&self->cursor.stack, ((TreeCursorEntry){
.subtree = next_child,
.position = position,
.child_index = child_index,
.structural_child_index = structural_child_index,
}));
if (iterator_tree_is_visible(self))
{
if (ts_subtree_padding(*next_child).bytes > 0)
{
self->in_padding = true;
}
else
{
self->visible_depth++;
}
}
else
{
iterator_descend(self, 0);
}
break;
}
}
}
typedef enum IteratorComparison
{
IteratorDiffers,
IteratorMayDiffer,
IteratorMatches,
} IteratorComparison;
static IteratorComparison iterator_compare(const Iterator *old_iter, const Iterator *new_iter)
{
Subtree old_tree = NULL_SUBTREE;
Subtree new_tree = NULL_SUBTREE;
uint32_t old_start = 0;
uint32_t new_start = 0;
TSSymbol old_alias_symbol = 0;
TSSymbol new_alias_symbol = 0;
iterator_get_visible_state(old_iter, &old_tree, &old_alias_symbol, &old_start);
iterator_get_visible_state(new_iter, &new_tree, &new_alias_symbol, &new_start);
if (!old_tree.ptr && !new_tree.ptr)
return IteratorMatches;
if (!old_tree.ptr || !new_tree.ptr)
return IteratorDiffers;
if (old_alias_symbol == new_alias_symbol && ts_subtree_symbol(old_tree) == ts_subtree_symbol(new_tree))
{
if (old_start == new_start && !ts_subtree_has_changes(old_tree) && ts_subtree_symbol(old_tree) != ts_builtin_sym_error &&
ts_subtree_size(old_tree).bytes == ts_subtree_size(new_tree).bytes && ts_subtree_parse_state(old_tree) != TS_TREE_STATE_NONE &&
ts_subtree_parse_state(new_tree) != TS_TREE_STATE_NONE &&
(ts_subtree_parse_state(old_tree) == ERROR_STATE) == (ts_subtree_parse_state(new_tree) == ERROR_STATE))
{
return IteratorMatches;
}
else
{
return IteratorMayDiffer;
}
}
return IteratorDiffers;
}
#ifdef DEBUG_GET_CHANGED_RANGES
static inline void iterator_print_state(Iterator *self)
{
TreeCursorEntry entry = *array_back(&self->cursor.stack);
TSPoint start = iterator_start_position(self).extent;
TSPoint end = iterator_end_position(self).extent;
const char *name = ts_language_symbol_name(self->language, ts_subtree_symbol(*entry.subtree));
printf("(%-25s %s\t depth:%u [%u, %u] - [%u, %u])", name, self->in_padding ? "(p)" : " ", self->visible_depth, start.row + 1,
start.column, end.row + 1, end.column);
}
#endif
unsigned ts_subtree_get_changed_ranges(const Subtree *old_tree, const Subtree *new_tree, TreeCursor *cursor1, TreeCursor *cursor2,
const TSLanguage *language, const TSRangeArray *included_range_differences, TSRange **ranges)
{
TSRangeArray results = array_new();
Iterator old_iter = iterator_new(cursor1, old_tree, language);
Iterator new_iter = iterator_new(cursor2, new_tree, language);
unsigned included_range_difference_index = 0;
Length position = iterator_start_position(&old_iter);
Length next_position = iterator_start_position(&new_iter);
if (position.bytes < next_position.bytes)
{
ts_range_array_add(&results, position, next_position);
position = next_position;
}
else if (position.bytes > next_position.bytes)
{
ts_range_array_add(&results, next_position, position);
next_position = position;
}
do
{
#ifdef DEBUG_GET_CHANGED_RANGES
printf("At [%-2u, %-2u] Compare ", position.extent.row + 1, position.extent.column);
iterator_print_state(&old_iter);
printf("\tvs\t");
iterator_print_state(&new_iter);
puts("");
#endif
// Compare the old and new subtrees.
IteratorComparison comparison = iterator_compare(&old_iter, &new_iter);
// Even if the two subtrees appear to be identical, they could differ
// internally if they contain a range of text that was previously
// excluded from the parse, and is now included, or vice-versa.
if (comparison == IteratorMatches && ts_range_array_intersects(included_range_differences, included_range_difference_index,
position.bytes, iterator_end_position(&old_iter).bytes))
{
comparison = IteratorMayDiffer;
}
bool is_changed = false;
switch (comparison)
{
// If the subtrees are definitely identical, move to the end
// of both subtrees.
case IteratorMatches:
next_position = iterator_end_position(&old_iter);
break;
// If the subtrees might differ internally, descend into both
// subtrees, finding the first child that spans the current position.
case IteratorMayDiffer:
if (iterator_descend(&old_iter, position.bytes))
{
if (!iterator_descend(&new_iter, position.bytes))
{
is_changed = true;
next_position = iterator_end_position(&old_iter);
}
}
else if (iterator_descend(&new_iter, position.bytes))
{
is_changed = true;
next_position = iterator_end_position(&new_iter);
}
else
{
next_position = length_min(iterator_end_position(&old_iter), iterator_end_position(&new_iter));
}
break;
// If the subtrees are different, record a change and then move
// to the end of both subtrees.
case IteratorDiffers:
is_changed = true;
next_position = length_min(iterator_end_position(&old_iter), iterator_end_position(&new_iter));
break;
}
// Ensure that both iterators are caught up to the current position.
while (!iterator_done(&old_iter) && iterator_end_position(&old_iter).bytes <= next_position.bytes)
iterator_advance(&old_iter);
while (!iterator_done(&new_iter) && iterator_end_position(&new_iter).bytes <= next_position.bytes)
iterator_advance(&new_iter);
// Ensure that both iterators are at the same depth in the tree.
while (old_iter.visible_depth > new_iter.visible_depth)
{
iterator_ascend(&old_iter);
}
while (new_iter.visible_depth > old_iter.visible_depth)
{
iterator_ascend(&new_iter);
}
if (is_changed)
{
#ifdef DEBUG_GET_CHANGED_RANGES
printf(" change: [[%u, %u] - [%u, %u]]\n", position.extent.row + 1, position.extent.column, next_position.extent.row + 1,
next_position.extent.column);
#endif
ts_range_array_add(&results, position, next_position);
}
position = next_position;
// Keep track of the current position in the included range differences
// array in order to avoid scanning the entire array on each iteration.
while (included_range_difference_index < included_range_differences->size)
{
const TSRange *range = &included_range_differences->contents[included_range_difference_index];
if (range->end_byte <= position.bytes)
{
included_range_difference_index++;
}
else
{
break;
}
}
} while (!iterator_done(&old_iter) && !iterator_done(&new_iter));
Length old_size = ts_subtree_total_size(*old_tree);
Length new_size = ts_subtree_total_size(*new_tree);
if (old_size.bytes < new_size.bytes)
{
ts_range_array_add(&results, old_size, new_size);
}
else if (new_size.bytes < old_size.bytes)
{
ts_range_array_add(&results, new_size, old_size);
}
*cursor1 = old_iter.cursor;
*cursor2 = new_iter.cursor;
*ranges = results.contents;
return results.size;
}

24
parser/nsrc/get_changed_ranges.h
View file

@ -1,24 +0,0 @@
#ifndef TREE_SITTER_GET_CHANGED_RANGES_H_
#define TREE_SITTER_GET_CHANGED_RANGES_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include "./subtree.h"
#include "./tree_cursor.h"
typedef Array(TSRange) TSRangeArray;
void ts_range_array_get_changed_ranges(const TSRange *old_ranges, unsigned old_range_count, const TSRange *new_ranges, unsigned new_range_count, TSRangeArray *differences);
bool ts_range_array_intersects(const TSRangeArray *self, unsigned start_index, uint32_t start_byte, uint32_t end_byte);
unsigned ts_subtree_get_changed_ranges(const Subtree *old_tree, const Subtree *new_tree, TreeCursor *cursor1, TreeCursor *cursor2, const TSLanguage *language, const TSRangeArray *included_range_differences, TSRange **ranges);
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_GET_CHANGED_RANGES_H_

20
parser/nsrc/host.h
View file

@ -1,20 +0,0 @@
// Determine endian and pointer size based on known defines.
// TS_BIG_ENDIAN and TS_PTR_SIZE can be set as -D compiler arguments
// to override this.
#if !defined(TS_BIG_ENDIAN)
# if (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) || (defined(__APPLE_CC__) && (defined(__ppc__) || defined(__ppc64__)))
# define TS_BIG_ENDIAN 1
# else
# define TS_BIG_ENDIAN 0
# endif
#endif
#if !defined(TS_PTR_SIZE)
# if UINTPTR_MAX == 0xFFFFFFFF
# define TS_PTR_SIZE 32
# else
# define TS_PTR_SIZE 64
# endif
#endif

215
parser/nsrc/language.c
View file

@ -1,215 +0,0 @@
#include "./language.h"
#include "./api.h"
#include <string.h>
const TSLanguage *ts_language_copy(const TSLanguage *self) {
return self;
}
void ts_language_delete(const TSLanguage *self) {
(void)(self);
}
uint32_t ts_language_symbol_count(const TSLanguage *self) {
return self->symbol_count + self->alias_count;
}
uint32_t ts_language_state_count(const TSLanguage *self) {
return self->state_count;
}
uint32_t ts_language_version(const TSLanguage *self) {
return self->version;
}
uint32_t ts_language_field_count(const TSLanguage *self) {
return self->field_count;
}
void ts_language_table_entry(
const TSLanguage *self,
TSStateId state,
TSSymbol symbol,
TableEntry *result
) {
if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
result->action_count = 0;
result->is_reusable = false;
result->actions = NULL;
} else {
assert(symbol < self->token_count);
uint32_t action_index = ts_language_lookup(self, state, symbol);
const TSParseActionEntry *entry = &self->parse_actions[action_index];
result->action_count = entry->entry.count;
result->is_reusable = entry->entry.reusable;
result->actions = (const TSParseAction *)(entry + 1);
}
}
TSSymbolMetadata ts_language_symbol_metadata(
const TSLanguage *self,
TSSymbol symbol
) {
if (symbol == ts_builtin_sym_error) {
return (TSSymbolMetadata) {.visible = true, .named = true};
} else if (symbol == ts_builtin_sym_error_repeat) {
return (TSSymbolMetadata) {.visible = false, .named = false};
} else {
return self->symbol_metadata[symbol];
}
}
TSSymbol ts_language_public_symbol(
const TSLanguage *self,
TSSymbol symbol
) {
if (symbol == ts_builtin_sym_error) return symbol;
return self->public_symbol_map[symbol];
}
TSStateId ts_language_next_state(
const TSLanguage *self,
TSStateId state,
TSSymbol symbol
) {
if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
return 0;
} else if (symbol < self->token_count) {
uint32_t count;
const TSParseAction *actions = ts_language_actions(self, state, symbol, &count);
if (count > 0) {
TSParseAction action = actions[count - 1];
if (action.type == TSParseActionTypeShift) {
return action.shift.extra ? state : action.shift.state;
}
}
return 0;
} else {
return ts_language_lookup(self, state, symbol);
}
}
const char *ts_language_symbol_name(
const TSLanguage *self,
TSSymbol symbol
) {
if (symbol == ts_builtin_sym_error) {
return "ERROR";
} else if (symbol == ts_builtin_sym_error_repeat) {
return "_ERROR";
} else if (symbol < ts_language_symbol_count(self)) {
return self->symbol_names[symbol];
} else {
return NULL;
}
}
TSSymbol ts_language_symbol_for_name(
const TSLanguage *self,
const char *string,
uint32_t length,
bool is_named
) {
if (!strncmp(string, "ERROR", length)) return ts_builtin_sym_error;
uint16_t count = (uint16_t)ts_language_symbol_count(self);
for (TSSymbol i = 0; i < count; i++) {
TSSymbolMetadata metadata = ts_language_symbol_metadata(self, i);
if ((!metadata.visible && !metadata.supertype) || metadata.named != is_named) continue;
const char *symbol_name = self->symbol_names[i];
if (!strncmp(symbol_name, string, length) && !symbol_name[length]) {
return self->public_symbol_map[i];
}
}
return 0;
}
TSSymbolType ts_language_symbol_type(
const TSLanguage *self,
TSSymbol symbol
) {
TSSymbolMetadata metadata = ts_language_symbol_metadata(self, symbol);
if (metadata.named && metadata.visible) {
return TSSymbolTypeRegular;
} else if (metadata.visible) {
return TSSymbolTypeAnonymous;
} else {
return TSSymbolTypeAuxiliary;
}
}
const char *ts_language_field_name_for_id(
const TSLanguage *self,
TSFieldId id
) {
uint32_t count = ts_language_field_count(self);
if (count && id <= count) {
return self->field_names[id];
} else {
return NULL;
}
}
TSFieldId ts_language_field_id_for_name(
const TSLanguage *self,
const char *name,
uint32_t name_length
) {
uint16_t count = (uint16_t)ts_language_field_count(self);
for (TSSymbol i = 1; i < count + 1; i++) {
switch (strncmp(name, self->field_names[i], name_length)) {
case 0:
if (self->field_names[i][name_length] == 0) return i;
break;
case -1:
return 0;
default:
break;
}
}
return 0;
}
TSLookaheadIterator *ts_lookahead_iterator_new(const TSLanguage *self, TSStateId state) {
if (state >= self->state_count) return NULL;
LookaheadIterator *iterator = ts_malloc(sizeof(LookaheadIterator));
*iterator = ts_language_lookaheads(self, state);
return (TSLookaheadIterator *)iterator;
}
void ts_lookahead_iterator_delete(TSLookaheadIterator *self) {
ts_free(self);
}
bool ts_lookahead_iterator_reset_state(TSLookaheadIterator * self, TSStateId state) {
LookaheadIterator *iterator = (LookaheadIterator *)self;
if (state >= iterator->language->state_count) return false;
*iterator = ts_language_lookaheads(iterator->language, state);
return true;
}
const TSLanguage *ts_lookahead_iterator_language(const TSLookaheadIterator *self) {
const LookaheadIterator *iterator = (const LookaheadIterator *)self;
return iterator->language;
}
bool ts_lookahead_iterator_reset(TSLookaheadIterator *self, const TSLanguage *language, TSStateId state) {
if (state >= language->state_count) return false;
LookaheadIterator *iterator = (LookaheadIterator *)self;
*iterator = ts_language_lookaheads(language, state);
return true;
}
bool ts_lookahead_iterator_next(TSLookaheadIterator *self) {
LookaheadIterator *iterator = (LookaheadIterator *)self;
return ts_lookahead_iterator__next(iterator);
}
TSSymbol ts_lookahead_iterator_current_symbol(const TSLookaheadIterator *self) {
const LookaheadIterator *iterator = (const LookaheadIterator *)self;
return iterator->symbol;
}
const char *ts_lookahead_iterator_current_symbol_name(const TSLookaheadIterator *self) {
const LookaheadIterator *iterator = (const LookaheadIterator *)self;
return ts_language_symbol_name(iterator->language, iterator->symbol);
}

299
parser/nsrc/language.h
View file

@ -1,299 +0,0 @@
#ifndef TREE_SITTER_LANGUAGE_H_
#define TREE_SITTER_LANGUAGE_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include "./parser.h"
#include "./subtree.h"
#define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1)
#define LANGUAGE_VERSION_WITH_PRIMARY_STATES 14
#define LANGUAGE_VERSION_USABLE_VIA_WASM 13
typedef struct TableEntry
{
const TSParseAction *actions;
uint32_t action_count;
bool is_reusable;
} TableEntry;
typedef struct LookaheadIterator
{
const TSLanguage *language;
const uint16_t *data;
const uint16_t *group_end;
TSStateId state;
uint16_t table_value;
uint16_t section_index;
uint16_t group_count;
bool is_small_state;
const TSParseAction *actions;
TSSymbol symbol;
TSStateId next_state;
uint16_t action_count;
} LookaheadIterator;
void ts_language_table_entry(const TSLanguage *, TSStateId, TSSymbol, TableEntry *);
TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *, TSSymbol);
TSSymbol ts_language_public_symbol(const TSLanguage *, TSSymbol);
TSStateId ts_language_next_state(const TSLanguage *self, TSStateId state, TSSymbol symbol);
static inline bool ts_language_is_symbol_external(const TSLanguage *self, TSSymbol symbol)
{
return 0 < symbol && symbol < self->external_token_count + 1;
}
static inline const TSParseAction *ts_language_actions(const TSLanguage *self, TSStateId state, TSSymbol symbol, uint32_t *count)
{
TableEntry entry;
ts_language_table_entry(self, state, symbol, &entry);
*count = entry.action_count;
return entry.actions;
}
static inline bool ts_language_has_reduce_action(const TSLanguage *self, TSStateId state, TSSymbol symbol)
{
TableEntry entry;
ts_language_table_entry(self, state, symbol, &entry);
return entry.action_count > 0 && entry.actions[0].type == TSParseActionTypeReduce;
}
// Lookup the table value for a given symbol and state.
//
// For non-terminal symbols, the table value represents a successor state.
// For terminal symbols, it represents an index in the actions table.
// For 'large' parse states, this is a direct lookup. For 'small' parse
// states, this requires searching through the symbol groups to find
// the given symbol.
static inline uint16_t ts_language_lookup(const TSLanguage *self, TSStateId state, TSSymbol symbol)
{
if (state >= self->large_state_count)
{
uint32_t index = self->small_parse_table_map[state - self->large_state_count];
const uint16_t *data = &self->small_parse_table[index];
uint16_t group_count = *(data++);
for (unsigned i = 0; i < group_count; i++)
{
uint16_t section_value = *(data++);
uint16_t symbol_count = *(data++);
for (unsigned j = 0; j < symbol_count; j++)
{
if (*(data++) == symbol)
return section_value;
}
}
return 0;
}
else
{
return self->parse_table[state * self->symbol_count + symbol];
}
}
static inline bool ts_language_has_actions(const TSLanguage *self, TSStateId state, TSSymbol symbol)
{
return ts_language_lookup(self, state, symbol) != 0;
}
// Iterate over all of the symbols that are valid in the given state.
//
// For 'large' parse states, this just requires iterating through
// all possible symbols and checking the parse table for each one.
// For 'small' parse states, this exploits the structure of the
// table to only visit the valid symbols.
static inline LookaheadIterator ts_language_lookaheads(const TSLanguage *self, TSStateId state)
{
bool is_small_state = state >= self->large_state_count;
const uint16_t *data;
const uint16_t *group_end = NULL;
uint16_t group_count = 0;
if (is_small_state)
{
uint32_t index = self->small_parse_table_map[state - self->large_state_count];
data = &self->small_parse_table[index];
group_end = data + 1;
group_count = *data;
}
else
{
data = &self->parse_table[state * self->symbol_count] - 1;
}
return (LookaheadIterator){
.language = self,
.data = data,
.group_end = group_end,
.group_count = group_count,
.is_small_state = is_small_state,
.symbol = UINT16_MAX,
.next_state = 0,
};
}
static inline bool ts_lookahead_iterator__next(LookaheadIterator *self)
{
// For small parse states, valid symbols are listed explicitly,
// grouped by their value. There's no need to look up the actions
// again until moving to the next group.
if (self->is_small_state)
{
self->data++;
if (self->data == self->group_end)
{
if (self->group_count == 0)
return false;
self->group_count--;
self->table_value = *(self->data++);
unsigned symbol_count = *(self->data++);
self->group_end = self->data + symbol_count;
self->symbol = *self->data;
}
else
{
self->symbol = *self->data;
return true;
}
}
// For large parse states, iterate through every symbol until one
// is found that has valid actions.
else
{
do
{
self->data++;
self->symbol++;
if (self->symbol >= self->language->symbol_count)
return false;
self->table_value = *self->data;
} while (!self->table_value);
}
// Depending on if the symbols is terminal or non-terminal, the table value either
// represents a list of actions or a successor state.
if (self->symbol < self->language->token_count)
{
const TSParseActionEntry *entry = &self->language->parse_actions[self->table_value];
self->action_count = entry->entry.count;
self->actions = (const TSParseAction *)(entry + 1);
self->next_state = 0;
}
else
{
self->action_count = 0;
self->next_state = self->table_value;
}
return true;
}
// Whether the state is a "primary state". If this returns false, it indicates that there exists
// another state that behaves identically to this one with respect to query analysis.
static inline bool ts_language_state_is_primary(const TSLanguage *self, TSStateId state)
{
if (self->version >= LANGUAGE_VERSION_WITH_PRIMARY_STATES)
{
return state == self->primary_state_ids[state];
}
else
{
return true;
}
}
static inline const bool *ts_language_enabled_external_tokens(const TSLanguage *self, unsigned external_scanner_state)
{
if (external_scanner_state == 0)
{
return NULL;
}
else
{
return self->external_scanner.states + self->external_token_count * external_scanner_state;
}
}
static inline const TSSymbol *ts_language_alias_sequence(const TSLanguage *self, uint32_t production_id)
{
return production_id ? &self->alias_sequences[production_id * self->max_alias_sequence_length] : NULL;
}
static inline TSSymbol ts_language_alias_at(const TSLanguage *self, uint32_t production_id, uint32_t child_index)
{
return production_id ? self->alias_sequences[production_id * self->max_alias_sequence_length + child_index] : 0;
}
static inline void ts_language_field_map(const TSLanguage *self, uint32_t production_id, const TSFieldMapEntry **start,
const TSFieldMapEntry **end)
{
if (self->field_count == 0)
{
*start = NULL;
*end = NULL;
return;
}
TSFieldMapSlice slice = self->field_map_slices[production_id];
*start = &self->field_map_entries[slice.index];
*end = &self->field_map_entries[slice.index] + slice.length;
}
static inline void ts_language_aliases_for_symbol(const TSLanguage *self, TSSymbol original_symbol, const TSSymbol **start,
const TSSymbol **end)
{
*start = &self->public_symbol_map[original_symbol];
*end = *start + 1;
unsigned idx = 0;
for (;;)
{
TSSymbol symbol = self->alias_map[idx++];
if (symbol == 0 || symbol > original_symbol)
break;
uint16_t count = self->alias_map[idx++];
if (symbol == original_symbol)
{
*start = &self->alias_map[idx];
*end = &self->alias_map[idx + count];
break;
}
idx += count;
}
}
static inline void ts_language_write_symbol_as_dot_string(const TSLanguage *self, FILE *f, TSSymbol symbol)
{
const char *name = ts_language_symbol_name(self, symbol);
for (const char *chr = name; *chr; chr++)
{
switch (*chr)
{
case '"':
case '\\':
fputc('\\', f);
fputc(*chr, f);
break;
case '\n':
fputs("\\n", f);
break;
case '\t':
fputs("\\t", f);
break;
default:
fputc(*chr, f);
break;
}
}
}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_LANGUAGE_H_

62
parser/nsrc/length.h
View file

@ -1,62 +0,0 @@
#ifndef TREE_SITTER_LENGTH_H_
#define TREE_SITTER_LENGTH_H_
#include "./api.h"
#include "./point.h"
#include <stdbool.h>
#include <stdlib.h>
typedef struct Length
{
uint32_t bytes;
TSPoint extent;
} Length;
static const Length LENGTH_UNDEFINED = {0, {0, 1}};
static const Length LENGTH_MAX = {UINT32_MAX, {UINT32_MAX, UINT32_MAX}};
static inline bool length_is_undefined(Length length)
{
return length.bytes == 0 && length.extent.column != 0;
}
static inline Length length_min(Length len1, Length len2)
{
return (len1.bytes < len2.bytes) ? len1 : len2;
}
static inline Length length_add(Length len1, Length len2)
{
Length result;
result.bytes = len1.bytes + len2.bytes;
result.extent = point_add(len1.extent, len2.extent);
return result;
}
static inline Length length_sub(Length len1, Length len2)
{
Length result;
result.bytes = len1.bytes - len2.bytes;
result.extent = point_sub(len1.extent, len2.extent);
return result;
}
static inline Length length_zero(void)
{
Length result = {0, {0, 0}};
return result;
}
static inline Length length_saturating_sub(Length len1, Length len2)
{
if (len1.bytes > len2.bytes)
{
return length_sub(len1, len2);
}
else
{
return length_zero();
}
}
#endif

419
parser/nsrc/lexer.c
View file

@ -1,419 +0,0 @@
#include <stdio.h>
#include "./lexer.h"
#include "./subtree.h"
#include "./length.h"
#include "./unicode.h"
#define LOG(message, character) \
if (self->logger.log) { \
snprintf( \
self->debug_buffer, \
TREE_SITTER_SERIALIZATION_BUFFER_SIZE, \
32 <= character && character < 127 ? \
message " character:'%c'" : \
message " character:%d", \
character \
); \
self->logger.log( \
self->logger.payload, \
TSLogTypeLex, \
self->debug_buffer \
); \
}
static const int32_t BYTE_ORDER_MARK = 0xFEFF;
static const TSRange DEFAULT_RANGE = {
.start_point = {
.row = 0,
.column = 0,
},
.end_point = {
.row = UINT32_MAX,
.column = UINT32_MAX,
},
.start_byte = 0,
.end_byte = UINT32_MAX
};
// Check if the lexer has reached EOF. This state is stored
// by setting the lexer's `current_included_range_index` such that
// it has consumed all of its available ranges.
static bool ts_lexer__eof(const TSLexer *_self) {
Lexer *self = (Lexer *)_self;
return self->current_included_range_index == self->included_range_count;
}
// Clear the currently stored chunk of source code, because the lexer's
// position has changed.
static void ts_lexer__clear_chunk(Lexer *self) {
self->chunk = NULL;
self->chunk_size = 0;
self->chunk_start = 0;
}
// Call the lexer's input callback to obtain a new chunk of source code
// for the current position.
static void ts_lexer__get_chunk(Lexer *self) {
self->chunk_start = self->current_position.bytes;
self->chunk = self->input.read(
self->input.payload,
self->current_position.bytes,
self->current_position.extent,
&self->chunk_size
);
if (!self->chunk_size) {
self->current_included_range_index = self->included_range_count;
self->chunk = NULL;
}
}
// Decode the next unicode character in the current chunk of source code.
// This assumes that the lexer has already retrieved a chunk of source
// code that spans the current position.
static void ts_lexer__get_lookahead(Lexer *self) {
uint32_t position_in_chunk = self->current_position.bytes - self->chunk_start;
uint32_t size = self->chunk_size - position_in_chunk;
if (size == 0) {
self->lookahead_size = 1;
self->data.lookahead = '\0';
return;
}
const uint8_t *chunk = (const uint8_t *)self->chunk + position_in_chunk;
UnicodeDecodeFunction decode = self->input.encoding == TSInputEncodingUTF8
? ts_decode_ascii
: ts_decode_ascii;
self->lookahead_size = decode(chunk, size, &self->data.lookahead);
// If this chunk ended in the middle of a multi-byte character,
// try again with a fresh chunk.
if (self->data.lookahead == TS_DECODE_ERROR && size < 4) {
ts_lexer__get_chunk(self);
chunk = (const uint8_t *)self->chunk;
size = self->chunk_size;
self->lookahead_size = decode(chunk, size, &self->data.lookahead);
}
if (self->data.lookahead == TS_DECODE_ERROR) {
self->lookahead_size = 1;
}
}
static void ts_lexer_goto(Lexer *self, Length position) {
self->current_position = position;
// Move to the first valid position at or after the given position.
bool found_included_range = false;
for (unsigned i = 0; i < self->included_range_count; i++) {
TSRange *included_range = &self->included_ranges[i];
if (
included_range->end_byte > self->current_position.bytes &&
included_range->end_byte > included_range->start_byte
) {
if (included_range->start_byte >= self->current_position.bytes) {
self->current_position = (Length) {
.bytes = included_range->start_byte,
.extent = included_range->start_point,
};
}
self->current_included_range_index = i;
found_included_range = true;
break;
}
}
if (found_included_range) {
// If the current position is outside of the current chunk of text,
// then clear out the current chunk of text.
if (self->chunk && (
self->current_position.bytes < self->chunk_start ||
self->current_position.bytes >= self->chunk_start + self->chunk_size
)) {
ts_lexer__clear_chunk(self);
}
self->lookahead_size = 0;
self->data.lookahead = '\0';
}
// If the given position is beyond any of included ranges, move to the EOF
// state - past the end of the included ranges.
else {
self->current_included_range_index = self->included_range_count;
TSRange *last_included_range = &self->included_ranges[self->included_range_count - 1];
self->current_position = (Length) {
.bytes = last_included_range->end_byte,
.extent = last_included_range->end_point,
};
ts_lexer__clear_chunk(self);
self->lookahead_size = 1;
self->data.lookahead = '\0';
}
}
// Intended to be called only from functions that control logging.
static void ts_lexer__do_advance(Lexer *self, bool skip) {
if (self->lookahead_size) {
self->current_position.bytes += self->lookahead_size;
if (self->data.lookahead == '\n') {
self->current_position.extent.row++;
self->current_position.extent.column = 0;
} else {
self->current_position.extent.column += self->lookahead_size;
}
}
const TSRange *current_range = &self->included_ranges[self->current_included_range_index];
while (
self->current_position.bytes >= current_range->end_byte ||
current_range->end_byte == current_range->start_byte
) {
if (self->current_included_range_index < self->included_range_count) {
self->current_included_range_index++;
}
if (self->current_included_range_index < self->included_range_count) {
current_range++;
self->current_position = (Length) {
current_range->start_byte,
current_range->start_point,
};
} else {
current_range = NULL;
break;
}
}
if (skip) self->token_start_position = self->current_position;
if (current_range) {
if (
self->current_position.bytes < self->chunk_start ||
self->current_position.bytes >= self->chunk_start + self->chunk_size
) {
ts_lexer__get_chunk(self);
}
ts_lexer__get_lookahead(self);
} else {
ts_lexer__clear_chunk(self);
self->data.lookahead = '\0';
self->lookahead_size = 1;
}
}
// Advance to the next character in the source code, retrieving a new
// chunk of source code if needed.
static void ts_lexer__advance(TSLexer *_self, bool skip) {
Lexer *self = (Lexer *)_self;
if (!self->chunk) return;
if (skip) {
LOG("skip", self->data.lookahead)
} else {
LOG("consume", self->data.lookahead)
}
ts_lexer__do_advance(self, skip);
}
// Mark that a token match has completed. This can be called multiple
// times if a longer match is found later.
static void ts_lexer__mark_end(TSLexer *_self) {
Lexer *self = (Lexer *)_self;
if (!ts_lexer__eof(&self->data)) {
// If the lexer is right at the beginning of included range,
// then the token should be considered to end at the *end* of the
// previous included range, rather than here.
TSRange *current_included_range = &self->included_ranges[
self->current_included_range_index
];
if (
self->current_included_range_index > 0 &&
self->current_position.bytes == current_included_range->start_byte
) {
TSRange *previous_included_range = current_included_range - 1;
self->token_end_position = (Length) {
previous_included_range->end_byte,
previous_included_range->end_point,
};
return;
}
}
self->token_end_position = self->current_position;
}
static uint32_t ts_lexer__get_column(TSLexer *_self) {
Lexer *self = (Lexer *)_self;
uint32_t goal_byte = self->current_position.bytes;
self->did_get_column = true;
self->current_position.bytes -= self->current_position.extent.column;
self->current_position.extent.column = 0;
if (self->current_position.bytes < self->chunk_start) {
ts_lexer__get_chunk(self);
}
uint32_t result = 0;
if (!ts_lexer__eof(_self)) {
ts_lexer__get_lookahead(self);
while (self->current_position.bytes < goal_byte && self->chunk) {
result++;
ts_lexer__do_advance(self, false);
if (ts_lexer__eof(_self)) break;
}
}
return result;
}
// Is the lexer at a boundary between two disjoint included ranges of
// source code? This is exposed as an API because some languages' external
// scanners need to perform custom actions at these boundaries.
static bool ts_lexer__is_at_included_range_start(const TSLexer *_self) {
const Lexer *self = (const Lexer *)_self;
if (self->current_included_range_index < self->included_range_count) {
TSRange *current_range = &self->included_ranges[self->current_included_range_index];
return self->current_position.bytes == current_range->start_byte;
} else {
return false;
}
}
void ts_lexer_init(Lexer *self) {
*self = (Lexer) {
.data = {
// The lexer's methods are stored as struct fields so that generated
// parsers can call them without needing to be linked against this
// library.
.advance = ts_lexer__advance,
.mark_end = ts_lexer__mark_end,
.get_column = ts_lexer__get_column,
.is_at_included_range_start = ts_lexer__is_at_included_range_start,
.eof = ts_lexer__eof,
.lookahead = 0,
.result_symbol = 0,
},
.chunk = NULL,
.chunk_size = 0,
.chunk_start = 0,
.current_position = {0, {0, 0}},
.logger = {
.payload = NULL,
.log = NULL
},
.included_ranges = NULL,
.included_range_count = 0,
.current_included_range_index = 0,
};
ts_lexer_set_included_ranges(self, NULL, 0);
}
void ts_lexer_delete(Lexer *self) {
ts_free(self->included_ranges);
}
void ts_lexer_set_input(Lexer *self, TSInput input) {
self->input = input;
ts_lexer__clear_chunk(self);
ts_lexer_goto(self, self->current_position);
}
// Move the lexer to the given position. This doesn't do any work
// if the parser is already at the given position.
void ts_lexer_reset(Lexer *self, Length position) {
if (position.bytes != self->current_position.bytes) {
ts_lexer_goto(self, position);
}
}
void ts_lexer_start(Lexer *self) {
self->token_start_position = self->current_position;
self->token_end_position = LENGTH_UNDEFINED;
self->data.result_symbol = 0;
self->did_get_column = false;
if (!ts_lexer__eof(&self->data)) {
if (!self->chunk_size) ts_lexer__get_chunk(self);
if (!self->lookahead_size) ts_lexer__get_lookahead(self);
if (
self->current_position.bytes == 0 &&
self->data.lookahead == BYTE_ORDER_MARK
) ts_lexer__advance(&self->data, true);
}
}
void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte) {
if (length_is_undefined(self->token_end_position)) {
ts_lexer__mark_end(&self->data);
}
// If the token ended at an included range boundary, then its end position
// will have been reset to the end of the preceding range. Reset the start
// position to match.
if (self->token_end_position.bytes < self->token_start_position.bytes) {
self->token_start_position = self->token_end_position;
}
uint32_t current_lookahead_end_byte = self->current_position.bytes + 1;
// In order to determine that a byte sequence is invalid UTF8 or UTF16,
// the character decoding algorithm may have looked at the following byte.
// Therefore, the next byte *after* the current (invalid) character
// affects the interpretation of the current character.
if (self->data.lookahead == TS_DECODE_ERROR) {
current_lookahead_end_byte += 4; // the maximum number of bytes read to identify an invalid code point
}
if (current_lookahead_end_byte > *lookahead_end_byte) {
*lookahead_end_byte = current_lookahead_end_byte;
}
}
void ts_lexer_advance_to_end(Lexer *self) {
while (self->chunk) {
ts_lexer__advance(&self->data, false);
}
}
void ts_lexer_mark_end(Lexer *self) {
ts_lexer__mark_end(&self->data);
}
bool ts_lexer_set_included_ranges(
Lexer *self,
const TSRange *ranges,
uint32_t count
) {
if (count == 0 || !ranges) {
ranges = &DEFAULT_RANGE;
count = 1;
} else {
uint32_t previous_byte = 0;
for (unsigned i = 0; i < count; i++) {
const TSRange *range = &ranges[i];
if (
range->start_byte < previous_byte ||
range->end_byte < range->start_byte
) return false;
previous_byte = range->end_byte;
}
}
size_t size = count * sizeof(TSRange);
self->included_ranges = ts_realloc(self->included_ranges, size);
memcpy(self->included_ranges, ranges, size);
self->included_range_count = count;
ts_lexer_goto(self, self->current_position);
return true;
}
TSRange *ts_lexer_included_ranges(const Lexer *self, uint32_t *count) {
*count = self->included_range_count;
return self->included_ranges;
}
#undef LOG

51
parser/nsrc/lexer.h
View file

@ -1,51 +0,0 @@
#ifndef TREE_SITTER_LEXER_H_
#define TREE_SITTER_LEXER_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include "./api.h"
#include "./length.h"
#include "./parser.h"
#include "./subtree.h"
typedef struct Lexer
{
TSLexer data;
Length current_position;
Length token_start_position;
Length token_end_position;
TSRange *included_ranges;
const char *chunk;
TSInput input;
TSLogger logger;
uint32_t included_range_count;
uint32_t current_included_range_index;
uint32_t chunk_start;
uint32_t chunk_size;
uint32_t lookahead_size;
bool did_get_column;
char debug_buffer[TREE_SITTER_SERIALIZATION_BUFFER_SIZE];
} Lexer;
void ts_lexer_init(Lexer *);
void ts_lexer_delete(Lexer *);
void ts_lexer_set_input(Lexer *, TSInput);
void ts_lexer_reset(Lexer *, Length);
void ts_lexer_start(Lexer *);
void ts_lexer_finish(Lexer *, uint32_t *);
void ts_lexer_advance_to_end(Lexer *);
void ts_lexer_mark_end(Lexer *);
bool ts_lexer_set_included_ranges(Lexer *self, const TSRange *ranges, uint32_t count);
TSRange *ts_lexer_included_ranges(const Lexer *self, uint32_t *count);
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_LEXER_H_

15
parser/nsrc/lib.c
View file

@ -1,15 +0,0 @@
#define _POSIX_C_SOURCE 200112L
#include "./alloc.c"
#include "./create_language.c"
#include "./get_changed_ranges.c"
#include "./language.c"
#include "./lexer.c"
#include "./node.c"
#include "./parser.c"
#include "./query.c"
#include "./scanner.c"
#include "./stack.c"
#include "./subtree.c"
#include "./tree.c"
#include "./tree_cursor.c"

907
parser/nsrc/node.c
View file

@ -1,907 +0,0 @@
#include "./language.h"
#include "./subtree.h"
#include "./tree.h"
#include "me/str/str.h"
#include <stdbool.h>
typedef struct NodeChildIterator
{
Subtree parent;
const TSTree *tree;
Length position;
uint32_t child_index;
uint32_t structural_child_index;
const TSSymbol *alias_sequence;
} NodeChildIterator;
// TSNode - constructors
TSNode ts_node_new(const TSTree *tree, const Subtree *subtree, Length position, TSSymbol alias)
{
return (TSNode){
{position.bytes, position.extent.row, position.extent.column, alias},
subtree,
tree,
};
}
static inline TSNode ts_node__null(void)
{
return ts_node_new(NULL, NULL, length_zero(), 0);
}
// TSNode - accessors
uint32_t ts_node_start_byte(TSNode self)
{
return self.context[0];
}
TSPoint ts_node_start_point(TSNode self)
{
return (TSPoint){self.context[1], self.context[2]};
}
static inline uint32_t ts_node__alias(const TSNode *self)
{
return self->context[3];
}
static inline Subtree ts_node__subtree(TSNode self)
{
return *(const Subtree *)self.id;
}
// NodeChildIterator
static inline NodeChildIterator ts_node_iterate_children(const TSNode *node)
{
Subtree subtree = ts_node__subtree(*node);
if (ts_subtree_child_count(subtree) == 0)
{
return (NodeChildIterator){NULL_SUBTREE, node->tree, length_zero(), 0, 0, NULL};
}
const TSSymbol *alias_sequence = ts_language_alias_sequence(node->tree->language, subtree.ptr->inner.non_terminal.production_id);
return (NodeChildIterator){
.tree = node->tree,
.parent = subtree,
.position = {ts_node_start_byte(*node), ts_node_start_point(*node)},
.child_index = 0,
.structural_child_index = 0,
.alias_sequence = alias_sequence,
};
}
static inline bool ts_node_child_iterator_done(NodeChildIterator *self)
{
return self->child_index == self->parent.ptr->child_count;
}
static inline bool ts_node_child_iterator_next(NodeChildIterator *self, TSNode *result)
{
if (!self->parent.ptr || ts_node_child_iterator_done(self))
return false;
const Subtree *child = &ts_subtree_children(self->parent)[self->child_index];
TSSymbol alias_symbol = 0;
if (!ts_subtree_extra(*child))
{
if (self->alias_sequence)
{
alias_symbol = self->alias_sequence[self->structural_child_index];
}
self->structural_child_index++;
}
if (self->child_index > 0)
{
self->position = length_add(self->position, ts_subtree_padding(*child));
}
*result = ts_node_new(self->tree, child, self->position, alias_symbol);
self->position = length_add(self->position, ts_subtree_size(*child));
self->child_index++;
return true;
}
// TSNode - private
static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous)
{
Subtree tree = ts_node__subtree(self);
if (include_anonymous)
{
return ts_subtree_visible(tree) || ts_node__alias(&self);
}
else
{
TSSymbol alias = ts_node__alias(&self);
if (alias)
{
return ts_language_symbol_metadata(self.tree->language, alias).named;
}
else
{
return ts_subtree_visible(tree) && ts_subtree_named(tree);
}
}
}
static inline uint32_t ts_node__relevant_child_count(TSNode self, bool include_anonymous)
{
Subtree tree = ts_node__subtree(self);
if (ts_subtree_child_count(tree) > 0)
{
if (include_anonymous)
{
return tree.ptr->inner.non_terminal.visible_child_count;
}
else
{
return tree.ptr->inner.non_terminal.named_child_count;
}
}
else
{
return 0;
}
}
static inline TSNode ts_node__child(TSNode self, uint32_t child_index, bool include_anonymous)
{
TSNode result = self;
bool did_descend = true;
while (did_descend)
{
did_descend = false;
TSNode child;
uint32_t index = 0;
NodeChildIterator iterator = ts_node_iterate_children(&result);
while (ts_node_child_iterator_next(&iterator, &child))
{
if (ts_node__is_relevant(child, include_anonymous))
{
if (index == child_index)
{
return child;
}
index++;
}
else
{
uint32_t grandchild_index = child_index - index;
uint32_t grandchild_count = ts_node__relevant_child_count(child, include_anonymous);
if (grandchild_index < grandchild_count)
{
did_descend = true;
result = child;
child_index = grandchild_index;
break;
}
index += grandchild_count;
}
}
}
return ts_node__null();
}
static bool ts_subtree_has_trailing_empty_descendant(Subtree self, Subtree other)
{
for (unsigned i = ts_subtree_child_count(self) - 1; i + 1 > 0; i--)
{
Subtree child = ts_subtree_children(self)[i];
if (ts_subtree_total_bytes(child) > 0)
break;
if (child.ptr == other.ptr || ts_subtree_has_trailing_empty_descendant(child, other))
{
return true;
}
}
return false;
}
static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous)
{
Subtree self_subtree = ts_node__subtree(self);
bool self_is_empty = ts_subtree_total_bytes(self_subtree) == 0;
uint32_t target_end_byte = ts_node_end_byte(self);
TSNode node = ts_node_parent(self);
TSNode earlier_node = ts_node__null();
bool earlier_node_is_relevant = false;
while (!ts_node_is_null(node))
{
TSNode earlier_child = ts_node__null();
bool earlier_child_is_relevant = false;
bool found_child_containing_target = false;
TSNode child;
NodeChildIterator iterator = ts_node_iterate_children(&node);
while (ts_node_child_iterator_next(&iterator, &child))
{
if (child.id == self.id)
break;
if (iterator.position.bytes > target_end_byte)
{
found_child_containing_target = true;
break;
}
if (iterator.position.bytes == target_end_byte &&
(!self_is_empty || ts_subtree_has_trailing_empty_descendant(ts_node__subtree(child), self_subtree)))
{
found_child_containing_target = true;
break;
}
if (ts_node__is_relevant(child, include_anonymous))
{
earlier_child = child;
earlier_child_is_relevant = true;
}
else if (ts_node__relevant_child_count(child, include_anonymous) > 0)
{
earlier_child = child;
earlier_child_is_relevant = false;
}
}
if (found_child_containing_target)
{
if (!ts_node_is_null(earlier_child))
{
earlier_node = earlier_child;
earlier_node_is_relevant = earlier_child_is_relevant;
}
node = child;
}
else if (earlier_child_is_relevant)
{
return earlier_child;
}
else if (!ts_node_is_null(earlier_child))
{
node = earlier_child;
}
else if (earlier_node_is_relevant)
{
return earlier_node;
}
else
{
node = earlier_node;
earlier_node = ts_node__null();
earlier_node_is_relevant = false;
}
}
return ts_node__null();
}
static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous)
{
uint32_t target_end_byte = ts_node_end_byte(self);
TSNode node = ts_node_parent(self);
TSNode later_node = ts_node__null();
bool later_node_is_relevant = false;
while (!ts_node_is_null(node))
{
TSNode later_child = ts_node__null();
bool later_child_is_relevant = false;
TSNode child_containing_target = ts_node__null();
TSNode child;
NodeChildIterator iterator = ts_node_iterate_children(&node);
while (ts_node_child_iterator_next(&iterator, &child))
{
if (iterator.position.bytes < target_end_byte)
continue;
if (ts_node_start_byte(child) <= ts_node_start_byte(self))
{
if (ts_node__subtree(child).ptr != ts_node__subtree(self).ptr)
{
child_containing_target = child;
}
}
else if (ts_node__is_relevant(child, include_anonymous))
{
later_child = child;
later_child_is_relevant = true;
break;
}
else if (ts_node__relevant_child_count(child, include_anonymous) > 0)
{
later_child = child;
later_child_is_relevant = false;
break;
}
}
if (!ts_node_is_null(child_containing_target))
{
if (!ts_node_is_null(later_child))
{
later_node = later_child;
later_node_is_relevant = later_child_is_relevant;
}
node = child_containing_target;
}
else if (later_child_is_relevant)
{
return later_child;
}
else if (!ts_node_is_null(later_child))
{
node = later_child;
}
else if (later_node_is_relevant)
{
return later_node;
}
else
{
node = later_node;
}
}
return ts_node__null();
}
static inline TSNode ts_node__first_child_for_byte(TSNode self, uint32_t goal, bool include_anonymous)
{
TSNode node = self;
bool did_descend = true;
while (did_descend)
{
did_descend = false;
TSNode child;
NodeChildIterator iterator = ts_node_iterate_children(&node);
while (ts_node_child_iterator_next(&iterator, &child))
{
if (ts_node_end_byte(child) > goal)
{
if (ts_node__is_relevant(child, include_anonymous))
{
return child;
}
else if (ts_node_child_count(child) > 0)
{
did_descend = true;
node = child;
break;
}
}
}
}
return ts_node__null();
}
static inline TSNode ts_node__descendant_for_byte_range(TSNode self, uint32_t range_start, uint32_t range_end, bool include_anonymous)
{
TSNode node = self;
TSNode last_visible_node = self;
bool did_descend = true;
while (did_descend)
{
did_descend = false;
TSNode child;
NodeChildIterator iterator = ts_node_iterate_children(&node);
while (ts_node_child_iterator_next(&iterator, &child))
{
uint32_t node_end = iterator.position.bytes;
// The end of this node must extend far enough forward to touch
// the end of the range and exceed the start of the range.
if (node_end < range_end)
continue;
if (node_end <= range_start)
continue;
// The start of this node must extend far enough backward to
// touch the start of the range.
if (range_start < ts_node_start_byte(child))
break;
node = child;
if (ts_node__is_relevant(node, include_anonymous))
{
last_visible_node = node;
}
did_descend = true;
break;
}
}
return last_visible_node;
}
static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint range_start, TSPoint range_end, bool include_anonymous)
{
TSNode node = self;
TSNode last_visible_node = self;
bool did_descend = true;
while (did_descend)
{
did_descend = false;
TSNode child;
NodeChildIterator iterator = ts_node_iterate_children(&node);
while (ts_node_child_iterator_next(&iterator, &child))
{
TSPoint node_end = iterator.position.extent;
// The end of this node must extend far enough forward to touch
// the end of the range and exceed the start of the range.
if (point_lt(node_end, range_end))
continue;
if (point_lte(node_end, range_start))
continue;
// The start of this node must extend far enough backward to
// touch the start of the range.
if (point_lt(range_start, ts_node_start_point(child)))
break;
node = child;
if (ts_node__is_relevant(node, include_anonymous))
{
last_visible_node = node;
}
did_descend = true;
break;
}
}
return last_visible_node;
}
// TSNode - public
uint32_t ts_node_end_byte(TSNode self)
{
return ts_node_start_byte(self) + ts_subtree_size(ts_node__subtree(self)).bytes;
}
TSPoint ts_node_end_point(TSNode self)
{
return point_add(ts_node_start_point(self), ts_subtree_size(ts_node__subtree(self)).extent);
}
TSSymbol ts_node_symbol(TSNode self)
{
TSSymbol symbol = ts_node__alias(&self);
if (!symbol)
symbol = ts_subtree_symbol(ts_node__subtree(self));
return ts_language_public_symbol(self.tree->language, symbol);
}
const char *ts_node_type(TSNode self)
{
TSSymbol symbol = ts_node__alias(&self);
if (!symbol)
symbol = ts_subtree_symbol(ts_node__subtree(self));
return ts_language_symbol_name(self.tree->language, symbol);
}
const TSLanguage *ts_node_language(TSNode self)
{
return self.tree->language;
}
TSSymbol ts_node_grammar_symbol(TSNode self)
{
return ts_subtree_symbol(ts_node__subtree(self));
}
const char *ts_node_grammar_type(TSNode self)
{
TSSymbol symbol = ts_subtree_symbol(ts_node__subtree(self));
return ts_language_symbol_name(self.tree->language, symbol);
}
char *ts_node_string(TSNode self)
{
TSSymbol alias_symbol = ts_node__alias(&self);
return ts_subtree_string(ts_node__subtree(self), alias_symbol, ts_language_symbol_metadata(self.tree->language, alias_symbol).visible,
self.tree->language, false);
}
bool ts_node_eq(TSNode self, TSNode other)
{
return self.tree == other.tree && self.id == other.id;
}
bool ts_node_is_null(TSNode self)
{
return self.id == 0;
}
bool ts_node_is_extra(TSNode self)
{
return ts_subtree_extra(ts_node__subtree(self));
}
bool ts_node_is_named(TSNode self)
{
TSSymbol alias = ts_node__alias(&self);
return alias ? ts_language_symbol_metadata(self.tree->language, alias).named : ts_subtree_named(ts_node__subtree(self));
}
bool ts_node_is_missing(TSNode self)
{
return ts_subtree_missing(ts_node__subtree(self));
}
bool ts_node_has_changes(TSNode self)
{
return ts_subtree_has_changes(ts_node__subtree(self));
}
bool ts_node_has_error(TSNode self)
{
return ts_subtree_error_cost(ts_node__subtree(self)) > 0;
}
bool ts_node_is_error(TSNode self)
{
TSSymbol symbol = ts_node_symbol(self);
return symbol == ts_builtin_sym_error;
}
uint32_t ts_node_descendant_count(TSNode self)
{
return ts_subtree_visible_descendant_count(ts_node__subtree(self)) + 1;
}
TSStateId ts_node_parse_state(TSNode self)
{
return ts_subtree_parse_state(ts_node__subtree(self));
}
TSStateId ts_node_next_parse_state(TSNode self)
{
const TSLanguage *language = self.tree->language;
uint16_t state = ts_node_parse_state(self);
if (state == TS_TREE_STATE_NONE)
{
return TS_TREE_STATE_NONE;
}
uint16_t symbol = ts_node_grammar_symbol(self);
return ts_language_next_state(language, state, symbol);
}
TSNode ts_node_parent(TSNode self)
{
TSNode node = ts_tree_root_node(self.tree);
if (node.id == self.id)
return ts_node__null();
while (true)
{
TSNode next_node = ts_node_child_containing_descendant(node, self);
if (ts_node_is_null(next_node))
break;
node = next_node;
}
return node;
}
TSNode ts_node_child_containing_descendant(TSNode self, TSNode subnode)
{
uint32_t start_byte = ts_node_start_byte(subnode);
uint32_t end_byte = ts_node_end_byte(subnode);
do
{
NodeChildIterator iter = ts_node_iterate_children(&self);
do
{
if (!ts_node_child_iterator_next(&iter, &self) || ts_node_start_byte(self) > start_byte || self.id == subnode.id)
{
return ts_node__null();
}
} while (iter.position.bytes < end_byte || ts_node_child_count(self) == 0);
} while (!ts_node__is_relevant(self, true));
return self;
}
TSNode ts_node_child(TSNode self, uint32_t child_index)
{
return ts_node__child(self, child_index, true);
}
TSNode ts_node_named_child(TSNode self, uint32_t child_index)
{
return ts_node__child(self, child_index, false);
}
TSNode ts_node_child_by_field_id(TSNode self, TSFieldId field_id)
{
recur:
if (!field_id || ts_node_child_count(self) == 0)
return ts_node__null();
const TSFieldMapEntry *field_map, *field_map_end;
ts_language_field_map(self.tree->language, ts_node__subtree(self).ptr->inner.non_terminal.production_id, &field_map, &field_map_end);
if (field_map == field_map_end)
return ts_node__null();
// The field mappings are sorted by their field id. Scan all
// the mappings to find the ones for the given field id.
while (field_map->field_id < field_id)
{
field_map++;
if (field_map == field_map_end)
return ts_node__null();
}
while (field_map_end[-1].field_id > field_id)
{
field_map_end--;
if (field_map == field_map_end)
return ts_node__null();
}
TSNode child;
NodeChildIterator iterator = ts_node_iterate_children(&self);
while (ts_node_child_iterator_next(&iterator, &child))
{
if (!ts_subtree_extra(ts_node__subtree(child)))
{
uint32_t index = iterator.structural_child_index - 1;
if (index < field_map->child_index)
continue;
// Hidden nodes' fields are "inherited" by their visible parent.
if (field_map->inherited)
{
// If this is the *last* possible child node for this field,
// then perform a tail call to avoid recursion.
if (field_map + 1 == field_map_end)
{
self = child;
goto recur;
}
// Otherwise, descend into this child, but if it doesn't contain
// the field, continue searching subsequent children.
else
{
TSNode result = ts_node_child_by_field_id(child, field_id);
if (result.id)
return result;
field_map++;
if (field_map == field_map_end)
return ts_node__null();
}
}
else if (ts_node__is_relevant(child, true))
{
return child;
}
// If the field refers to a hidden node with visible children,
// return the first visible child.
else if (ts_node_child_count(child) > 0)
{
return ts_node_child(child, 0);
}
// Otherwise, continue searching subsequent children.
else
{
field_map++;
if (field_map == field_map_end)
return ts_node__null();
}
}
}
return ts_node__null();
}
static inline TSFieldId ts_node__field_id_from_language(TSNode self, uint32_t structural_child_index)
{
const TSFieldMapEntry *field_map, *field_map_end;
ts_language_field_map(self.tree->language, ts_node__subtree(self).ptr->inner.non_terminal.production_id, &field_map, &field_map_end);
for (; field_map != field_map_end; field_map++)
{
if (!field_map->inherited && field_map->child_index == structural_child_index)
{
return field_map->field_id;
}
}
return 0;
}
static inline const char *ts_node__field_name_from_language(TSNode self, uint32_t structural_child_index)
{
const TSFieldMapEntry *field_map, *field_map_end;
ts_language_field_map(self.tree->language, ts_node__subtree(self).ptr->inner.non_terminal.production_id, &field_map, &field_map_end);
for (; field_map != field_map_end; field_map++)
{
if (!field_map->inherited && field_map->child_index == structural_child_index)
{
return self.tree->language->field_names[field_map->field_id];
}
}
return NULL;
}
const char *ts_node_field_name_for_child(TSNode self, uint32_t child_index)
{
TSNode result = self;
bool did_descend = true;
const char *inherited_field_name = NULL;
while (did_descend)
{
did_descend = false;
TSNode child;
uint32_t index = 0;
NodeChildIterator iterator = ts_node_iterate_children(&result);
while (ts_node_child_iterator_next(&iterator, &child))
{
if (ts_node__is_relevant(child, true))
{
if (index == child_index)
{
if (ts_node_is_extra(child))
{
return NULL;
}
const char *field_name = ts_node__field_name_from_language(result, iterator.structural_child_index - 1);
if (field_name)
return field_name;
return inherited_field_name;
}
index++;
}
else
{
uint32_t grandchild_index = child_index - index;
uint32_t grandchild_count = ts_node__relevant_child_count(child, true);
if (grandchild_index < grandchild_count)
{
const char *field_name = ts_node__field_name_from_language(result, iterator.structural_child_index - 1);
if (field_name)
inherited_field_name = field_name;
did_descend = true;
result = child;
child_index = grandchild_index;
break;
}
index += grandchild_count;
}
}
}
return NULL;
}
TSNode ts_node_child_by_field_name(TSNode self, const char *name, uint32_t name_length)
{
TSFieldId field_id = ts_language_field_id_for_name(self.tree->language, name, name_length);
return ts_node_child_by_field_id(self, field_id);
}
uint32_t ts_node_child_count(TSNode self)
{
Subtree tree = ts_node__subtree(self);
if (ts_subtree_child_count(tree) > 0)
{
return tree.ptr->inner.non_terminal.visible_child_count;
}
else
{
return 0;
}
}
uint32_t ts_node_named_child_count(TSNode self)
{
Subtree tree = ts_node__subtree(self);
if (ts_subtree_child_count(tree) > 0)
{
return tree.ptr->inner.non_terminal.named_child_count;
}
else
{
return 0;
}
}
TSFieldId ts_node_field_id_for_child(TSNode self, uint32_t child_index)
{
const char *field_name;
field_name = ts_node_field_name_for_child(self, child_index);
if (field_name != NULL)
return (ts_language_field_id_for_name(ts_node_language(self), field_name, str_len(field_name)));
return 0;
}
TSNode ts_node_next_sibling(TSNode self)
{
return ts_node__next_sibling(self, true);
}
TSNode ts_node_next_named_sibling(TSNode self)
{
return ts_node__next_sibling(self, false);
}
TSNode ts_node_prev_sibling(TSNode self)
{
return ts_node__prev_sibling(self, true);
}
TSNode ts_node_prev_named_sibling(TSNode self)
{
return ts_node__prev_sibling(self, false);
}
TSNode ts_node_first_child_for_byte(TSNode self, uint32_t byte)
{
return ts_node__first_child_for_byte(self, byte, true);
}
TSNode ts_node_first_named_child_for_byte(TSNode self, uint32_t byte)
{
return ts_node__first_child_for_byte(self, byte, false);
}
TSNode ts_node_descendant_for_byte_range(TSNode self, uint32_t start, uint32_t end)
{
return ts_node__descendant_for_byte_range(self, start, end, true);
}
TSNode ts_node_named_descendant_for_byte_range(TSNode self, uint32_t start, uint32_t end)
{
return ts_node__descendant_for_byte_range(self, start, end, false);
}
TSNode ts_node_descendant_for_point_range(TSNode self, TSPoint start, TSPoint end)
{
return ts_node__descendant_for_point_range(self, start, end, true);
}
TSNode ts_node_named_descendant_for_point_range(TSNode self, TSPoint start, TSPoint end)
{
return ts_node__descendant_for_point_range(self, start, end, false);
}
void ts_node_edit(TSNode *self, const TSInputEdit *edit)
{
uint32_t start_byte = ts_node_start_byte(*self);
TSPoint start_point = ts_node_start_point(*self);
if (start_byte >= edit->old_end_byte)
{
start_byte = edit->new_end_byte + (start_byte - edit->old_end_byte);
start_point = point_add(edit->new_end_point, point_sub(start_point, edit->old_end_point));
}
else if (start_byte > edit->start_byte)
{
start_byte = edit->new_end_byte;
start_point = edit->new_end_point;
}
self->context[0] = start_byte;
self->context[1] = start_point.row;
self->context[2] = start_point.column;
}

2101
parser/nsrc/parser.c
View file

File diff suppressed because it is too large Load diff

285
parser/nsrc/parser.h
View file

@ -1,285 +0,0 @@
#ifndef TREE_SITTER_PARSER_H_
#define TREE_SITTER_PARSER_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#define ts_builtin_sym_error ((TSSymbol)-1)
#define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
#ifndef TREE_SITTER_API_H_
typedef uint16_t TSStateId;
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
#endif
typedef struct TSFieldMapEntry
{
TSFieldId field_id;
uint8_t child_index;
bool inherited;
} TSFieldMapEntry;
typedef struct TSFieldMapSlice
{
uint16_t index;
uint16_t length;
} TSFieldMapSlice;
typedef struct TSSymbolMetadata
{
bool visible;
bool named;
bool supertype;
} TSSymbolMetadata;
typedef struct TSLexer TSLexer;
struct TSLexer
{
int32_t lookahead;
TSSymbol result_symbol;
void (*advance)(TSLexer *, bool);
void (*mark_end)(TSLexer *);
uint32_t (*get_column)(TSLexer *);
bool (*is_at_included_range_start)(const TSLexer *);
bool (*eof)(const TSLexer *);
};
typedef enum TSParseActionType
{
TSParseActionTypeShift,
TSParseActionTypeReduce,
TSParseActionTypeAccept,
TSParseActionTypeRecover,
} TSParseActionType;
typedef union TSParseAction {
struct TSParseActionShift
{
uint8_t type;
TSStateId state;
bool extra;
bool repetition;
} shift;
struct TSParseActionReduce
{
uint8_t type;
uint8_t child_count;
TSSymbol symbol;
int16_t dynamic_precedence;
uint16_t production_id;
} reduce;
uint8_t type;
} TSParseAction;
typedef struct TSLexMode
{
uint16_t lex_state;
uint16_t external_lex_state;
} TSLexMode;
typedef union TSParseActionEntry {
TSParseAction action;
struct TSParseActionEntryData
{
uint8_t count;
bool reusable;
} entry;
} TSParseActionEntry;
typedef struct TSCharacterRange
{
int32_t start;
int32_t end;
} TSCharacterRange;
struct TSLanguage
{
uint32_t version;
uint32_t symbol_count;
uint32_t alias_count;
uint32_t token_count;
uint32_t external_token_count;
uint32_t state_count;
uint32_t large_state_count;
uint32_t production_id_count;
uint32_t field_count;
uint16_t max_alias_sequence_length;
const uint16_t *parse_table;
const uint16_t *small_parse_table;
const uint32_t *small_parse_table_map;
const TSParseActionEntry *parse_actions;
const char *const *symbol_names;
const char *const *field_names;
const TSFieldMapSlice *field_map_slices;
const TSFieldMapEntry *field_map_entries;
const TSSymbolMetadata *symbol_metadata;
const TSSymbol *public_symbol_map;
const uint16_t *alias_map;
const TSSymbol *alias_sequences;
const TSLexMode *lex_modes;
bool (*lex_fn)(TSLexer *, TSStateId);
bool (*keyword_lex_fn)(TSLexer *, TSStateId);
TSSymbol keyword_capture_token;
struct ExternalScannerDefinition
{
const bool *states;
const TSSymbol *symbol_map;
void *(*create)(void);
void (*destroy)(void *);
bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
unsigned (*serialize)(void *, char *);
void (*deserialize)(void *, const char *, unsigned);
} external_scanner;
const TSStateId *primary_state_ids;
};
static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead)
{
uint32_t index = 0;
uint32_t size = len - index;
while (size > 1)
{
uint32_t half_size = size / 2;
uint32_t mid_index = index + half_size;
TSCharacterRange *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end)
{
return true;
}
else if (lookahead > range->end)
{
index = mid_index;
}
size -= half_size;
}
TSCharacterRange *range = &ranges[index];
return (lookahead >= range->start && lookahead <= range->end);
}
/*
* Lexer Macros
*/
#ifdef _MSC_VER
# define UNUSED __pragma(warning(suppress : 4101))
#else
# define UNUSED __attribute__((unused))
#endif
#define START_LEXER() \
bool result = false; \
bool skip = false; \
UNUSED \
bool eof = false; \
int32_t lookahead; \
goto start; \
next_state: \
lexer->advance(lexer, skip); \
start: \
skip = false; \
lookahead = lexer->lookahead;
#define ADVANCE(state_value) \
{ \
state = state_value; \
goto next_state; \
}
#define ADVANCE_MAP(...) \
{ \
static const uint16_t map[] = {__VA_ARGS__}; \
for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) \
{ \
if (map[i] == lookahead) \
{ \
state = map[i + 1]; \
goto next_state; \
} \
} \
}
#define SKIP(state_value) \
{ \
skip = true; \
state = state_value; \
goto next_state; \
}
#define ACCEPT_TOKEN(symbol_value) \
result = true; \
lexer->result_symbol = symbol_value; \
lexer->mark_end(lexer);
#define END_STATE() return result;
/*
* Parse Table Macros
*/
#define SMALL_STATE(id) ((id)-LARGE_STATE_COUNT)
#define STATE(id) id
#define ACTIONS(id) id
#define SHIFT(state_value) \
{ \
{ \
.shift = {.type = TSParseActionTypeShift, .state = (state_value) } \
} \
}
#define SHIFT_REPEAT(state_value) \
{ \
{ \
.shift = {.type = TSParseActionTypeShift, .state = (state_value), .repetition = true } \
} \
}
#define SHIFT_EXTRA() \
{ \
{ \
.shift = {.type = TSParseActionTypeShift, .extra = true } \
} \
}
#define REDUCE(symbol_name, children, precedence, prod_id) \
{ \
{ \
.reduce = {.type = TSParseActionTypeReduce, \
.symbol = symbol_name, \
.child_count = children, \
.dynamic_precedence = precedence, \
.production_id = prod_id}, \
} \
}
#define RECOVER() \
{ \
{ \
.type = TSParseActionTypeRecover \
} \
}
#define ACCEPT_INPUT() \
{ \
{ \
.type = TSParseActionTypeAccept \
} \
}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_PARSER_H_

72
parser/nsrc/point.h
View file

@ -1,72 +0,0 @@
#ifndef TREE_SITTER_POINT_H_
#define TREE_SITTER_POINT_H_
#include "./api.h"
#define POINT_ZERO ((TSPoint){0, 0})
#define POINT_MAX ((TSPoint){UINT32_MAX, UINT32_MAX})
static inline TSPoint point__new(unsigned row, unsigned column)
{
TSPoint result = {row, column};
return result;
}
static inline TSPoint point_add(TSPoint a, TSPoint b)
{
if (b.row > 0)
return point__new(a.row + b.row, b.column);
else
return point__new(a.row, a.column + b.column);
}
static inline TSPoint point_sub(TSPoint a, TSPoint b)
{
if (a.row > b.row)
return point__new(a.row - b.row, a.column);
else
return point__new(0, a.column - b.column);
}
static inline bool point_lte(TSPoint a, TSPoint b)
{
return (a.row < b.row) || (a.row == b.row && a.column <= b.column);
}
static inline bool point_lt(TSPoint a, TSPoint b)
{
return (a.row < b.row) || (a.row == b.row && a.column < b.column);
}
static inline bool point_gt(TSPoint a, TSPoint b)
{
return (a.row > b.row) || (a.row == b.row && a.column > b.column);
}
static inline bool point_gte(TSPoint a, TSPoint b)
{
return (a.row > b.row) || (a.row == b.row && a.column >= b.column);
}
static inline bool point_eq(TSPoint a, TSPoint b)
{
return a.row == b.row && a.column == b.column;
}
static inline TSPoint point_min(TSPoint a, TSPoint b)
{
if (a.row < b.row || (a.row == b.row && a.column < b.column))
return a;
else
return b;
}
static inline TSPoint point_max(TSPoint a, TSPoint b)
{
if (a.row > b.row || (a.row == b.row && a.column > b.column))
return a;
else
return b;
}
#endif

4146
parser/nsrc/query.c
View file

File diff suppressed because it is too large Load diff

37
parser/nsrc/reduce_action.h
View file

@ -1,37 +0,0 @@
#ifndef TREE_SITTER_REDUCE_ACTION_H_
#define TREE_SITTER_REDUCE_ACTION_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include "./api.h"
#include "./array.h"
typedef struct ReduceAction
{
uint32_t count;
TSSymbol symbol;
int dynamic_precedence;
unsigned short production_id;
} ReduceAction;
typedef Array(ReduceAction) ReduceActionSet;
static inline void ts_reduce_action_set_add(ReduceActionSet *self, ReduceAction new_action)
{
for (uint32_t i = 0; i < self->size; i++)
{
ReduceAction action = self->contents[i];
if (action.symbol == new_action.symbol && action.count == new_action.count)
return;
}
array_push(self, new_action);
}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_REDUCE_ACTION_H_

111
parser/nsrc/reusable_node.h
View file

@ -1,111 +0,0 @@
#include "./subtree.h"
typedef struct StackEntry
{
Subtree tree;
uint32_t child_index;
uint32_t byte_offset;
} StackEntry;
typedef struct ReusableNode
{
Array(StackEntry) stack;
Subtree last_external_token;
} ReusableNode;
static inline ReusableNode reusable_node_new(void)
{
return (ReusableNode){array_new(), NULL_SUBTREE};
}
static inline void reusable_node_clear(ReusableNode *self)
{
array_clear(&self->stack);
self->last_external_token = NULL_SUBTREE;
}
static inline Subtree reusable_node_tree(ReusableNode *self)
{
return self->stack.size > 0 ? self->stack.contents[self->stack.size - 1].tree : NULL_SUBTREE;
}
static inline uint32_t reusable_node_byte_offset(ReusableNode *self)
{
return self->stack.size > 0 ? self->stack.contents[self->stack.size - 1].byte_offset : UINT32_MAX;
}
static inline void reusable_node_delete(ReusableNode *self)
{
array_delete(&self->stack);
}
static inline void reusable_node_advance(ReusableNode *self)
{
StackEntry last_entry = *array_back(&self->stack);
uint32_t byte_offset = last_entry.byte_offset + ts_subtree_total_bytes(last_entry.tree);
if (ts_subtree_has_external_tokens(last_entry.tree))
{
self->last_external_token = ts_subtree_last_external_token(last_entry.tree);
}
Subtree tree;
uint32_t next_index;
do
{
StackEntry popped_entry = array_pop(&self->stack);
next_index = popped_entry.child_index + 1;
if (self->stack.size == 0)
return;
tree = array_back(&self->stack)->tree;
} while (ts_subtree_child_count(tree) <= next_index);
array_push(&self->stack, ((StackEntry){
.tree = ts_subtree_children(tree)[next_index],
.child_index = next_index,
.byte_offset = byte_offset,
}));
}
static inline bool reusable_node_descend(ReusableNode *self)
{
StackEntry last_entry = *array_back(&self->stack);
if (ts_subtree_child_count(last_entry.tree) > 0)
{
array_push(&self->stack, ((StackEntry){
.tree = ts_subtree_children(last_entry.tree)[0],
.child_index = 0,
.byte_offset = last_entry.byte_offset,
}));
return true;
}
else
{
return false;
}
}
static inline void reusable_node_advance_past_leaf(ReusableNode *self)
{
while (reusable_node_descend(self))
{
}
reusable_node_advance(self);
}
static inline void reusable_node_reset(ReusableNode *self, Subtree tree)
{
reusable_node_clear(self);
array_push(&self->stack, ((StackEntry){
.tree = tree,
.child_index = 0,
.byte_offset = 0,
}));
// Never reuse the root node, because it has a non-standard internal structure
// due to transformations that are applied when it is accepted: adding the EOF
// child and any extra children.
if (!reusable_node_descend(self))
{
reusable_node_clear(self);
}
}

1241
parser/nsrc/scanner.c
View file

File diff suppressed because it is too large Load diff

992
parser/nsrc/stack.c
View file

@ -1,992 +0,0 @@
#include "./stack.h"
#include "./alloc.h"
#include "./array.h"
#include "./language.h"
#include "./length.h"
#include "./subtree.h"
#include <assert.h>
#include <inttypes.h>
#include <stdio.h>
#define MAX_LINK_COUNT 8
#define MAX_NODE_POOL_SIZE 50
#define MAX_ITERATOR_COUNT 64
#if defined _WIN32 && !defined __GNUC__
# define forceinline __forceinline
#else
# define forceinline static inline __attribute__((always_inline))
#endif
typedef struct StackNode StackNode;
typedef struct StackLink
{
StackNode *node;
Subtree subtree;
bool is_pending;
} StackLink;
struct StackNode
{
TSStateId state;
Length position;
StackLink links[MAX_LINK_COUNT];
short unsigned int link_count;
uint32_t ref_count;
unsigned error_cost;
unsigned node_count;
int dynamic_precedence;
};
typedef struct StackIterator
{
StackNode *node;
SubtreeArray subtrees;
uint32_t subtree_count;
bool is_pending;
} StackIterator;
typedef Array(StackNode *) StackNodeArray;
typedef enum StackStatus
{
StackStatusActive,
StackStatusPaused,
StackStatusHalted,
} StackStatus;
typedef struct StackHead
{
StackNode *node;
StackSummary *summary;
unsigned node_count_at_last_error;
Subtree last_external_token;
Subtree lookahead_when_paused;
StackStatus status;
} StackHead;
struct Stack
{
Array(StackHead) heads;
StackSliceArray slices;
Array(StackIterator) iterators;
StackNodeArray node_pool;
StackNode *base_node;
SubtreePool *subtree_pool;
};
typedef unsigned StackAction;
enum StackAction
{
StackActionNone,
StackActionStop = 1,
StackActionPop = 2,
};
typedef StackAction (*StackCallback)(void *, const StackIterator *);
static void stack_node_retain(StackNode *self)
{
if (!self)
return;
assert(self->ref_count > 0);
self->ref_count++;
assert(self->ref_count != 0);
}
static void stack_node_release(StackNode *self, StackNodeArray *pool, SubtreePool *subtree_pool)
{
recur:
assert(self->ref_count != 0);
self->ref_count--;
if (self->ref_count > 0)
return;
StackNode *first_predecessor = NULL;
if (self->link_count > 0)
{
for (unsigned i = self->link_count - 1; i > 0; i--)
{
StackLink link = self->links[i];
if (link.subtree.ptr)
ts_subtree_release(subtree_pool, link.subtree);
stack_node_release(link.node, pool, subtree_pool);
}
StackLink link = self->links[0];
if (link.subtree.ptr)
ts_subtree_release(subtree_pool, link.subtree);
first_predecessor = self->links[0].node;
}
if (pool->size < MAX_NODE_POOL_SIZE)
{
array_push(pool, self);
}
else
{
ts_free(self);
}
if (first_predecessor)
{
self = first_predecessor;
goto recur;
}
}
/// Get the number of nodes in the subtree, for the purpose of measuring
/// how much progress has been made by a given version of the stack.
static uint32_t stack__subtree_node_count(Subtree subtree)
{
uint32_t count = ts_subtree_visible_descendant_count(subtree);
if (ts_subtree_visible(subtree))
count++;
// Count intermediate error nodes even though they are not visible,
// because a stack version's node count is used to check whether it
// has made any progress since the last time it encountered an error.
if (ts_subtree_symbol(subtree) == ts_builtin_sym_error_repeat)
count++;
return count;
}
static StackNode *stack_node_new(StackNode *previous_node, Subtree subtree, bool is_pending, TSStateId state, StackNodeArray *pool)
{
StackNode *node = pool->size > 0 ? array_pop(pool) : ts_malloc(sizeof(StackNode));
*node = (StackNode){.ref_count = 1, .link_count = 0, .state = state};
if (previous_node)
{
node->link_count = 1;
node->links[0] = (StackLink){
.node = previous_node,
.subtree = subtree,
.is_pending = is_pending,
};
node->position = previous_node->position;
node->error_cost = previous_node->error_cost;
node->dynamic_precedence = previous_node->dynamic_precedence;
node->node_count = previous_node->node_count;
if (subtree.ptr)
{
node->error_cost += ts_subtree_error_cost(subtree);
node->position = length_add(node->position, ts_subtree_total_size(subtree));
node->node_count += stack__subtree_node_count(subtree);
node->dynamic_precedence += ts_subtree_dynamic_precedence(subtree);
}
}
else
{
node->position = length_zero();
node->error_cost = 0;
}
return node;
}
static bool stack__subtree_is_equivalent(Subtree left, Subtree right)
{
if (left.ptr == right.ptr)
return true;
if (!left.ptr || !right.ptr)
return false;
// Symbols must match
if (ts_subtree_symbol(left) != ts_subtree_symbol(right))
return false;
// If both have errors, don't bother keeping both.
if (ts_subtree_error_cost(left) > 0 && ts_subtree_error_cost(right) > 0)
return true;
return (ts_subtree_padding(left).bytes == ts_subtree_padding(right).bytes &&
ts_subtree_size(left).bytes == ts_subtree_size(right).bytes && ts_subtree_child_count(left) == ts_subtree_child_count(right) &&
ts_subtree_extra(left) == ts_subtree_extra(right) && ts_subtree_external_scanner_state_eq(left, right));
}
static void stack_node_add_link(StackNode *self, StackLink link, SubtreePool *subtree_pool)
{
if (link.node == self)
return;
for (int i = 0; i < self->link_count; i++)
{
StackLink *existing_link = &self->links[i];
if (stack__subtree_is_equivalent(existing_link->subtree, link.subtree))
{
// In general, we preserve ambiguities until they are removed from the stack
// during a pop operation where multiple paths lead to the same node. But in
// the special case where two links directly connect the same pair of nodes,
// we can safely remove the ambiguity ahead of time without changing behavior.
if (existing_link->node == link.node)
{
if (ts_subtree_dynamic_precedence(link.subtree) > ts_subtree_dynamic_precedence(existing_link->subtree))
{
ts_subtree_retain(link.subtree);
ts_subtree_release(subtree_pool, existing_link->subtree);
existing_link->subtree = link.subtree;
self->dynamic_precedence = link.node->dynamic_precedence + ts_subtree_dynamic_precedence(link.subtree);
}
return;
}
// If the previous nodes are mergeable, merge them recursively.
if (existing_link->node->state == link.node->state && existing_link->node->position.bytes == link.node->position.bytes &&
existing_link->node->error_cost == link.node->error_cost)
{
for (int j = 0; j < link.node->link_count; j++)
{
stack_node_add_link(existing_link->node, link.node->links[j], subtree_pool);
}
int32_t dynamic_precedence = link.node->dynamic_precedence;
if (link.subtree.ptr)
{
dynamic_precedence += ts_subtree_dynamic_precedence(link.subtree);
}
if (dynamic_precedence > self->dynamic_precedence)
{
self->dynamic_precedence = dynamic_precedence;
}
return;
}
}
}
if (self->link_count == MAX_LINK_COUNT)
return;
stack_node_retain(link.node);
unsigned node_count = link.node->node_count;
int dynamic_precedence = link.node->dynamic_precedence;
self->links[self->link_count++] = link;
if (link.subtree.ptr)
{
ts_subtree_retain(link.subtree);
node_count += stack__subtree_node_count(link.subtree);
dynamic_precedence += ts_subtree_dynamic_precedence(link.subtree);
}
if (node_count > self->node_count)
self->node_count = node_count;
if (dynamic_precedence > self->dynamic_precedence)
self->dynamic_precedence = dynamic_precedence;
}
static void stack_head_delete(StackHead *self, StackNodeArray *pool, SubtreePool *subtree_pool)
{
if (self->node)
{
if (self->last_external_token.ptr)
{
ts_subtree_release(subtree_pool, self->last_external_token);
}
if (self->lookahead_when_paused.ptr)
{
ts_subtree_release(subtree_pool, self->lookahead_when_paused);
}
if (self->summary)
{
array_delete(self->summary);
ts_free(self->summary);
}
stack_node_release(self->node, pool, subtree_pool);
}
}
static StackVersion ts_stack__add_version(Stack *self, StackVersion original_version, StackNode *node)
{
StackHead head = {
.node = node,
.node_count_at_last_error = self->heads.contents[original_version].node_count_at_last_error,
.last_external_token = self->heads.contents[original_version].last_external_token,
.status = StackStatusActive,
.lookahead_when_paused = NULL_SUBTREE,
};
array_push(&self->heads, head);
stack_node_retain(node);
if (head.last_external_token.ptr)
ts_subtree_retain(head.last_external_token);
return (StackVersion)(self->heads.size - 1);
}
static void ts_stack__add_slice(Stack *self, StackVersion original_version, StackNode *node, SubtreeArray *subtrees)
{
for (uint32_t i = self->slices.size - 1; i + 1 > 0; i--)
{
StackVersion version = self->slices.contents[i].version;
if (self->heads.contents[version].node == node)
{
StackSlice slice = {*subtrees, version};
array_insert(&self->slices, i + 1, slice);
return;
}
}
StackVersion version = ts_stack__add_version(self, original_version, node);
StackSlice slice = {*subtrees, version};
array_push(&self->slices, slice);
}
static StackSliceArray stack__iter(Stack *self, StackVersion version, StackCallback callback, void *payload, int goal_subtree_count)
{
array_clear(&self->slices);
array_clear(&self->iterators);
StackHead *head = array_get(&self->heads, version);
StackIterator new_iterator = {
.node = head->node,
.subtrees = array_new(),
.subtree_count = 0,
.is_pending = true,
};
bool include_subtrees = false;
if (goal_subtree_count >= 0)
{
include_subtrees = true;
array_reserve(&new_iterator.subtrees, (uint32_t)ts_subtree_alloc_size(goal_subtree_count) / sizeof(Subtree));
}
array_push(&self->iterators, new_iterator);
while (self->iterators.size > 0)
{
for (uint32_t i = 0, size = self->iterators.size; i < size; i++)
{
StackIterator *iterator = &self->iterators.contents[i];
StackNode *node = iterator->node;
StackAction action = callback(payload, iterator);
bool should_pop = action & StackActionPop;
bool should_stop = action & StackActionStop || node->link_count == 0;
if (should_pop)
{
SubtreeArray subtrees = iterator->subtrees;
if (!should_stop)
{
ts_subtree_array_copy(subtrees, &subtrees);
}
ts_subtree_array_reverse(&subtrees);
ts_stack__add_slice(self, version, node, &subtrees);
}
if (should_stop)
{
if (!should_pop)
{
ts_subtree_array_delete(self->subtree_pool, &iterator->subtrees);
}
array_erase(&self->iterators, i);
i--, size--;
continue;
}
for (uint32_t j = 1; j <= node->link_count; j++)
{
StackIterator *next_iterator;
StackLink link;
if (j == node->link_count)
{
link = node->links[0];
next_iterator = &self->iterators.contents[i];
}
else
{
if (self->iterators.size >= MAX_ITERATOR_COUNT)
continue;
link = node->links[j];
StackIterator current_iterator = self->iterators.contents[i];
array_push(&self->iterators, current_iterator);
next_iterator = array_back(&self->iterators);
ts_subtree_array_copy(next_iterator->subtrees, &next_iterator->subtrees);
}
next_iterator->node = link.node;
if (link.subtree.ptr)
{
if (include_subtrees)
{
array_push(&next_iterator->subtrees, link.subtree);
ts_subtree_retain(link.subtree);
}
if (!ts_subtree_extra(link.subtree))
{
next_iterator->subtree_count++;
if (!link.is_pending)
{
next_iterator->is_pending = false;
}
}
}
else
{
next_iterator->subtree_count++;
next_iterator->is_pending = false;
}
}
}
}
return self->slices;
}
Stack *ts_stack_new(SubtreePool *subtree_pool)
{
Stack *self = ts_calloc(1, sizeof(Stack));
array_init(&self->heads);
array_init(&self->slices);
array_init(&self->iterators);
array_init(&self->node_pool);
array_reserve(&self->heads, 4);
array_reserve(&self->slices, 4);
array_reserve(&self->iterators, 4);
array_reserve(&self->node_pool, MAX_NODE_POOL_SIZE);
self->subtree_pool = subtree_pool;
self->base_node = stack_node_new(NULL, NULL_SUBTREE, false, 1, &self->node_pool);
ts_stack_clear(self);
return self;
}
void ts_stack_delete(Stack *self)
{
if (self->slices.contents)
array_delete(&self->slices);
if (self->iterators.contents)
array_delete(&self->iterators);
stack_node_release(self->base_node, &self->node_pool, self->subtree_pool);
for (uint32_t i = 0; i < self->heads.size; i++)
{
stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool);
}
array_clear(&self->heads);
if (self->node_pool.contents)
{
for (uint32_t i = 0; i < self->node_pool.size; i++)
ts_free(self->node_pool.contents[i]);
array_delete(&self->node_pool);
}
array_delete(&self->heads);
ts_free(self);
}
uint32_t ts_stack_version_count(const Stack *self)
{
return self->heads.size;
}
TSStateId ts_stack_state(const Stack *self, StackVersion version)
{
return array_get(&self->heads, version)->node->state;
}
Length ts_stack_position(const Stack *self, StackVersion version)
{
return array_get(&self->heads, version)->node->position;
}
Subtree ts_stack_last_external_token(const Stack *self, StackVersion version)
{
return array_get(&self->heads, version)->last_external_token;
}
void ts_stack_set_last_external_token(Stack *self, StackVersion version, Subtree token)
{
StackHead *head = array_get(&self->heads, version);
if (token.ptr)
ts_subtree_retain(token);
if (head->last_external_token.ptr)
ts_subtree_release(self->subtree_pool, head->last_external_token);
head->last_external_token = token;
}
unsigned ts_stack_error_cost(const Stack *self, StackVersion version)
{
StackHead *head = array_get(&self->heads, version);
unsigned result = head->node->error_cost;
if (head->status == StackStatusPaused || (head->node->state == ERROR_STATE && !head->node->links[0].subtree.ptr))
{
result += ERROR_COST_PER_RECOVERY;
}
return result;
}
unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version)
{
StackHead *head = array_get(&self->heads, version);
if (head->node->node_count < head->node_count_at_last_error)
{
head->node_count_at_last_error = head->node->node_count;
}
return head->node->node_count - head->node_count_at_last_error;
}
void ts_stack_push(Stack *self, StackVersion version, Subtree subtree, bool pending, TSStateId state)
{
StackHead *head = array_get(&self->heads, version);
StackNode *new_node = stack_node_new(head->node, subtree, pending, state, &self->node_pool);
if (!subtree.ptr)
head->node_count_at_last_error = new_node->node_count;
head->node = new_node;
}
forceinline StackAction pop_count_callback(void *payload, const StackIterator *iterator)
{
unsigned *goal_subtree_count = payload;
if (iterator->subtree_count == *goal_subtree_count)
{
return StackActionPop | StackActionStop;
}
else
{
return StackActionNone;
}
}
StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t count)
{
return stack__iter(self, version, pop_count_callback, &count, (int)count);
}
forceinline StackAction pop_pending_callback(void *payload, const StackIterator *iterator)
{
(void)payload;
if (iterator->subtree_count >= 1)
{
if (iterator->is_pending)
{
return StackActionPop | StackActionStop;
}
else
{
return StackActionStop;
}
}
else
{
return StackActionNone;
}
}
StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version)
{
StackSliceArray pop = stack__iter(self, version, pop_pending_callback, NULL, 0);
if (pop.size > 0)
{
ts_stack_renumber_version(self, pop.contents[0].version, version);
pop.contents[0].version = version;
}
return pop;
}
forceinline StackAction pop_error_callback(void *payload, const StackIterator *iterator)
{
if (iterator->subtrees.size > 0)
{
bool *found_error = payload;
if (!*found_error && ts_subtree_is_error(iterator->subtrees.contents[0]))
{
*found_error = true;
return StackActionPop | StackActionStop;
}
else
{
return StackActionStop;
}
}
else
{
return StackActionNone;
}
}
SubtreeArray ts_stack_pop_error(Stack *self, StackVersion version)
{
StackNode *node = array_get(&self->heads, version)->node;
for (unsigned i = 0; i < node->link_count; i++)
{
if (node->links[i].subtree.ptr && ts_subtree_is_error(node->links[i].subtree))
{
bool found_error = false;
StackSliceArray pop = stack__iter(self, version, pop_error_callback, &found_error, 1);
if (pop.size > 0)
{
assert(pop.size == 1);
ts_stack_renumber_version(self, pop.contents[0].version, version);
return pop.contents[0].subtrees;
}
break;
}
}
return (SubtreeArray){.size = 0};
}
forceinline StackAction pop_all_callback(void *payload, const StackIterator *iterator)
{
(void)payload;
return iterator->node->link_count == 0 ? StackActionPop : StackActionNone;
}
StackSliceArray ts_stack_pop_all(Stack *self, StackVersion version)
{
return stack__iter(self, version, pop_all_callback, NULL, 0);
}
typedef struct SummarizeStackSession
{
StackSummary *summary;
unsigned max_depth;
} SummarizeStackSession;
forceinline StackAction summarize_stack_callback(void *payload, const StackIterator *iterator)
{
SummarizeStackSession *session = payload;
TSStateId state = iterator->node->state;
unsigned depth = iterator->subtree_count;
if (depth > session->max_depth)
return StackActionStop;
for (unsigned i = session->summary->size - 1; i + 1 > 0; i--)
{
StackSummaryEntry entry = session->summary->contents[i];
if (entry.depth < depth)
break;
if (entry.depth == depth && entry.state == state)
return StackActionNone;
}
array_push(session->summary, ((StackSummaryEntry){
.position = iterator->node->position,
.depth = depth,
.state = state,
}));
return StackActionNone;
}
void ts_stack_record_summary(Stack *self, StackVersion version, unsigned max_depth)
{
SummarizeStackSession session = {.summary = ts_malloc(sizeof(StackSummary)), .max_depth = max_depth};
array_init(session.summary);
stack__iter(self, version, summarize_stack_callback, &session, -1);
StackHead *head = &self->heads.contents[version];
if (head->summary)
{
array_delete(head->summary);
ts_free(head->summary);
}
head->summary = session.summary;
}
StackSummary *ts_stack_get_summary(Stack *self, StackVersion version)
{
return array_get(&self->heads, version)->summary;
}
int ts_stack_dynamic_precedence(Stack *self, StackVersion version)
{
return array_get(&self->heads, version)->node->dynamic_precedence;
}
bool ts_stack_has_advanced_since_error(const Stack *self, StackVersion version)
{
const StackHead *head = array_get(&self->heads, version);
const StackNode *node = head->node;
if (node->error_cost == 0)
return true;
while (node)
{
if (node->link_count > 0)
{
Subtree subtree = node->links[0].subtree;
if (subtree.ptr)
{
if (ts_subtree_total_bytes(subtree) > 0)
{
return true;
}
else if (node->node_count > head->node_count_at_last_error && ts_subtree_error_cost(subtree) == 0)
{
node = node->links[0].node;
continue;
}
}
}
break;
}
return false;
}
void ts_stack_remove_version(Stack *self, StackVersion version)
{
stack_head_delete(array_get(&self->heads, version), &self->node_pool, self->subtree_pool);
array_erase(&self->heads, version);
}
void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2)
{
if (v1 == v2)
return;
assert(v2 < v1);
assert((uint32_t)v1 < self->heads.size);
StackHead *source_head = &self->heads.contents[v1];
StackHead *target_head = &self->heads.contents[v2];
if (target_head->summary && !source_head->summary)
{
source_head->summary = target_head->summary;
target_head->summary = NULL;
}
stack_head_delete(target_head, &self->node_pool, self->subtree_pool);
*target_head = *source_head;
array_erase(&self->heads, v1);
}
void ts_stack_swap_versions(Stack *self, StackVersion v1, StackVersion v2)
{
StackHead temporary_head = self->heads.contents[v1];
self->heads.contents[v1] = self->heads.contents[v2];
self->heads.contents[v2] = temporary_head;
}
StackVersion ts_stack_copy_version(Stack *self, StackVersion version)
{
assert(version < self->heads.size);
array_push(&self->heads, self->heads.contents[version]);
StackHead *head = array_back(&self->heads);
stack_node_retain(head->node);
if (head->last_external_token.ptr)
ts_subtree_retain(head->last_external_token);
head->summary = NULL;
return self->heads.size - 1;
}
bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2)
{
if (!ts_stack_can_merge(self, version1, version2))
return false;
StackHead *head1 = &self->heads.contents[version1];
StackHead *head2 = &self->heads.contents[version2];
for (uint32_t i = 0; i < head2->node->link_count; i++)
{
stack_node_add_link(head1->node, head2->node->links[i], self->subtree_pool);
}
if (head1->node->state == ERROR_STATE)
{
head1->node_count_at_last_error = head1->node->node_count;
}
ts_stack_remove_version(self, version2);
return true;
}
bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version2)
{
StackHead *head1 = &self->heads.contents[version1];
StackHead *head2 = &self->heads.contents[version2];
return head1->status == StackStatusActive && head2->status == StackStatusActive && head1->node->state == head2->node->state &&
head1->node->position.bytes == head2->node->position.bytes && head1->node->error_cost == head2->node->error_cost &&
ts_subtree_external_scanner_state_eq(head1->last_external_token, head2->last_external_token);
}
void ts_stack_halt(Stack *self, StackVersion version)
{
array_get(&self->heads, version)->status = StackStatusHalted;
}
void ts_stack_pause(Stack *self, StackVersion version, Subtree lookahead)
{
StackHead *head = array_get(&self->heads, version);
head->status = StackStatusPaused;
head->lookahead_when_paused = lookahead;
head->node_count_at_last_error = head->node->node_count;
}
bool ts_stack_is_active(const Stack *self, StackVersion version)
{
return array_get(&self->heads, version)->status == StackStatusActive;
}
bool ts_stack_is_halted(const Stack *self, StackVersion version)
{
return array_get(&self->heads, version)->status == StackStatusHalted;
}
bool ts_stack_is_paused(const Stack *self, StackVersion version)
{
return array_get(&self->heads, version)->status == StackStatusPaused;
}
Subtree ts_stack_resume(Stack *self, StackVersion version)
{
StackHead *head = array_get(&self->heads, version);
assert(head->status == StackStatusPaused);
Subtree result = head->lookahead_when_paused;
head->status = StackStatusActive;
head->lookahead_when_paused = NULL_SUBTREE;
return result;
}
void ts_stack_clear(Stack *self)
{
stack_node_retain(self->base_node);
for (uint32_t i = 0; i < self->heads.size; i++)
{
stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool);
}
array_clear(&self->heads);
array_push(&self->heads, ((StackHead){
.node = self->base_node,
.status = StackStatusActive,
.last_external_token = NULL_SUBTREE,
.lookahead_when_paused = NULL_SUBTREE,
}));
}
bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
{
array_reserve(&self->iterators, 32);
if (!f)
f = stderr;
fprintf(f, "digraph stack {\n");
fprintf(f, "rankdir=\"RL\";\n");
fprintf(f, "edge [arrowhead=none]\n");
Array(StackNode *) visited_nodes = array_new();
array_clear(&self->iterators);
for (uint32_t i = 0; i < self->heads.size; i++)
{
StackHead *head = &self->heads.contents[i];
if (head->status == StackStatusHalted)
continue;
fprintf(f, "node_head_%u [shape=none, label=\"\"]\n", i);
fprintf(f, "node_head_%u -> node_%p [", i, (void *)head->node);
if (head->status == StackStatusPaused)
{
fprintf(f, "color=red ");
}
fprintf(f, "label=%u, fontcolor=blue, weight=10000, labeltooltip=\"node_count: %u\nerror_cost: %u", i,
ts_stack_node_count_since_error(self, i), ts_stack_error_cost(self, i));
if (head->summary)
{
fprintf(f, "\nsummary:");
for (uint32_t j = 0; j < head->summary->size; j++)
fprintf(f, " %u", head->summary->contents[j].state);
}
if (head->last_external_token.ptr)
{
const ExternalScannerState *state = &head->last_external_token.ptr->inner.external_scanner_state;
const char *data = ts_external_scanner_state_data(state);
fprintf(f, "\nexternal_scanner_state:");
for (uint32_t j = 0; j < state->length; j++)
fprintf(f, " %2X", data[j]);
}
fprintf(f, "\"]\n");
array_push(&self->iterators, ((StackIterator){.node = head->node}));
}
bool all_iterators_done = false;
while (!all_iterators_done)
{
all_iterators_done = true;
for (uint32_t i = 0; i < self->iterators.size; i++)
{
StackIterator iterator = self->iterators.contents[i];
StackNode *node = iterator.node;
for (uint32_t j = 0; j < visited_nodes.size; j++)
{
if (visited_nodes.contents[j] == node)
{
node = NULL;
break;
}
}
if (!node)
continue;
all_iterators_done = false;
fprintf(f, "node_%p [", (void *)node);
if (node->state == ERROR_STATE)
{
fprintf(f, "label=\"?\"");
}
else if (node->link_count == 1 && node->links[0].subtree.ptr && ts_subtree_extra(node->links[0].subtree))
{
fprintf(f, "shape=point margin=0 label=\"\"");
}
else
{
fprintf(f, "label=\"%d\"", node->state);
}
fprintf(f, " tooltip=\"position: %u,%u\nnode_count:%u\nerror_cost: %u\ndynamic_precedence: %d\"];\n",
node->position.extent.row + 1, node->position.extent.column, node->node_count, node->error_cost,
node->dynamic_precedence);
for (int j = 0; j < node->link_count; j++)
{
StackLink link = node->links[j];
fprintf(f, "node_%p -> node_%p [", (void *)node, (void *)link.node);
if (link.is_pending)
fprintf(f, "style=dashed ");
if (link.subtree.ptr && ts_subtree_extra(link.subtree))
fprintf(f, "fontcolor=gray ");
if (!link.subtree.ptr)
{
fprintf(f, "color=red");
}
else
{
fprintf(f, "label=\"");
bool quoted = ts_subtree_visible(link.subtree) && !ts_subtree_named(link.subtree);
if (quoted)
fprintf(f, "'");
ts_language_write_symbol_as_dot_string(language, f, ts_subtree_symbol(link.subtree));
if (quoted)
fprintf(f, "'");
fprintf(f, "\"");
fprintf(f, "labeltooltip=\"error_cost: %u\ndynamic_precedence: %" PRId32 "\"", ts_subtree_error_cost(link.subtree),
ts_subtree_dynamic_precedence(link.subtree));
}
fprintf(f, "];\n");
StackIterator *next_iterator;
if (j == 0)
{
next_iterator = &self->iterators.contents[i];
}
else
{
array_push(&self->iterators, iterator);
next_iterator = array_back(&self->iterators);
}
next_iterator->node = link.node;
}
array_push(&visited_nodes, node);
}
}
fprintf(f, "}\n");
array_delete(&visited_nodes);
return true;
}
#undef forceinline

136
parser/nsrc/stack.h
View file

@ -1,136 +0,0 @@
#ifndef TREE_SITTER_PARSE_STACK_H_
#define TREE_SITTER_PARSE_STACK_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include "./array.h"
#include "./error_costs.h"
#include "./subtree.h"
#include <stdio.h>
typedef struct Stack Stack;
typedef unsigned StackVersion;
#define STACK_VERSION_NONE ((StackVersion)-1)
typedef struct StackSlice
{
SubtreeArray subtrees;
StackVersion version;
} StackSlice;
typedef Array(StackSlice) StackSliceArray;
typedef struct StackSummaryEntry
{
Length position;
unsigned depth;
TSStateId state;
} StackSummaryEntry;
typedef Array(StackSummaryEntry) StackSummary;
// Create a stack.
Stack *ts_stack_new(SubtreePool *);
// Release the memory reserved for a given stack.
void ts_stack_delete(Stack *);
// Get the stack's current number of versions.
uint32_t ts_stack_version_count(const Stack *);
// Get the state at the top of the given version of the stack. If the stack is
// empty, this returns the initial state, 0.
TSStateId ts_stack_state(const Stack *, StackVersion);
// Get the last external token associated with a given version of the stack.
Subtree ts_stack_last_external_token(const Stack *, StackVersion);
// Set the last external token associated with a given version of the stack.
void ts_stack_set_last_external_token(Stack *, StackVersion, Subtree);
// Get the position of the given version of the stack within the document.
Length ts_stack_position(const Stack *, StackVersion);
// Push a tree and state onto the given version of the stack.
//
// This transfers ownership of the tree to the Stack. Callers that
// need to retain ownership of the tree for their own purposes should
// first retain the tree.
void ts_stack_push(Stack *, StackVersion, Subtree, bool, TSStateId);
// Pop the given number of entries from the given version of the stack. This
// operation can increase the number of stack versions by revealing multiple
// versions which had previously been merged. It returns an array that
// specifies the index of each revealed version and the trees that were
// removed from that version.
StackSliceArray ts_stack_pop_count(Stack *, StackVersion, uint32_t count);
// Remove an error at the top of the given version of the stack.
SubtreeArray ts_stack_pop_error(Stack *, StackVersion);
// Remove any pending trees from the top of the given version of the stack.
StackSliceArray ts_stack_pop_pending(Stack *, StackVersion);
// Remove any all trees from the given version of the stack.
StackSliceArray ts_stack_pop_all(Stack *, StackVersion);
// Get the maximum number of tree nodes reachable from this version of the stack
// since the last error was detected.
unsigned ts_stack_node_count_since_error(const Stack *, StackVersion);
int ts_stack_dynamic_precedence(Stack *, StackVersion);
bool ts_stack_has_advanced_since_error(const Stack *, StackVersion);
// Compute a summary of all the parse states near the top of the given
// version of the stack and store the summary for later retrieval.
void ts_stack_record_summary(Stack *, StackVersion, unsigned max_depth);
// Retrieve a summary of all the parse states near the top of the
// given version of the stack.
StackSummary *ts_stack_get_summary(Stack *, StackVersion);
// Get the total cost of all errors on the given version of the stack.
unsigned ts_stack_error_cost(const Stack *, StackVersion version);
// Merge the given two stack versions if possible, returning true
// if they were successfully merged and false otherwise.
bool ts_stack_merge(Stack *, StackVersion, StackVersion);
// Determine whether the given two stack versions can be merged.
bool ts_stack_can_merge(Stack *, StackVersion, StackVersion);
Subtree ts_stack_resume(Stack *, StackVersion);
void ts_stack_pause(Stack *, StackVersion, Subtree);
void ts_stack_halt(Stack *, StackVersion);
bool ts_stack_is_active(const Stack *, StackVersion);
bool ts_stack_is_paused(const Stack *, StackVersion);
bool ts_stack_is_halted(const Stack *, StackVersion);
void ts_stack_renumber_version(Stack *, StackVersion, StackVersion);
void ts_stack_swap_versions(Stack *, StackVersion, StackVersion);
StackVersion ts_stack_copy_version(Stack *, StackVersion);
// Remove the given version from the stack.
void ts_stack_remove_version(Stack *, StackVersion);
void ts_stack_clear(Stack *);
bool ts_stack_print_dot_graph(Stack *, const TSLanguage *, FILE *);
typedef void (*StackIterateCallback)(void *, TSStateId, uint32_t);
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_PARSE_STACK_H_

1108
parser/nsrc/subtree.c
View file

File diff suppressed because it is too large Load diff

456
parser/nsrc/subtree.h
View file

@ -1,456 +0,0 @@
#ifndef TREE_SITTER_SUBTREE_H_
#define TREE_SITTER_SUBTREE_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include "./api.h"
#include "./array.h"
#include "./error_costs.h"
#include "./host.h"
#include "./length.h"
#include "./parser.h"
#include <limits.h>
#include <stdbool.h>
#include <stdio.h>
#define TS_TREE_STATE_NONE USHRT_MAX
#define NULL_SUBTREE ((Subtree){.ptr = NULL})
// The serialized state of an external scanner.
//
// Every time an external token subtree is created after a call to an
// external scanner, the scanner's `serialize` function is called to
// retrieve a serialized copy of its state. The bytes are then copied
// onto the subtree itself so that the scanner's state can later be
// restored using its `deserialize` function.
//
// Small byte arrays are stored inline, and long ones are allocated
// separately on the heap.
typedef struct ExternalScannerState
{
union ExternalScannerStateData {
char *long_data;
char short_data[24];
} data;
uint32_t length;
} ExternalScannerState;
// A compact representation of a subtree.
//
// This representation is used for small leaf nodes that are not
// errors, and were not created by an external scanner.
//
// The idea behind the layout of this struct is that the `is_inline`
// bit will fall exactly into the same location as the least significant
// bit of the pointer in `Subtree` or `MutableSubtree`, respectively.
// Because of alignment, for any valid pointer this will be 0, giving
// us the opportunity to make use of this bit to signify whether to use
// the pointer or the inline struct.
typedef struct SubtreeInlineData SubtreeInlineData;
#define SUBTREE_BITS \
bool visible : 1; \
bool named : 1; \
bool extra : 1; \
bool has_changes : 1; \
bool is_missing : 1; \
bool is_keyword : 1;
#define SUBTREE_SIZE \
uint8_t padding_columns; \
uint8_t padding_rows : 4; \
uint8_t lookahead_bytes : 4; \
uint8_t padding_bytes; \
uint8_t size_bytes;
#if TS_BIG_ENDIAN
# if TS_PTR_SIZE == 32
struct SubtreeInlineData
{
uint16_t parse_state;
uint8_t symbol;
SUBTREE_BITS
bool unused : 1;
bool is_inline : 1;
SUBTREE_SIZE
};
# else
struct SubtreeInlineData
{
SUBTREE_SIZE
uint16_t parse_state;
uint8_t symbol;
SUBTREE_BITS
bool unused : 1;
bool is_inline : 1;
};
# endif
#else
struct SubtreeInlineData
{
bool is_inline : 1;
SUBTREE_BITS
uint8_t symbol;
uint16_t parse_state;
SUBTREE_SIZE
};
#endif
#undef SUBTREE_BITS
#undef SUBTREE_SIZE
// A heap-allocated representation of a subtree.
//
// This representation is used for parent nodes, external tokens,
// errors, and other leaf nodes whose data is too large to fit into
// the inline representation.
typedef struct SubtreeHeapData
{
volatile uint32_t ref_count;
Length padding;
Length size;
uint32_t lookahead_bytes;
uint32_t error_cost;
uint32_t child_count;
TSSymbol symbol;
TSStateId parse_state;
bool visible : 1;
bool named : 1;
bool extra : 1;
bool fragile_left : 1;
bool fragile_right : 1;
bool has_changes : 1;
bool has_external_tokens : 1;
bool has_external_scanner_state_change : 1;
bool depends_on_column : 1;
bool is_missing : 1;
bool is_keyword : 1;
union SubtreeHeapDataInner {
// Non-terminal subtrees (`child_count > 0`)
struct SubtreeHeapDataInnerNonTerminal
{
uint32_t visible_child_count;
uint32_t named_child_count;
uint32_t visible_descendant_count;
int32_t dynamic_precedence;
uint16_t repeat_depth;
uint16_t production_id;
struct SubtreeHeapDataInnerNonTerminalFirstLeaf
{
TSSymbol symbol;
TSStateId parse_state;
} first_leaf;
} non_terminal;
// External terminal subtrees (`child_count == 0 && has_external_tokens`)
ExternalScannerState external_scanner_state;
// Error terminal subtrees (`child_count == 0 && symbol == ts_builtin_sym_error`)
int32_t lookahead_char;
} inner;
} SubtreeHeapData;
// The fundamental building block of a syntax tree.
typedef union Subtree {
SubtreeInlineData data;
const SubtreeHeapData *ptr;
} Subtree;
// Like Subtree, but mutable.
typedef union MutableSubtree {
SubtreeInlineData data;
SubtreeHeapData *ptr;
} MutableSubtree;
typedef Array(Subtree) SubtreeArray;
typedef Array(MutableSubtree) MutableSubtreeArray;
typedef struct SubtreePool
{
MutableSubtreeArray free_trees;
MutableSubtreeArray tree_stack;
} SubtreePool;
void ts_external_scanner_state_init(ExternalScannerState *, const char *, unsigned);
const char *ts_external_scanner_state_data(const ExternalScannerState *);
bool ts_external_scanner_state_eq(const ExternalScannerState *self, const char *, unsigned);
void ts_external_scanner_state_delete(ExternalScannerState *self);
void ts_subtree_array_copy(SubtreeArray, SubtreeArray *);
void ts_subtree_array_clear(SubtreePool *, SubtreeArray *);
void ts_subtree_array_delete(SubtreePool *, SubtreeArray *);
void ts_subtree_array_remove_trailing_extras(SubtreeArray *, SubtreeArray *);
void ts_subtree_array_reverse(SubtreeArray *);
SubtreePool ts_subtree_pool_new(uint32_t capacity);
void ts_subtree_pool_delete(SubtreePool *);
Subtree ts_subtree_new_leaf(SubtreePool *, TSSymbol, Length, Length, uint32_t, TSStateId, bool, bool, bool, const TSLanguage *);
Subtree ts_subtree_new_error(SubtreePool *, int32_t, Length, Length, uint32_t, TSStateId, const TSLanguage *);
MutableSubtree ts_subtree_new_node(TSSymbol, SubtreeArray *, unsigned, const TSLanguage *);
Subtree ts_subtree_new_error_node(SubtreeArray *, bool, const TSLanguage *);
Subtree ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, Length, uint32_t, const TSLanguage *);
MutableSubtree ts_subtree_make_mut(SubtreePool *, Subtree);
void ts_subtree_retain(Subtree);
void ts_subtree_release(SubtreePool *, Subtree);
int ts_subtree_compare(Subtree, Subtree, SubtreePool *);
void ts_subtree_set_symbol(MutableSubtree *, TSSymbol, const TSLanguage *);
void ts_subtree_summarize(MutableSubtree, const Subtree *, uint32_t, const TSLanguage *);
void ts_subtree_summarize_children(MutableSubtree, const TSLanguage *);
void ts_subtree_balance(Subtree, SubtreePool *, const TSLanguage *);
Subtree ts_subtree_edit(Subtree, const TSInputEdit *edit, SubtreePool *);
char *ts_subtree_string(Subtree, TSSymbol, bool, const TSLanguage *, bool include_all);
void ts_subtree_print_dot_graph(Subtree, const TSLanguage *, FILE *);
Subtree ts_subtree_last_external_token(Subtree);
const ExternalScannerState *ts_subtree_external_scanner_state(Subtree self);
bool ts_subtree_external_scanner_state_eq(Subtree, Subtree);
#define SUBTREE_GET(self, name) ((self).data.is_inline ? (self).data.name : (self).ptr->name)
static inline TSSymbol ts_subtree_symbol(Subtree self)
{
return SUBTREE_GET(self, symbol);
}
static inline bool ts_subtree_visible(Subtree self)
{
return SUBTREE_GET(self, visible);
}
static inline bool ts_subtree_named(Subtree self)
{
return SUBTREE_GET(self, named);
}
static inline bool ts_subtree_extra(Subtree self)
{
return SUBTREE_GET(self, extra);
}
static inline bool ts_subtree_has_changes(Subtree self)
{
return SUBTREE_GET(self, has_changes);
}
static inline bool ts_subtree_missing(Subtree self)
{
return SUBTREE_GET(self, is_missing);
}
static inline bool ts_subtree_is_keyword(Subtree self)
{
return SUBTREE_GET(self, is_keyword);
}
static inline TSStateId ts_subtree_parse_state(Subtree self)
{
return SUBTREE_GET(self, parse_state);
}
static inline uint32_t ts_subtree_lookahead_bytes(Subtree self)
{
return SUBTREE_GET(self, lookahead_bytes);
}
#undef SUBTREE_GET
// Get the size needed to store a heap-allocated subtree with the given
// number of children.
static inline size_t ts_subtree_alloc_size(uint32_t child_count)
{
return child_count * sizeof(Subtree) + sizeof(SubtreeHeapData);
}
// Get a subtree's children, which are allocated immediately before the
// tree's own heap data.
#define ts_subtree_children(self) ((self).data.is_inline ? NULL : (Subtree *)((self).ptr) - (self).ptr->child_count)
static inline void ts_subtree_set_extra(MutableSubtree *self, bool is_extra)
{
if (self->data.is_inline)
{
self->data.extra = is_extra;
}
else
{
self->ptr->extra = is_extra;
}
}
static inline TSSymbol ts_subtree_leaf_symbol(Subtree self)
{
if (self.data.is_inline)
return self.data.symbol;
if (self.ptr->child_count == 0)
return self.ptr->symbol;
return self.ptr->inner.non_terminal.first_leaf.symbol;
}
static inline TSStateId ts_subtree_leaf_parse_state(Subtree self)
{
if (self.data.is_inline)
return self.data.parse_state;
if (self.ptr->child_count == 0)
return self.ptr->parse_state;
return self.ptr->inner.non_terminal.first_leaf.parse_state;
}
static inline Length ts_subtree_padding(Subtree self)
{
if (self.data.is_inline)
{
Length result = {self.data.padding_bytes, {self.data.padding_rows, self.data.padding_columns}};
return result;
}
else
{
return self.ptr->padding;
}
}
static inline Length ts_subtree_size(Subtree self)
{
if (self.data.is_inline)
{
Length result = {self.data.size_bytes, {0, self.data.size_bytes}};
return result;
}
else
{
return self.ptr->size;
}
}
static inline Length ts_subtree_total_size(Subtree self)
{
return length_add(ts_subtree_padding(self), ts_subtree_size(self));
}
static inline uint32_t ts_subtree_total_bytes(Subtree self)
{
return ts_subtree_total_size(self).bytes;
}
static inline uint32_t ts_subtree_child_count(Subtree self)
{
return self.data.is_inline ? 0 : self.ptr->child_count;
}
static inline uint32_t ts_subtree_repeat_depth(Subtree self)
{
return self.data.is_inline ? 0 : self.ptr->inner.non_terminal.repeat_depth;
}
static inline uint32_t ts_subtree_is_repetition(Subtree self)
{
return self.data.is_inline ? 0 : !self.ptr->named && !self.ptr->visible && self.ptr->child_count != 0;
}
static inline uint32_t ts_subtree_visible_descendant_count(Subtree self)
{
return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->inner.non_terminal.visible_descendant_count;
}
static inline uint32_t ts_subtree_visible_child_count(Subtree self)
{
if (ts_subtree_child_count(self) > 0)
{
return self.ptr->inner.non_terminal.visible_child_count;
}
else
{
return 0;
}
}
static inline uint32_t ts_subtree_error_cost(Subtree self)
{
if (ts_subtree_missing(self))
{
return ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY;
}
else
{
return self.data.is_inline ? 0 : self.ptr->error_cost;
}
}
static inline int32_t ts_subtree_dynamic_precedence(Subtree self)
{
return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->inner.non_terminal.dynamic_precedence;
}
static inline uint16_t ts_subtree_production_id(Subtree self)
{
if (ts_subtree_child_count(self) > 0)
{
return self.ptr->inner.non_terminal.production_id;
}
else
{
return 0;
}
}
static inline bool ts_subtree_fragile_left(Subtree self)
{
return self.data.is_inline ? false : self.ptr->fragile_left;
}
static inline bool ts_subtree_fragile_right(Subtree self)
{
return self.data.is_inline ? false : self.ptr->fragile_right;
}
static inline bool ts_subtree_has_external_tokens(Subtree self)
{
return self.data.is_inline ? false : self.ptr->has_external_tokens;
}
static inline bool ts_subtree_has_external_scanner_state_change(Subtree self)
{
return self.data.is_inline ? false : self.ptr->has_external_scanner_state_change;
}
static inline bool ts_subtree_depends_on_column(Subtree self)
{
return self.data.is_inline ? false : self.ptr->depends_on_column;
}
static inline bool ts_subtree_is_fragile(Subtree self)
{
return self.data.is_inline ? false : (self.ptr->fragile_left || self.ptr->fragile_right);
}
static inline bool ts_subtree_is_error(Subtree self)
{
return ts_subtree_symbol(self) == ts_builtin_sym_error;
}
static inline bool ts_subtree_is_eof(Subtree self)
{
return ts_subtree_symbol(self) == ts_builtin_sym_end;
}
static inline Subtree ts_subtree_from_mut(MutableSubtree self)
{
Subtree result;
result.data = self.data;
return result;
}
static inline MutableSubtree ts_subtree_to_mut_unsafe(Subtree self)
{
MutableSubtree result;
result.data = self.data;
return result;
}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_SUBTREE_H_

165
parser/nsrc/tree.c
View file

@ -1,165 +0,0 @@
#define _POSIX_C_SOURCE 200112L
#include "./api.h"
#include "./array.h"
#include "./get_changed_ranges.h"
#include "./length.h"
#include "./subtree.h"
#include "./tree_cursor.h"
#include "./tree.h"
TSTree *ts_tree_new(
Subtree root, const TSLanguage *language,
const TSRange *included_ranges, unsigned included_range_count
) {
TSTree *result = ts_malloc(sizeof(TSTree));
result->root = root;
result->language = ts_language_copy(language);
result->included_ranges = ts_calloc(included_range_count, sizeof(TSRange));
memcpy(result->included_ranges, included_ranges, included_range_count * sizeof(TSRange));
result->included_range_count = included_range_count;
return result;
}
TSTree *ts_tree_copy(const TSTree *self) {
ts_subtree_retain(self->root);
return ts_tree_new(self->root, self->language, self->included_ranges, self->included_range_count);
}
void ts_tree_delete(TSTree *self) {
if (!self) return;
SubtreePool pool = ts_subtree_pool_new(0);
ts_subtree_release(&pool, self->root);
ts_subtree_pool_delete(&pool);
ts_language_delete(self->language);
ts_free(self->included_ranges);
ts_free(self);
}
TSNode ts_tree_root_node(const TSTree *self) {
return ts_node_new(self, &self->root, ts_subtree_padding(self->root), 0);
}
TSNode ts_tree_root_node_with_offset(
const TSTree *self,
uint32_t offset_bytes,
TSPoint offset_extent
) {
Length offset = {offset_bytes, offset_extent};
return ts_node_new(self, &self->root, length_add(offset, ts_subtree_padding(self->root)), 0);
}
const TSLanguage *ts_tree_language(const TSTree *self) {
return self->language;
}
void ts_tree_edit(TSTree *self, const TSInputEdit *edit) {
for (unsigned i = 0; i < self->included_range_count; i++) {
TSRange *range = &self->included_ranges[i];
if (range->end_byte >= edit->old_end_byte) {
if (range->end_byte != UINT32_MAX) {
range->end_byte = edit->new_end_byte + (range->end_byte - edit->old_end_byte);
range->end_point = point_add(
edit->new_end_point,
point_sub(range->end_point, edit->old_end_point)
);
if (range->end_byte < edit->new_end_byte) {
range->end_byte = UINT32_MAX;
range->end_point = POINT_MAX;
}
}
} else if (range->end_byte > edit->start_byte) {
range->end_byte = edit->start_byte;
range->end_point = edit->start_point;
}
if (range->start_byte >= edit->old_end_byte) {
range->start_byte = edit->new_end_byte + (range->start_byte - edit->old_end_byte);
range->start_point = point_add(
edit->new_end_point,
point_sub(range->start_point, edit->old_end_point)
);
if (range->start_byte < edit->new_end_byte) {
range->start_byte = UINT32_MAX;
range->start_point = POINT_MAX;
}
} else if (range->start_byte > edit->start_byte) {
range->start_byte = edit->start_byte;
range->start_point = edit->start_point;
}
}
SubtreePool pool = ts_subtree_pool_new(0);
self->root = ts_subtree_edit(self->root, edit, &pool);
ts_subtree_pool_delete(&pool);
}
TSRange *ts_tree_included_ranges(const TSTree *self, uint32_t *length) {
*length = self->included_range_count;
TSRange *ranges = ts_calloc(self->included_range_count, sizeof(TSRange));
memcpy(ranges, self->included_ranges, self->included_range_count * sizeof(TSRange));
return ranges;
}
TSRange *ts_tree_get_changed_ranges(const TSTree *old_tree, const TSTree *new_tree, uint32_t *length) {
TreeCursor cursor1 = {NULL, array_new(), 0};
TreeCursor cursor2 = {NULL, array_new(), 0};
ts_tree_cursor_init(&cursor1, ts_tree_root_node(old_tree));
ts_tree_cursor_init(&cursor2, ts_tree_root_node(new_tree));
TSRangeArray included_range_differences = array_new();
ts_range_array_get_changed_ranges(
old_tree->included_ranges, old_tree->included_range_count,
new_tree->included_ranges, new_tree->included_range_count,
&included_range_differences
);
TSRange *result;
*length = ts_subtree_get_changed_ranges(
&old_tree->root, &new_tree->root, &cursor1, &cursor2,
old_tree->language, &included_range_differences, &result
);
array_delete(&included_range_differences);
array_delete(&cursor1.stack);
array_delete(&cursor2.stack);
return result;
}
#ifdef _WIN32
#include <io.h>
#include <windows.h>
int _ts_dup(HANDLE handle) {
HANDLE dup_handle;
if (!DuplicateHandle(
GetCurrentProcess(), handle,
GetCurrentProcess(), &dup_handle,
0, FALSE, DUPLICATE_SAME_ACCESS
)) return -1;
return _open_osfhandle((intptr_t)dup_handle, 0);
}
void ts_tree_print_dot_graph(const TSTree *self, int fd) {
FILE *file = _fdopen(_ts_dup((HANDLE)_get_osfhandle(fd)), "a");
ts_subtree_print_dot_graph(self->root, self->language, file);
fclose(file);
}
#else
#include <unistd.h>
int _ts_dup(int file_descriptor) {
return dup(file_descriptor);
}
void ts_tree_print_dot_graph(const TSTree *self, int file_descriptor) {
FILE *file = fdopen(_ts_dup(file_descriptor), "a");
ts_subtree_print_dot_graph(self->root, self->language, file);
fclose(file);
}
#endif

34
parser/nsrc/tree.h
View file

@ -1,34 +0,0 @@
#ifndef TREE_SITTER_TREE_H_
#define TREE_SITTER_TREE_H_
#include "./subtree.h"
#ifdef __cplusplus
extern "C"
{
#endif
typedef struct ParentCacheEntry
{
const Subtree *child;
const Subtree *parent;
Length position;
TSSymbol alias_symbol;
} ParentCacheEntry;
struct TSTree
{
Subtree root;
const TSLanguage *language;
TSRange *included_ranges;
unsigned included_range_count;
};
TSTree *ts_tree_new(Subtree root, const TSLanguage *language, const TSRange *, unsigned);
TSNode ts_node_new(const TSTree *, const Subtree *, Length, TSSymbol);
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_TREE_H_

714
parser/nsrc/tree_cursor.c
View file

@ -1,714 +0,0 @@
#include "./api.h"
#include "./alloc.h"
#include "./tree_cursor.h"
#include "./language.h"
#include "./tree.h"
typedef struct CursorChildIterator{
Subtree parent;
const TSTree *tree;
Length position;
uint32_t child_index;
uint32_t structural_child_index;
uint32_t descendant_index;
const TSSymbol *alias_sequence;
} CursorChildIterator;
// CursorChildIterator
static inline bool ts_tree_cursor_is_entry_visible(const TreeCursor *self, uint32_t index) {
TreeCursorEntry *entry = &self->stack.contents[index];
if (index == 0 || ts_subtree_visible(*entry->subtree)) {
return true;
} else if (!ts_subtree_extra(*entry->subtree)) {
TreeCursorEntry *parent_entry = &self->stack.contents[index - 1];
return ts_language_alias_at(
self->tree->language,
parent_entry->subtree->ptr->inner.non_terminal.production_id,
entry->structural_child_index
);
} else {
return false;
}
}
static inline CursorChildIterator ts_tree_cursor_iterate_children(const TreeCursor *self) {
TreeCursorEntry *last_entry = array_back(&self->stack);
if (ts_subtree_child_count(*last_entry->subtree) == 0) {
return (CursorChildIterator) {NULL_SUBTREE, self->tree, length_zero(), 0, 0, 0, NULL};
}
const TSSymbol *alias_sequence = ts_language_alias_sequence(
self->tree->language,
last_entry->subtree->ptr->inner.non_terminal.production_id
);
uint32_t descendant_index = last_entry->descendant_index;
if (ts_tree_cursor_is_entry_visible(self, self->stack.size - 1)) {
descendant_index += 1;
}
return (CursorChildIterator) {
.tree = self->tree,
.parent = *last_entry->subtree,
.position = last_entry->position,
.child_index = 0,
.structural_child_index = 0,
.descendant_index = descendant_index,
.alias_sequence = alias_sequence,
};
}
static inline bool ts_tree_cursor_child_iterator_next(
CursorChildIterator *self,
TreeCursorEntry *result,
bool *visible
) {
if (!self->parent.ptr || self->child_index == self->parent.ptr->child_count) return false;
const Subtree *child = &ts_subtree_children(self->parent)[self->child_index];
*result = (TreeCursorEntry) {
.subtree = child,
.position = self->position,
.child_index = self->child_index,
.structural_child_index = self->structural_child_index,
.descendant_index = self->descendant_index,
};
*visible = ts_subtree_visible(*child);
bool extra = ts_subtree_extra(*child);
if (!extra) {
if (self->alias_sequence) {
*visible |= self->alias_sequence[self->structural_child_index];
}
self->structural_child_index++;
}
self->descendant_index += ts_subtree_visible_descendant_count(*child);
if (*visible) {
self->descendant_index += 1;
}
self->position = length_add(self->position, ts_subtree_size(*child));
self->child_index++;
if (self->child_index < self->parent.ptr->child_count) {
Subtree next_child = ts_subtree_children(self->parent)[self->child_index];
self->position = length_add(self->position, ts_subtree_padding(next_child));
}
return true;
}
// Return a position that, when `b` is added to it, yields `a`. This
// can only be computed if `b` has zero rows. Otherwise, this function
// returns `LENGTH_UNDEFINED`, and the caller needs to recompute
// the position some other way.
static inline Length length_backtrack(Length a, Length b) {
if (length_is_undefined(a) || b.extent.row != 0) {
return LENGTH_UNDEFINED;
}
Length result;
result.bytes = a.bytes - b.bytes;
result.extent.row = a.extent.row;
result.extent.column = a.extent.column - b.extent.column;
return result;
}
static inline bool ts_tree_cursor_child_iterator_previous(
CursorChildIterator *self,
TreeCursorEntry *result,
bool *visible
) {
// this is mostly a reverse `ts_tree_cursor_child_iterator_next` taking into
// account unsigned underflow
if (!self->parent.ptr || (int8_t)self->child_index == -1) return false;
const Subtree *child = &ts_subtree_children(self->parent)[self->child_index];
*result = (TreeCursorEntry) {
.subtree = child,
.position = self->position,
.child_index = self->child_index,
.structural_child_index = self->structural_child_index,
};
*visible = ts_subtree_visible(*child);
bool extra = ts_subtree_extra(*child);
if (!extra && self->alias_sequence) {
*visible |= self->alias_sequence[self->structural_child_index];
self->structural_child_index--;
}
self->position = length_backtrack(self->position, ts_subtree_padding(*child));
self->child_index--;
// unsigned can underflow so compare it to child_count
if (self->child_index < self->parent.ptr->child_count) {
Subtree previous_child = ts_subtree_children(self->parent)[self->child_index];
Length size = ts_subtree_size(previous_child);
self->position = length_backtrack(self->position, size);
}
return true;
}
// TSTreeCursor - lifecycle
TSTreeCursor ts_tree_cursor_new(TSNode node) {
TSTreeCursor self = {NULL, NULL, {0, 0, 0}};
ts_tree_cursor_init((TreeCursor *)&self, node);
return self;
}
void ts_tree_cursor_reset(TSTreeCursor *_self, TSNode node) {
ts_tree_cursor_init((TreeCursor *)_self, node);
}
void ts_tree_cursor_init(TreeCursor *self, TSNode node) {
self->tree = node.tree;
self->root_alias_symbol = node.context[3];
array_clear(&self->stack);
array_push(&self->stack, ((TreeCursorEntry) {
.subtree = (const Subtree *)node.id,
.position = {
ts_node_start_byte(node),
ts_node_start_point(node)
},
.child_index = 0,
.structural_child_index = 0,
.descendant_index = 0,
}));
}
void ts_tree_cursor_delete(TSTreeCursor *_self) {
TreeCursor *self = (TreeCursor *)_self;
array_delete(&self->stack);
}
// TSTreeCursor - walking the tree
TreeCursorStep ts_tree_cursor_goto_first_child_internal(TSTreeCursor *_self) {
TreeCursor *self = (TreeCursor *)_self;
bool visible;
TreeCursorEntry entry;
CursorChildIterator iterator = ts_tree_cursor_iterate_children(self);
while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) {
if (visible) {
array_push(&self->stack, entry);
return TreeCursorStepVisible;
}
if (ts_subtree_visible_child_count(*entry.subtree) > 0) {
array_push(&self->stack, entry);
return TreeCursorStepHidden;
}
}
return TreeCursorStepNone;
}
bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) {
for (;;) {
switch (ts_tree_cursor_goto_first_child_internal(self)) {
case TreeCursorStepHidden:
continue;
case TreeCursorStepVisible:
return true;
default:
return false;
}
}
return false;
}
TreeCursorStep ts_tree_cursor_goto_last_child_internal(TSTreeCursor *_self) {
TreeCursor *self = (TreeCursor *)_self;
bool visible;
TreeCursorEntry entry;
CursorChildIterator iterator = ts_tree_cursor_iterate_children(self);
if (!iterator.parent.ptr || iterator.parent.ptr->child_count == 0) return TreeCursorStepNone;
TreeCursorEntry last_entry = {0};
TreeCursorStep last_step = TreeCursorStepNone;
while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) {
if (visible) {
last_entry = entry;
last_step = TreeCursorStepVisible;
}
else if (ts_subtree_visible_child_count(*entry.subtree) > 0) {
last_entry = entry;
last_step = TreeCursorStepHidden;
}
}
if (last_entry.subtree) {
array_push(&self->stack, last_entry);
return last_step;
}
return TreeCursorStepNone;
}
bool ts_tree_cursor_goto_last_child(TSTreeCursor *self) {
for (;;) {
switch (ts_tree_cursor_goto_last_child_internal(self)) {
case TreeCursorStepHidden:
continue;
case TreeCursorStepVisible:
return true;
default:
return false;
}
}
return false;
}
static inline int64_t ts_tree_cursor_goto_first_child_for_byte_and_point(
TSTreeCursor *_self,
uint32_t goal_byte,
TSPoint goal_point
) {
TreeCursor *self = (TreeCursor *)_self;
uint32_t initial_size = self->stack.size;
uint32_t visible_child_index = 0;
bool did_descend;
do {
did_descend = false;
bool visible;
TreeCursorEntry entry;
CursorChildIterator iterator = ts_tree_cursor_iterate_children(self);
while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) {
Length entry_end = length_add(entry.position, ts_subtree_size(*entry.subtree));
bool at_goal = entry_end.bytes >= goal_byte && point_gte(entry_end.extent, goal_point);
uint32_t visible_child_count = ts_subtree_visible_child_count(*entry.subtree);
if (at_goal) {
if (visible) {
array_push(&self->stack, entry);
return visible_child_index;
}
if (visible_child_count > 0) {
array_push(&self->stack, entry);
did_descend = true;
break;
}
} else if (visible) {
visible_child_index++;
} else {
visible_child_index += visible_child_count;
}
}
} while (did_descend);
self->stack.size = initial_size;
return -1;
}
int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *self, uint32_t goal_byte) {
return ts_tree_cursor_goto_first_child_for_byte_and_point(self, goal_byte, POINT_ZERO);
}
int64_t ts_tree_cursor_goto_first_child_for_point(TSTreeCursor *self, TSPoint goal_point) {
return ts_tree_cursor_goto_first_child_for_byte_and_point(self, 0, goal_point);
}
TreeCursorStep ts_tree_cursor_goto_sibling_internal(
TSTreeCursor *_self,
bool (*advance)(CursorChildIterator *, TreeCursorEntry *, bool *)) {
TreeCursor *self = (TreeCursor *)_self;
uint32_t initial_size = self->stack.size;
while (self->stack.size > 1) {
TreeCursorEntry entry = array_pop(&self->stack);
CursorChildIterator iterator = ts_tree_cursor_iterate_children(self);
iterator.child_index = entry.child_index;
iterator.structural_child_index = entry.structural_child_index;
iterator.position = entry.position;
iterator.descendant_index = entry.descendant_index;
bool visible = false;
advance(&iterator, &entry, &visible);
if (visible && self->stack.size + 1 < initial_size) break;
while (advance(&iterator, &entry, &visible)) {
if (visible) {
array_push(&self->stack, entry);
return TreeCursorStepVisible;
}
if (ts_subtree_visible_child_count(*entry.subtree)) {
array_push(&self->stack, entry);
return TreeCursorStepHidden;
}
}
}
self->stack.size = initial_size;
return TreeCursorStepNone;
}
TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *_self) {
return ts_tree_cursor_goto_sibling_internal(_self, ts_tree_cursor_child_iterator_next);
}
bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) {
switch (ts_tree_cursor_goto_next_sibling_internal(self)) {
case TreeCursorStepHidden:
ts_tree_cursor_goto_first_child(self);
return true;
case TreeCursorStepVisible:
return true;
default:
return false;
}
}
TreeCursorStep ts_tree_cursor_goto_previous_sibling_internal(TSTreeCursor *_self) {
// since subtracting across row loses column information, we may have to
// restore it
TreeCursor *self = (TreeCursor *)_self;
// for that, save current position before traversing
TreeCursorStep step = ts_tree_cursor_goto_sibling_internal(
_self, ts_tree_cursor_child_iterator_previous);
if (step == TreeCursorStepNone)
return step;
// if length is already valid, there's no need to recompute it
if (!length_is_undefined(array_back(&self->stack)->position))
return step;
// restore position from the parent node
const TreeCursorEntry *parent = &self->stack.contents[self->stack.size - 2];
Length position = parent->position;
uint32_t child_index = array_back(&self->stack)->child_index;
const Subtree *children = ts_subtree_children((*(parent->subtree)));
if (child_index > 0) {
// skip first child padding since its position should match the position of the parent
position = length_add(position, ts_subtree_size(children[0]));
for (uint32_t i = 1; i < child_index; ++i) {
position = length_add(position, ts_subtree_total_size(children[i]));
}
position = length_add(position, ts_subtree_padding(children[child_index]));
}
array_back(&self->stack)->position = position;
return step;
}
bool ts_tree_cursor_goto_previous_sibling(TSTreeCursor *self) {
switch (ts_tree_cursor_goto_previous_sibling_internal(self)) {
case TreeCursorStepHidden:
ts_tree_cursor_goto_last_child(self);
return true;
case TreeCursorStepVisible:
return true;
default:
return false;
}
}
bool ts_tree_cursor_goto_parent(TSTreeCursor *_self) {
TreeCursor *self = (TreeCursor *)_self;
for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) {
if (ts_tree_cursor_is_entry_visible(self, i)) {
self->stack.size = i + 1;
return true;
}
}
return false;
}
void ts_tree_cursor_goto_descendant(
TSTreeCursor *_self,
uint32_t goal_descendant_index
) {
TreeCursor *self = (TreeCursor *)_self;
// Ascend to the lowest ancestor that contains the goal node.
for (;;) {
uint32_t i = self->stack.size - 1;
TreeCursorEntry *entry = &self->stack.contents[i];
uint32_t next_descendant_index =
entry->descendant_index +
(ts_tree_cursor_is_entry_visible(self, i) ? 1 : 0) +
ts_subtree_visible_descendant_count(*entry->subtree);
if (
(entry->descendant_index <= goal_descendant_index) &&
(next_descendant_index > goal_descendant_index)
) {
break;
} else if (self->stack.size <= 1) {
return;
} else {
self->stack.size--;
}
}
// Descend to the goal node.
bool did_descend = true;
do {
did_descend = false;
bool visible;
TreeCursorEntry entry;
CursorChildIterator iterator = ts_tree_cursor_iterate_children(self);
if (iterator.descendant_index > goal_descendant_index) {
return;
}
while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) {
if (iterator.descendant_index > goal_descendant_index) {
array_push(&self->stack, entry);
if (visible && entry.descendant_index == goal_descendant_index) {
return;
} else {
did_descend = true;
break;
}
}
}
} while (did_descend);
}
uint32_t ts_tree_cursor_current_descendant_index(const TSTreeCursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self;
TreeCursorEntry *last_entry = array_back(&self->stack);
return last_entry->descendant_index;
}
TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self;
TreeCursorEntry *last_entry = array_back(&self->stack);
TSSymbol alias_symbol = self->root_alias_symbol;
if (self->stack.size > 1 && !ts_subtree_extra(*last_entry->subtree)) {
TreeCursorEntry *parent_entry = &self->stack.contents[self->stack.size - 2];
alias_symbol = ts_language_alias_at(
self->tree->language,
parent_entry->subtree->ptr->inner.non_terminal.production_id,
last_entry->structural_child_index
);
}
return ts_node_new(
self->tree,
last_entry->subtree,
last_entry->position,
alias_symbol
);
}
// Private - Get various facts about the current node that are needed
// when executing tree queries.
void ts_tree_cursor_current_status(
const TSTreeCursor *_self,
TSFieldId *field_id,
bool *has_later_siblings,
bool *has_later_named_siblings,
bool *can_have_later_siblings_with_this_field,
TSSymbol *supertypes,
unsigned *supertype_count
) {
const TreeCursor *self = (const TreeCursor *)_self;
unsigned max_supertypes = *supertype_count;
*field_id = 0;
*supertype_count = 0;
*has_later_siblings = false;
*has_later_named_siblings = false;
*can_have_later_siblings_with_this_field = false;
// Walk up the tree, visiting the current node and its invisible ancestors,
// because fields can refer to nodes through invisible *wrapper* nodes,
for (unsigned i = self->stack.size - 1; i > 0; i--) {
TreeCursorEntry *entry = &self->stack.contents[i];
TreeCursorEntry *parent_entry = &self->stack.contents[i - 1];
const TSSymbol *alias_sequence = ts_language_alias_sequence(
self->tree->language,
parent_entry->subtree->ptr->inner.non_terminal.production_id
);
#define subtree_symbol(subtree, structural_child_index) \
(( \
!ts_subtree_extra(subtree) && \
alias_sequence && \
alias_sequence[structural_child_index] \
) ? \
alias_sequence[structural_child_index] : \
ts_subtree_symbol(subtree))
// Stop walking up when a visible ancestor is found.
TSSymbol entry_symbol = subtree_symbol(
*entry->subtree,
entry->structural_child_index
);
TSSymbolMetadata entry_metadata = ts_language_symbol_metadata(
self->tree->language,
entry_symbol
);
if (i != self->stack.size - 1 && entry_metadata.visible) break;
// Record any supertypes
if (entry_metadata.supertype && *supertype_count < max_supertypes) {
supertypes[*supertype_count] = entry_symbol;
(*supertype_count)++;
}
// Determine if the current node has later siblings.
if (!*has_later_siblings) {
unsigned sibling_count = parent_entry->subtree->ptr->child_count;
unsigned structural_child_index = entry->structural_child_index;
if (!ts_subtree_extra(*entry->subtree)) structural_child_index++;
for (unsigned j = entry->child_index + 1; j < sibling_count; j++) {
Subtree sibling = ts_subtree_children(*parent_entry->subtree)[j];
TSSymbolMetadata sibling_metadata = ts_language_symbol_metadata(
self->tree->language,
subtree_symbol(sibling, structural_child_index)
);
if (sibling_metadata.visible) {
*has_later_siblings = true;
if (*has_later_named_siblings) break;
if (sibling_metadata.named) {
*has_later_named_siblings = true;
break;
}
} else if (ts_subtree_visible_child_count(sibling) > 0) {
*has_later_siblings = true;
if (*has_later_named_siblings) break;
if (sibling.ptr->inner.non_terminal.named_child_count > 0) {
*has_later_named_siblings = true;
break;
}
}
if (!ts_subtree_extra(sibling)) structural_child_index++;
}
}
#undef subtree_symbol
if (!ts_subtree_extra(*entry->subtree)) {
const TSFieldMapEntry *field_map, *field_map_end;
ts_language_field_map(
self->tree->language,
parent_entry->subtree->ptr->inner.non_terminal.production_id,
&field_map, &field_map_end
);
// Look for a field name associated with the current node.
if (!*field_id) {
for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) {
if (!map->inherited && map->child_index == entry->structural_child_index) {
*field_id = map->field_id;
break;
}
}
}
// Determine if the current node can have later siblings with the same field name.
if (*field_id) {
for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) {
if (
map->field_id == *field_id &&
map->child_index > entry->structural_child_index
) {
*can_have_later_siblings_with_this_field = true;
break;
}
}
}
}
}
}
uint32_t ts_tree_cursor_current_depth(const TSTreeCursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self;
uint32_t depth = 0;
for (unsigned i = 1; i < self->stack.size; i++) {
if (ts_tree_cursor_is_entry_visible(self, i)) {
depth++;
}
}
return depth;
}
TSNode ts_tree_cursor_parent_node(const TSTreeCursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self;
for (int i = (int)self->stack.size - 2; i >= 0; i--) {
TreeCursorEntry *entry = &self->stack.contents[i];
bool is_visible = true;
TSSymbol alias_symbol = 0;
if (i > 0) {
TreeCursorEntry *parent_entry = &self->stack.contents[i - 1];
alias_symbol = ts_language_alias_at(
self->tree->language,
parent_entry->subtree->ptr->inner.non_terminal.production_id,
entry->structural_child_index
);
is_visible = (alias_symbol != 0) || ts_subtree_visible(*entry->subtree);
}
if (is_visible) {
return ts_node_new(
self->tree,
entry->subtree,
entry->position,
alias_symbol
);
}
}
return ts_node_new(NULL, NULL, length_zero(), 0);
}
TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self;
// Walk up the tree, visiting the current node and its invisible ancestors.
for (unsigned i = self->stack.size - 1; i > 0; i--) {
TreeCursorEntry *entry = &self->stack.contents[i];
TreeCursorEntry *parent_entry = &self->stack.contents[i - 1];
// Stop walking up when another visible node is found.
if (
i != self->stack.size - 1 &&
ts_tree_cursor_is_entry_visible(self, i)
) break;
if (ts_subtree_extra(*entry->subtree)) break;
const TSFieldMapEntry *field_map, *field_map_end;
ts_language_field_map(
self->tree->language,
parent_entry->subtree->ptr->inner.non_terminal.production_id,
&field_map, &field_map_end
);
for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) {
if (!map->inherited && map->child_index == entry->structural_child_index) {
return map->field_id;
}
}
}
return 0;
}
const char *ts_tree_cursor_current_field_name(const TSTreeCursor *_self) {
TSFieldId id = ts_tree_cursor_current_field_id(_self);
if (id) {
const TreeCursor *self = (const TreeCursor *)_self;
return self->tree->language->field_names[id];
} else {
return NULL;
}
}
TSTreeCursor ts_tree_cursor_copy(const TSTreeCursor *_cursor) {
const TreeCursor *cursor = (const TreeCursor *)_cursor;
TSTreeCursor res = {NULL, NULL, {0, 0}};
TreeCursor *copy = (TreeCursor *)&res;
copy->tree = cursor->tree;
copy->root_alias_symbol = cursor->root_alias_symbol;
array_init(&copy->stack);
array_push_all(&copy->stack, &cursor->stack);
return res;
}
void ts_tree_cursor_reset_to(TSTreeCursor *_dst, const TSTreeCursor *_src) {
const TreeCursor *cursor = (const TreeCursor *)_src;
TreeCursor *copy = (TreeCursor *)_dst;
copy->tree = cursor->tree;
copy->root_alias_symbol = cursor->root_alias_symbol;
array_clear(&copy->stack);
array_push_all(&copy->stack, &cursor->stack);
}

44
parser/nsrc/tree_cursor.h
View file

@ -1,44 +0,0 @@
#ifndef TREE_SITTER_TREE_CURSOR_H_
#define TREE_SITTER_TREE_CURSOR_H_
#include "./subtree.h"
typedef struct TreeCursorEntry
{
const Subtree *subtree;
Length position;
uint32_t child_index;
uint32_t structural_child_index;
uint32_t descendant_index;
} TreeCursorEntry;
typedef struct TreeCursor
{
const TSTree *tree;
Array(TreeCursorEntry) stack;
TSSymbol root_alias_symbol;
} TreeCursor;
typedef enum TreeCursorStep
{
TreeCursorStepNone,
TreeCursorStepHidden,
TreeCursorStepVisible,
} TreeCursorStep;
void ts_tree_cursor_init(TreeCursor *, TSNode);
void ts_tree_cursor_current_status(const TSTreeCursor *, TSFieldId *, bool *, bool *, bool *, TSSymbol *, unsigned *);
TreeCursorStep ts_tree_cursor_goto_first_child_internal(TSTreeCursor *);
TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *);
static inline Subtree ts_tree_cursor_current_subtree(const TSTreeCursor *_self)
{
const TreeCursor *self = (const TreeCursor *)_self;
TreeCursorEntry *last_entry = array_back(&self->stack);
return *last_entry->subtree;
}
TSNode ts_tree_cursor_parent_node(const TSTreeCursor *);
#endif // TREE_SITTER_TREE_CURSOR_H_

32
parser/nsrc/unicode.h
View file

@ -1,32 +0,0 @@
#ifndef TREE_SITTER_UNICODE_H_
#define TREE_SITTER_UNICODE_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include <limits.h>
#include <stdint.h>
#define U_EXPORT
#define U_EXPORT2
static const int32_t TS_DECODE_ERROR = -1;
// These functions read one unicode code point from the given string,
// returning the number of bytes consumed.
typedef uint32_t (*UnicodeDecodeFunction)(const uint8_t *string, uint32_t length, int32_t *code_point);
static inline uint32_t ts_decode_ascii(const uint8_t *string, uint32_t length, int32_t *code_point)
{
if (length >= 1 && string[0] <= 127)
return (*code_point = string[0], 1);
return (0);
}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_UNICODE_H_

8
parser/parse_types.h
View file

@ -6,7 +6,7 @@
/* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */ /* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */
/* +#+#+#+#+#+ +#+ */ /* +#+#+#+#+#+ +#+ */
/* Created: 2024/04/24 23:01:45 by maiboyer #+# #+# */ /* Created: 2024/04/24 23:01:45 by maiboyer #+# #+# */
/* Updated: 2024/06/29 21:11:19 by maiboyer ### ########.fr */ /* Updated: 2024/06/30 18:03:54 by maiboyer ### ########.fr */
/* */ /* */
/* ************************************************************************** */ /* ************************************************************************** */
@ -16,9 +16,9 @@
#include <stdbool.h> #include <stdbool.h>
#include <stdint.h> #include <stdint.h>
#include "./nsrc/api.h" #include "./nnsrc/api.h"
#include "./nsrc/lexer.h" #include "./nnsrc/lexer.h"
#include "./nsrc/parser.h" #include "./nnsrc/parser.h"
#include "me/types.h" #include "me/types.h"
#include "parser/types/types_lexer_state.h" #include "parser/types/types_lexer_state.h"

65
parser/src/api.h
View file

@ -1,65 +0,0 @@
#ifndef TREE_SITTER_ARRAY_H_
#define TREE_SITTER_ARRAY_H_
#include "me/char/char.h"
#include "me/mem/mem.h"
#include <assert.h>
#include <limits.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "./api_structs.h"
#include "./array.h"
#include "./funcs.h"
#define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1)
#define LANGUAGE_VERSION_WITH_PRIMARY_STATES 14
#define LANGUAGE_VERSION_USABLE_VIA_WASM 13
#define ERROR_STATE 0
#define ERROR_COST_PER_RECOVERY 500
#define ERROR_COST_PER_MISSING_TREE 110
#define ERROR_COST_PER_SKIPPED_TREE 100
#define ERROR_COST_PER_SKIPPED_LINE 30
#define ERROR_COST_PER_SKIPPED_CHAR 1
#define MAX_STEP_CAPTURE_COUNT 3
#define MAX_NEGATED_FIELD_COUNT 8
#define MAX_STATE_PREDECESSOR_COUNT 256
#define MAX_ANALYSIS_STATE_DEPTH 8
#define MAX_ANALYSIS_ITERATION_COUNT 256
#define MAX_LINK_COUNT 8
#define MAX_NODE_POOL_SIZE 50
#define MAX_ITERATOR_COUNT 64
#define TS_MAX_INLINE_TREE_LENGTH UINT8_MAX
#define TS_MAX_TREE_POOL_SIZE 32
#define ts_builtin_sym_error ((t_symbol)-1)
#define ts_builtin_sym_end 0
#define POINT_ZERO ((t_point){0, 0})
#define POINT_MAX ((t_point){UINT32_MAX, UINT32_MAX})
#define TS_TREE_STATE_NONE USHRT_MAX
#define NULL_SUBTREE ((t_subtree){.ptr = NULL})
#define STACK_VERSION_NONE ((t_stack_version)-1)
#define TS_DECODE_ERROR (-1)
#if true
# undef malloc
# undef calloc
# undef realloc
# undef free
# define malloc(s) mem_alloc((s))
# define calloc(s, l) mem_alloc_array((s), (l))
# define realloc(p, t) mem_realloc((p), (t))
# define free(p) mem_free((p))
#endif
// Get a subtree's children, which are allocated immediately before the
// tree's own heap data.
#define ts_subtree_children(self) ((self).data.is_inline ? NULL : (t_subtree *)((self).ptr) - (self).ptr->child_count)
static const t_length LENGTH_UNDEFINED = {0, {0, 1}};
static const t_length LENGTH_MAX = {UINT32_MAX, {UINT32_MAX, UINT32_MAX}};
#endif // TREE_SITTER_TREE_H_

546
parser/src/api_structs.h
View file

@ -1,546 +0,0 @@
#ifndef API_STRUCTS_H
#define API_STRUCTS_H
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
#include "./array.h"
#include "me/types.h"
#include <stdint.h>
typedef uint16_t t_field_id;
typedef uint16_t t_state_id;
typedef uint16_t t_symbol;
typedef uint64_t t_parser_clock;
typedef uint64_t t_parser_duration;
typedef uint32_t t_stack_version;
typedef union u_parse_action_entry t_parse_action_entry;
typedef union u_subtree t_subtree;
typedef union u_mutable_subtree t_mutable_subtree;
typedef union u_parse_action t_parse_action;
typedef struct s_char_range t_char_range;
typedef struct s_external_scanner_state t_external_scanner_state;
typedef struct s_field_map_entry t_field_map_entry;
typedef struct s_field_map_slice t_field_map_slice;
typedef struct s_first_parser t_first_parser;
typedef struct s_first_tree t_first_tree;
typedef struct s_input_edit t_input_edit;
typedef struct s_language t_language;
typedef struct s_length t_length;
typedef struct s_lex_mode t_lex_mode;
typedef struct s_lexer t_lexer;
typedef struct s_lexer_data t_lexer_data;
typedef struct s_lookahead_iterator t_lookahead_iterator;
typedef struct s_parse_input t_parse_input;
typedef struct s_parse_logger t_parse_logger;
typedef struct s_parse_node t_parse_node;
typedef struct s_parse_query t_parse_query;
typedef struct s_parse_query_cursor t_parse_query_cursor;
typedef struct s_parse_query_error t_parse_query_error;
typedef struct s_parse_query_error_cost t_parse_query_error_cost;
typedef struct s_parse_range t_parse_range;
typedef struct s_parse_state t_parse_state;
typedef struct s_point t_point;
typedef struct s_query_capture t_query_capture;
typedef struct s_query_cursor t_query_cursor;
typedef struct s_query_match t_query_match;
typedef struct s_query_predicate_step t_query_predicate_step;
typedef struct s_reduce_action t_reduce_action;
typedef struct s_reusable_node t_reusable_node;
typedef struct s_stack t_stack;
typedef struct s_stack_entry t_stack_entry;
typedef struct s_stack_slice t_stack_slice;
typedef struct s_stack_summary_entry t_stack_summary_entry;
typedef struct s_subtree_heap_data t_subtree_heap_data;
typedef struct s_subtree_inline_data t_subtree_inline_data;
typedef struct s_subtree_pool t_subtree_pool;
typedef struct s_symbol_metadata t_symbol_metadata;
typedef struct s_table_entry t_table_entry;
typedef struct s_tree_cursor t_tree_cursor;
typedef struct s_tree_cursor_entry t_tree_cursor_entry;
typedef enum e_input_encoding t_input_encoding;
typedef enum e_log_type t_log_type;
typedef enum e_parse_action_type t_parse_action_type;
typedef enum e_quantifier t_quantifier;
typedef enum e_query_error t_query_error;
typedef enum e_query_predicate_step_type t_query_predicate_step_type;
typedef enum e_symbol_type t_symbol_type;
typedef Array(t_parse_range) t_range_array;
typedef Array(t_subtree) t_subtree_array;
typedef Array(t_mutable_subtree) t_mutable_subtree_array;
typedef Array(t_reduce_action) t_reduce_action_set;
typedef Array(t_stack_slice) t_stack_slice_array;
typedef Array(t_stack_summary_entry) t_stack_summary;
typedef void (*StackIterateCallback)(void *, t_state_id, uint32_t);
struct s_point
{
uint32_t row;
uint32_t column;
};
struct s_length
{
uint32_t bytes;
t_point extent;
};
struct s_stack_slice
{
t_subtree_array subtrees;
t_stack_version version;
};
struct s_stack_summary_entry
{
t_length position;
unsigned depth;
t_state_id state;
};
enum e_input_encoding
{
TSInputEncodingUTF8,
TSInputEncodingUTF16,
};
enum e_symbol_type
{
TSSymbolTypeRegular,
TSSymbolTypeAnonymous,
TSSymbolTypeAuxiliary,
};
struct s_parse_range
{
t_point start_point;
t_point end_point;
uint32_t start_byte;
uint32_t end_byte;
};
struct s_parse_input
{
void *payload;
const char *(*read)(void *payload, uint32_t byte_index, t_point position, uint32_t *bytes_read);
t_input_encoding encoding;
};
enum e_log_type
{
TSLogTypeParse,
TSLogTypeLex,
};
struct s_parse_logger
{
void *payload;
void (*log)(void *payload, t_log_type log_type, const char *buffer);
};
struct s_input_edit
{
uint32_t start_byte;
uint32_t old_end_byte;
uint32_t new_end_byte;
t_point start_point;
t_point old_end_point;
t_point new_end_point;
};
struct s_parse_node
{
uint32_t context[4];
const void *id;
const t_first_tree *tree;
};
struct s_tree_cursor_entry
{
const t_subtree *subtree;
t_length position;
uint32_t child_index;
uint32_t structural_child_index;
uint32_t descendant_index;
};
struct s_tree_cursor
{
const t_first_tree *tree;
Array(t_tree_cursor_entry) stack;
t_symbol root_alias_symbol;
};
struct s_query_capture
{
t_parse_node node;
uint32_t index;
};
enum e_quantifier
{
TSQuantifierZero = 0, // must match the array initialization value
TSQuantifierZeroOrOne,
TSQuantifierZeroOrMore,
TSQuantifierOne,
TSQuantifierOneOrMore,
};
struct s_query_match
{
uint32_t id;
uint16_t pattern_index;
uint16_t capture_count;
const t_query_capture *captures;
};
enum e_query_predicate_step_type
{
TSQueryPredicateStepTypeDone,
TSQueryPredicateStepTypeCapture,
TSQueryPredicateStepTypeString,
};
struct s_query_predicate_step
{
t_query_predicate_step_type type;
uint32_t value_id;
};
enum e_query_error
{
TSQueryErrorNone = 0,
TSQueryErrorSyntax,
TSQueryErrorNodeType,
TSQueryErrorField,
TSQueryErrorCapture,
TSQueryErrorStructure,
TSQueryErrorLanguage,
};
struct s_parent_cache_entry
{
const t_subtree *child;
const t_subtree *parent;
t_length position;
t_symbol alias_symbol;
};
typedef enum e_tree_cursor_step
{
TreeCursorStepNone,
TreeCursorStepHidden,
TreeCursorStepVisible,
} t_tree_cursor_step;
// The serialized state of an external scanner.
//
// Every time an external token subtree is created after a call to an
// external scanner, the scanner's `serialize` function is called to
// retrieve a serialized copy of its state. The bytes are then copied
// onto the subtree itself so that the scanner's state can later be
// restored using its `deserialize` function.
//
// Small byte arrays are stored inline, and long ones are allocated
// separately on the heap.
struct s_external_scanner_state
{
// TODO: extract this anonymous struct
union {
char *long_data;
char short_data[24];
};
uint32_t length;
};
struct s_subtree_inline_data
{
bool is_inline : 1;
bool visible : 1;
bool named : 1;
bool extra : 1;
bool has_changes : 1;
bool is_missing : 1;
bool is_keyword : 1;
uint8_t symbol;
uint16_t parse_state;
uint8_t padding_columns;
uint8_t padding_rows : 4;
uint8_t lookahead_bytes : 4;
uint8_t padding_bytes;
uint8_t size_bytes;
};
struct s_subtree_heap_data
{
volatile uint32_t ref_count;
t_length padding;
t_length size;
uint32_t lookahead_bytes;
uint32_t error_cost;
uint32_t child_count;
t_symbol symbol;
t_state_id parse_state;
bool visible : 1;
bool named : 1;
bool extra : 1;
bool fragile_left : 1;
bool fragile_right : 1;
bool has_changes : 1;
bool has_external_tokens : 1;
bool has_external_scanner_state_change : 1;
bool depends_on_column : 1;
bool is_missing : 1;
bool is_keyword : 1;
// TODO: extract these anonymous struct
union {
// Non-terminal subtrees (`child_count > 0`)
struct
{
uint32_t visible_child_count;
uint32_t named_child_count;
uint32_t visible_descendant_count;
int32_t dynamic_precedence;
uint16_t repeat_depth;
uint16_t production_id;
struct
{
t_symbol symbol;
t_state_id parse_state;
} first_leaf;
};
// External terminal subtrees (`child_count == 0 &&
// has_external_tokens`)
t_external_scanner_state external_scanner_state;
// Error terminal subtrees (`child_count == 0 && symbol ==
// ts_builtin_sym_error`)
int32_t lookahead_char;
};
};
// The fundamental building block of a syntax tree.
union u_subtree {
t_subtree_inline_data data;
const t_subtree_heap_data *ptr;
};
// Like t_subtree, but mutable.
union u_mutable_subtree {
t_subtree_inline_data data;
t_subtree_heap_data *ptr;
};
struct s_subtree_pool
{
t_mutable_subtree_array free_trees;
t_mutable_subtree_array tree_stack;
};
union u_parse_action {
// TODO: extract this anonymous struct
struct
{
uint8_t type;
t_state_id state;
bool extra;
bool repetition;
} shift;
// TODO: extract this anonymous struct
struct
{
uint8_t type;
uint8_t child_count;
t_symbol symbol;
int16_t dynamic_precedence;
uint16_t production_id;
} reduce;
uint8_t type;
};
struct s_table_entry
{
const t_parse_action *actions;
uint32_t action_count;
bool is_reusable;
};
struct s_lookahead_iterator
{
const t_language *language;
const uint16_t *data;
const uint16_t *group_end;
t_state_id state;
uint16_t table_value;
uint16_t section_index;
uint16_t group_count;
bool is_small_state;
const t_parse_action *actions;
t_symbol symbol;
t_state_id next_state;
uint16_t action_count;
};
struct s_symbol_metadata
{
bool visible;
bool named;
bool supertype;
};
enum e_parse_action_type
{
TSParseActionTypeShift,
TSParseActionTypeReduce,
TSParseActionTypeAccept,
TSParseActionTypeRecover,
};
union u_parse_action_entry {
t_parse_action action;
// TODO: extract this anonymous struct
struct
{
uint8_t count;
bool reusable;
} entry;
};
struct s_field_map_entry
{
t_field_id field_id;
uint8_t child_index;
bool inherited;
};
struct s_field_map_slice
{
uint16_t index;
uint16_t length;
};
struct s_lexer_data
{
int32_t lookahead;
t_symbol result_symbol;
void (*advance)(t_lexer_data *, bool);
void (*mark_end)(t_lexer_data *);
uint32_t (*get_column)(t_lexer_data *);
bool (*is_at_included_range_start)(const t_lexer_data *);
bool (*eof)(const t_lexer_data *);
};
struct s_lex_mode
{
uint16_t lex_state;
uint16_t external_lex_state;
};
struct s_char_range
{
int32_t start;
int32_t end;
};
struct s_language
{
uint32_t version;
uint32_t symbol_count;
uint32_t alias_count;
uint32_t token_count;
uint32_t external_token_count;
uint32_t state_count;
uint32_t large_state_count;
uint32_t production_id_count;
uint32_t field_count;
uint16_t max_alias_sequence_length;
const uint16_t *parse_table;
const uint16_t *small_parse_table;
const uint32_t *small_parse_table_map;
const t_parse_action_entry *parse_actions;
const char *const *symbol_names;
const char *const *field_names;
const t_field_map_slice *field_map_slices;
const t_field_map_entry *field_map_entries;
const t_symbol_metadata *symbol_metadata;
const t_symbol *public_symbol_map;
const uint16_t *alias_map;
const t_symbol *alias_sequences;
const t_lex_mode *lex_modes;
bool (*lex_fn)(t_lexer_data *, t_state_id);
bool (*keyword_lex_fn)(t_lexer_data *, t_state_id);
t_symbol keyword_capture_token;
// TODO: extract this anonymous struct
struct
{
const bool *states;
const t_symbol *symbol_map;
void *(*create)(void);
void (*destroy)(void *);
bool (*scan)(void *, t_lexer_data *, const bool *symbol_whitelist);
uint32_t (*serialize)(void *, char *);
void (*deserialize)(void *, const char *, uint32_t);
} external_scanner;
const t_state_id *primary_state_ids;
};
struct s_lexer
{
t_lexer_data data;
t_length current_position;
t_length token_start_position;
t_length token_end_position;
t_parse_range *included_ranges;
const char *chunk;
t_parse_input input;
t_parse_logger logger;
uint32_t included_range_count;
uint32_t current_included_range_index;
uint32_t chunk_start;
uint32_t chunk_size;
uint32_t lookahead_size;
bool did_get_column;
char debug_buffer[TREE_SITTER_SERIALIZATION_BUFFER_SIZE];
};
struct s_reduce_action
{
uint32_t count;
t_symbol symbol;
int32_t dynamic_precedence;
uint16_t production_id;
};
struct s_stack_entry
{
t_subtree tree;
uint32_t child_index;
uint32_t byte_offset;
};
struct s_reusable_node
{
Array(t_stack_entry) stack;
t_subtree last_external_token;
};
struct s_first_tree
{
t_subtree root;
const t_language *language;
t_parse_range *included_ranges;
uint32_t included_range_count;
};
#endif // API_STRUCTS_H

283
parser/src/array.h
View file

@ -1,283 +0,0 @@
#ifndef ARRAY_H
#define ARRAY_H
#include <assert.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#if true
# undef malloc
# undef calloc
# undef realloc
# undef free
# define malloc(s) mem_alloc((s))
# define calloc(s, l) mem_alloc_array((s), (l))
# define realloc(p, t) mem_realloc((p), (t))
# define free(p) mem_free((p))
#endif
#define Array(T) \
struct \
{ \
T *contents; \
uint32_t size; \
uint32_t capacity; \
}
#ifndef inline
# define inline __inline__
#endif
/// Initialize an array.
#define array_init(self) ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
/// Create an empty array.
#define array_new() \
{ \
NULL, 0, 0 \
}
/// Get a pointer to the element at a given `index` in the array.
#define array_get(self, _index) (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
/// Get a pointer to the first element in the array.
#define array_front(self) array_get(self, 0)
/// Get a pointer to the last element in the array.
#define array_back(self) array_get(self, (self)->size - 1)
/// Clear the array, setting its size to zero. Note that this does not free any
/// memory allocated for the array's contents.
#define array_clear(self) ((self)->size = 0)
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
/// less than the array's current capacity, this function has no effect.
#define array_reserve(self, new_capacity) _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
/// Free any memory allocated for this array. Note that this does not free any
/// memory allocated for the array's contents.
#define array_delete(self) _array__delete((Array *)(self))
/// Push a new `element` onto the end of the array.
#define array_push(self, element) (_array__grow((Array *)(self), 1, array_elem_size(self)), (self)->contents[(self)->size++] = (element))
/// Increase the array's size by `count` elements.
/// New elements are zero-initialized.
#define array_grow_by(self, count) \
do \
{ \
if ((count) == 0) \
break; \
_array__grow((Array *)(self), count, array_elem_size(self)); \
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
(self)->size += (count); \
} while (0)
/// Append all elements from one array to the end of another.
#define array_push_all(self, other) array_extend((self), (other)->size, (other)->contents)
/// Append `count` elements to the end of the array, reading their values from
/// the `contents` pointer.
#define array_extend(self, count, contents) _array__splice((Array *)(self), array_elem_size(self), (self)->size, 0, count, contents)
/// Remove `old_count` elements from the array starting at the given `index`. At
/// the same index, insert `new_count` new elements, reading their values from
/// the `new_contents` pointer.
#define array_splice(self, _index, old_count, new_count, new_contents) \
_array__splice((Array *)(self), array_elem_size(self), _index, old_count, new_count, new_contents)
/// Insert one `element` into the array at the given `index`.
#define array_insert(self, _index, element) _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
/// Remove one element from the array at the given `index`.
#define array_erase(self, _index) _array__erase((Array *)(self), array_elem_size(self), _index)
/// Pop the last element off the array, returning the element by value.
#define array_pop(self) ((self)->contents[--(self)->size])
/// Assign the contents of one array to another, reallocating if necessary.
#define array_assign(self, other) _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
/// Swap one array with another
#define array_swap(self, other) _array__swap((Array *)(self), (Array *)(other))
/// Get the size of the array contents
#define array_elem_size(self) (sizeof *(self)->contents)
/// Search a sorted array for a given `needle` value, using the given `compare`
/// callback to determine the order.
///
/// If an existing element is found to be equal to `needle`, then the `index`
/// out-parameter is set to the existing value's index, and the `exists`
/// out-parameter is set to true. Otherwise, `index` is set to an index where
/// `needle` should be inserted in order to preserve the sorting, and `exists`
/// is set to false.
#define array_search_sorted_with(self, compare, needle, _index, _exists) _array__search_sorted(self, 0, compare, , needle, _index, _exists)
/// Helper macro for the `_sorted_by` routines below. This takes the left
/// (existing) parameter by reference in order to work with the generic sorting
/// function above.
#define _compare_int(a, b) ((int)*(a) - (int)(b))
/// Search a sorted array for a given `needle` value, using integer comparisons
/// of a given struct field (specified with a leading dot) to determine the
/// order.
///
/// See also `array_search_sorted_with`.
#define array_search_sorted_by(self, field, needle, _index, _exists) _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
/// Insert a given `value` into a sorted array, using the given `compare`
/// callback to determine the order.
#define array_insert_sorted_with(self, compare, value) \
do \
{ \
unsigned _index, _exists; \
array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
if (!_exists) \
array_insert(self, _index, value); \
} while (0)
/// Insert a given `value` into a sorted array, using integer comparisons of
/// a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_by`.
#define array_insert_sorted_by(self, field, value) \
do \
{ \
unsigned _index, _exists; \
array_search_sorted_by(self, field, (value)field, &_index, &_exists); \
if (!_exists) \
array_insert(self, _index, value); \
} while (0)
typedef Array(void) Array;
/// This is not what you're looking for, see `array_delete`.
static inline void _array__delete(Array *self)
{
if (self->contents)
{
free(self->contents);
self->contents = NULL;
self->size = 0;
self->capacity = 0;
}
}
/// This is not what you're looking for, see `array_erase`.
static inline void _array__erase(Array *self, size_t element_size, uint32_t index)
{
assert(index < self->size);
char *contents = (char *)self->contents;
memmove(contents + index * element_size, contents + (index + 1) * element_size, (self->size - index - 1) * element_size);
self->size--;
}
/// This is not what you're looking for, see `array_reserve`.
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity)
{
if (new_capacity > self->capacity)
{
if (self->contents)
{
self->contents = realloc(self->contents, new_capacity * element_size);
}
else
{
self->contents = malloc(new_capacity * element_size);
}
self->capacity = new_capacity;
}
}
/// This is not what you're looking for, see `array_assign`.
static inline void _array__assign(Array *self, const Array *other, size_t element_size)
{
_array__reserve(self, element_size, other->size);
self->size = other->size;
memcpy(self->contents, other->contents, self->size * element_size);
}
/// This is not what you're looking for, see `array_swap`.
static inline void _array__swap(Array *self, Array *other)
{
Array swap = *other;
*other = *self;
*self = swap;
}
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
static inline void _array__grow(Array *self, uint32_t count, size_t element_size)
{
uint32_t new_size = self->size + count;
if (new_size > self->capacity)
{
uint32_t new_capacity = self->capacity * 2;
if (new_capacity < 8)
new_capacity = 8;
if (new_capacity < new_size)
new_capacity = new_size;
_array__reserve(self, element_size, new_capacity);
}
}
/// This is not what you're looking for, see `array_splice`.
static inline void _array__splice(Array *self, size_t element_size, uint32_t index, uint32_t old_count, uint32_t new_count, const void *elements)
{
uint32_t new_size = self->size + new_count - old_count;
uint32_t old_end = index + old_count;
uint32_t new_end = index + new_count;
assert(old_end <= self->size);
_array__reserve(self, element_size, new_size);
char *contents = (char *)self->contents;
if (self->size > old_end)
{
memmove(contents + new_end * element_size, contents + old_end * element_size, (self->size - old_end) * element_size);
}
if (new_count > 0)
{
if (elements)
{
memcpy((contents + index * element_size), elements, new_count * element_size);
}
else
{
memset((contents + index * element_size), 0, new_count * element_size);
}
}
self->size += new_count - old_count;
}
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
/// This is not what you're looking for, see `array_search_sorted_with` or
/// `array_search_sorted_by`.
#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
do \
{ \
*(_index) = start; \
*(_exists) = false; \
uint32_t size = (self)->size - *(_index); \
if (size == 0) \
break; \
int comparison; \
while (size > 1) \
{ \
uint32_t half_size = size / 2; \
uint32_t mid_index = *(_index) + half_size; \
comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
if (comparison <= 0) \
*(_index) = mid_index; \
size -= half_size; \
} \
comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
if (comparison == 0) \
*(_exists) = true; \
else if (comparison < 0) \
*(_index) += 1; \
} while (0)
#endif // ARRAY_H

10456
parser/src/combined.c
View file

File diff suppressed because it is too large Load diff

272
parser/src/combined.h
View file

@ -1,272 +0,0 @@
/* ************************************************************************** */
/* */
/* ::: :::::::: */
/* combined.h :+: :+: :+: */
/* +:+ +:+ +:+ */
/* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */
/* +#+#+#+#+#+ +#+ */
/* Created: 2024/06/18 17:01:23 by maiboyer #+# #+# */
/* Updated: 2024/06/18 17:12:45 by maiboyer ### ########.fr */
/* */
/* ************************************************************************** */
#ifndef COMBINED_H
#define COMBINED_H
#include "./api.h"
#include "./structs.h"
t_u32 ascii_decode(const t_u8 *chunk, t_u32 size, t_i32 *codepoint);
int _ts_dup(int file_descriptor);
t_external_scanner_state ts_external_scanner_state_copy(const t_external_scanner_state *self);
const char *ts_external_scanner_state_data(const t_external_scanner_state *self);
void ts_external_scanner_state_delete(t_external_scanner_state *self);
bool ts_external_scanner_state_eq(const t_external_scanner_state *self, const char *buffer, unsigned length);
void ts_external_scanner_state_init(t_external_scanner_state *self, const char *data, unsigned length);
const t_language *ts_language_copy(const t_language *self);
void ts_language_delete(const t_language *self);
t_u32 ts_language_field_count(const t_language *self);
t_field_id ts_language_field_id_for_name(const t_language *self, const char *name, t_u32 name_length);
const char *ts_language_field_name_for_id(const t_language *self, t_field_id id);
t_state_id ts_language_next_state(const t_language *self, t_state_id state, t_symbol symbol);
t_symbol ts_language_public_symbol(const t_language *self, t_symbol symbol);
t_u32 ts_language_state_count(const t_language *self);
t_u32 ts_language_symbol_count(const t_language *self);
t_symbol ts_language_symbol_for_name(const t_language *self, const char *string, t_u32 length, bool is_named);
t_symbol_metadata ts_language_symbol_metadata(const t_language *self, t_symbol symbol);
const char *ts_language_symbol_name(const t_language *self, t_symbol symbol);
t_symbol_type ts_language_symbol_type(const t_language *self, t_symbol symbol);
void ts_language_table_entry(const t_language *self, t_state_id state, t_symbol symbol, t_table_entry *result);
t_u32 ts_language_version(const t_language *self);
void ts_lexer_advance_to_end(t_lexer *self);
void ts_lexer_delete(t_lexer *self);
void ts_lexer_finish(t_lexer *self, t_u32 *lookahead_end_byte);
t_parse_range *ts_lexer_included_ranges(const t_lexer *self, t_u32 *count);
void ts_lexer_init(t_lexer *self);
void ts_lexer_mark_end(t_lexer *self);
void ts_lexer_reset(t_lexer *self, t_length position);
bool ts_lexer_set_included_ranges(t_lexer *self, const t_parse_range *ranges, t_u32 count);
void ts_lexer_set_input(t_lexer *self, t_parse_input input);
void ts_lexer_start(t_lexer *self);
// START PROBABLY DELETE WORTHY
t_symbol ts_lookahead_iterator_current_symbol(const t_lookahead_iterator *self);
const char *ts_lookahead_iterator_current_symbol_name(const t_lookahead_iterator *self);
void ts_lookahead_iterator_delete(t_lookahead_iterator *self);
const t_language *ts_lookahead_iterator_language(const t_lookahead_iterator *self);
t_lookahead_iterator *ts_lookahead_iterator_new(const t_language *self, t_state_id state);
bool ts_lookahead_iterator_next(t_lookahead_iterator *self);
bool ts_lookahead_iterator_reset(t_lookahead_iterator *self, const t_language *language, t_state_id state);
bool ts_lookahead_iterator_reset_state(t_lookahead_iterator *self, t_state_id state);
// END PROBABLY DELETE WORTHY
t_parse_node ts_node_child(t_parse_node self, t_u32 child_index);
t_parse_node ts_node_child_by_field_id(t_parse_node self, t_field_id field_id);
t_parse_node ts_node_child_by_field_name(t_parse_node self, const char *name, t_u32 name_length);
t_parse_node ts_node_child_containing_descendant(t_parse_node self, t_parse_node subnode);
t_u32 ts_node_child_count(t_parse_node self);
t_u32 ts_node_descendant_count(t_parse_node self);
t_parse_node ts_node_descendant_for_byte_range(t_parse_node self, t_u32 start, t_u32 end);
t_parse_node ts_node_descendant_for_point_range(t_parse_node self, t_point start, t_point end);
void ts_node_edit(t_parse_node *self, const t_input_edit *edit);
t_u32 ts_node_end_byte(t_parse_node self);
t_point ts_node_end_point(t_parse_node self);
bool ts_node_eq(t_parse_node self, t_parse_node other);
t_field_id ts_node_field_id_for_child(t_parse_node self, t_u32 child_index);
const char *ts_node_field_name_for_child(t_parse_node self, t_u32 child_index);
t_parse_node ts_node_first_child_for_byte(t_parse_node self, t_u32 byte);
t_parse_node ts_node_first_named_child_for_byte(t_parse_node self, t_u32 byte);
t_symbol ts_node_grammar_symbol(t_parse_node self);
const char *ts_node_grammar_type(t_parse_node self);
bool ts_node_has_changes(t_parse_node self);
bool ts_node_has_error(t_parse_node self);
bool ts_node_is_error(t_parse_node self);
bool ts_node_is_extra(t_parse_node self);
bool ts_node_is_missing(t_parse_node self);
bool ts_node_is_named(t_parse_node self);
bool ts_node_is_null(t_parse_node self);
const t_language *ts_node_language(t_parse_node self);
t_parse_node ts_node_named_child(t_parse_node self, t_u32 child_index);
t_u32 ts_node_named_child_count(t_parse_node self);
t_parse_node ts_node_named_descendant_for_byte_range(t_parse_node self, t_u32 start, t_u32 end);
t_parse_node ts_node_named_descendant_for_point_range(t_parse_node self, t_point start, t_point end);
t_parse_node ts_node_new(const t_first_tree *tree, const t_subtree *subtree, t_length position, t_symbol alias);
t_parse_node ts_node_next_named_sibling(t_parse_node self);
t_state_id ts_node_next_parse_state(t_parse_node self);
t_parse_node ts_node_next_sibling(t_parse_node self);
t_parse_node ts_node_parent(t_parse_node self);
t_state_id ts_node_parse_state(t_parse_node self);
t_parse_node ts_node_prev_named_sibling(t_parse_node self);
t_parse_node ts_node_prev_sibling(t_parse_node self);
t_u32 ts_node_start_byte(t_parse_node self);
t_point ts_node_start_point(t_parse_node self);
char *ts_node_string(t_parse_node self);
t_symbol ts_node_symbol(t_parse_node self);
const char *ts_node_type(t_parse_node self);
const size_t *ts_parser_cancellation_flag(const t_first_parser *self);
void ts_parser_delete(t_first_parser *self);
const t_parse_range *ts_parser_included_ranges(const t_first_parser *self, t_u32 *count);
const t_language *ts_parser_language(const t_first_parser *self);
t_parse_logger ts_parser_logger(const t_first_parser *self);
t_first_parser *ts_parser_new(void);
t_first_tree *ts_parser_parse(t_first_parser *self, const t_first_tree *old_tree, t_parse_input input);
t_first_tree *ts_parser_parse_string(t_first_parser *self, const t_first_tree *old_tree, const char *string, t_u32 length);
t_first_tree *ts_parser_parse_string_encoding(t_first_parser *self, const t_first_tree *old_tree, const char *string, t_u32 length, t_input_encoding encoding);
void ts_parser_print_dot_graphs(t_first_parser *self, int fd);
void ts_parser_reset(t_first_parser *self);
void ts_parser_set_cancellation_flag(t_first_parser *self, const size_t *flag);
bool ts_parser_set_included_ranges(t_first_parser *self, const t_parse_range *ranges, t_u32 count);
bool ts_parser_set_language(t_first_parser *self, const t_language *language);
void ts_parser_set_logger(t_first_parser *self, t_parse_logger logger);
void ts_parser_set_timeout_micros(t_first_parser *self, t_u64 timeout_micros);
t_u64 ts_parser_timeout_micros(const t_first_parser *self);
// START PROBABLY DELETE WORTHY
bool ts_query__step_is_fallible(const t_parse_query *self, t_u16 step_index);
t_u32 ts_query_capture_count(const t_parse_query *self);
const char *ts_query_capture_name_for_id(const t_parse_query *self, t_u32 index, t_u32 *length);
t_quantifier ts_query_capture_quantifier_for_id(const t_parse_query *self, t_u32 pattern_index, t_u32 capture_index);
void ts_query_cursor__compare_captures(t_query_cursor *self, t_query_state *left_state, t_query_state *right_state, bool *left_contains_right, bool *right_contains_left);
int ts_query_cursor__compare_nodes(t_parse_node left, t_parse_node right);
void ts_query_cursor_delete(t_query_cursor *self);
bool ts_query_cursor_did_exceed_match_limit(const t_query_cursor *self);
void ts_query_cursor_exec(t_query_cursor *self, const t_parse_query *query, t_parse_node node);
t_u32 ts_query_cursor_match_limit(const t_query_cursor *self);
t_query_cursor *ts_query_cursor_new(void);
bool ts_query_cursor_next_capture(t_query_cursor *self, t_query_match *match, t_u32 *capture_index);
bool ts_query_cursor_next_match(t_query_cursor *self, t_query_match *match);
void ts_query_cursor_remove_match(t_query_cursor *self, t_u32 match_id);
void ts_query_cursor_set_byte_range(t_query_cursor *self, t_u32 start_byte, t_u32 end_byte);
void ts_query_cursor_set_match_limit(t_query_cursor *self, t_u32 limit);
void ts_query_cursor_set_max_start_depth(t_query_cursor *self, t_u32 max_start_depth);
void ts_query_cursor_set_point_range(t_query_cursor *self, t_point start_point, t_point end_point);
void ts_query_delete(t_parse_query *self);
void ts_query_disable_capture(t_parse_query *self, const char *name, t_u32 length);
void ts_query_disable_pattern(t_parse_query *self, t_u32 pattern_index);
bool ts_query_is_pattern_guaranteed_at_step(const t_parse_query *self, t_u32 byte_offset);
bool ts_query_is_pattern_non_local(const t_parse_query *self, t_u32 pattern_index);
bool ts_query_is_pattern_rooted(const t_parse_query *self, t_u32 pattern_index);
t_parse_query *ts_query_new(const t_language *language, const char *source, t_u32 source_len, t_u32 *error_offset, t_query_error *error_type);
t_u32 ts_query_pattern_count(const t_parse_query *self);
const t_query_predicate_step *ts_query_predicates_for_pattern(const t_parse_query *self, t_u32 pattern_index, t_u32 *step_count);
t_u32 ts_query_start_byte_for_pattern(const t_parse_query *self, t_u32 pattern_index);
t_u32 ts_query_string_count(const t_parse_query *self);
const char *ts_query_string_value_for_id(const t_parse_query *self, t_u32 index, t_u32 *length);
// END PROBABLY DELETE WORTHY
void ts_range_array_get_changed_ranges(const t_parse_range *old_ranges, unsigned old_range_count, const t_parse_range *new_ranges, unsigned new_range_count, t_range_array *differences);
bool ts_range_array_intersects(const t_range_array *self, unsigned start_index, t_u32 start_byte, t_u32 end_byte);
bool ts_stack_can_merge(t_stack *self, t_stack_version version1, t_stack_version version2);
void ts_stack_clear(t_stack *self);
t_stack_version ts_stack_copy_version(t_stack *self, t_stack_version version);
void ts_stack_delete(t_stack *self);
int ts_stack_dynamic_precedence(t_stack *self, t_stack_version version);
unsigned ts_stack_error_cost(const t_stack *self, t_stack_version version);
t_stack_summary *ts_stack_get_summary(t_stack *self, t_stack_version version);
void ts_stack_halt(t_stack *self, t_stack_version version);
bool ts_stack_has_advanced_since_error(const t_stack *self, t_stack_version version);
bool ts_stack_is_active(const t_stack *self, t_stack_version version);
bool ts_stack_is_halted(const t_stack *self, t_stack_version version);
bool ts_stack_is_paused(const t_stack *self, t_stack_version version);
t_subtree ts_stack_last_external_token(const t_stack *self, t_stack_version version);
bool ts_stack_merge(t_stack *self, t_stack_version version1, t_stack_version version2);
t_stack *ts_stack_new(t_subtree_pool *subtree_pool);
unsigned ts_stack_node_count_since_error(const t_stack *self, t_stack_version version);
void ts_stack_pause(t_stack *self, t_stack_version version, t_subtree lookahead);
t_stack_slice_array ts_stack_pop_all(t_stack *self, t_stack_version version);
t_stack_slice_array ts_stack_pop_count(t_stack *self, t_stack_version version, t_u32 count);
t_subtree_array ts_stack_pop_error(t_stack *self, t_stack_version version);
t_stack_slice_array ts_stack_pop_pending(t_stack *self, t_stack_version version);
t_length ts_stack_position(const t_stack *self, t_stack_version version);
bool ts_stack_print_dot_graph(t_stack *self, const t_language *language, void *f);
void ts_stack_push(t_stack *self, t_stack_version version, t_subtree subtree, bool pending, t_state_id state);
void ts_stack_record_summary(t_stack *self, t_stack_version version, unsigned max_depth);
void ts_stack_remove_version(t_stack *self, t_stack_version version);
void ts_stack_renumber_version(t_stack *self, t_stack_version v1, t_stack_version v2);
t_subtree ts_stack_resume(t_stack *self, t_stack_version version);
void ts_stack_set_last_external_token(t_stack *self, t_stack_version version, t_subtree token);
t_state_id ts_stack_state(const t_stack *self, t_stack_version version);
void ts_stack_swap_versions(t_stack *self, t_stack_version v1, t_stack_version v2);
t_u32 ts_stack_version_count(const t_stack *self);
void ts_subtree__print_dot_graph(const t_subtree *self, t_u32 start_offset, const t_language *language, t_symbol alias_symbol, void *f);
void ts_subtree_array_clear(t_subtree_pool *pool, t_subtree_array *self);
void ts_subtree_array_copy(t_subtree_array self, t_subtree_array *dest);
void ts_subtree_array_delete(t_subtree_pool *pool, t_subtree_array *self);
void ts_subtree_array_remove_trailing_extras(t_subtree_array *self, t_subtree_array *destination);
void ts_subtree_array_reverse(t_subtree_array *self);
void ts_subtree_balance(t_subtree self, t_subtree_pool *pool, const t_language *language);
t_mutable_subtree ts_subtree_clone(t_subtree self);
int ts_subtree_compare(t_subtree left, t_subtree right, t_subtree_pool *pool);
t_subtree ts_subtree_edit(t_subtree self, const t_input_edit *input_edit, t_subtree_pool *pool);
const t_external_scanner_state *ts_subtree_external_scanner_state(t_subtree self);
bool ts_subtree_external_scanner_state_eq(t_subtree self, t_subtree other);
unsigned ts_subtree_get_changed_ranges(const t_subtree *old_tree, const t_subtree *new_tree, t_tree_cursor *cursor1, t_tree_cursor *cursor2, const t_language *language, const t_range_array *included_range_differences, t_parse_range **ranges);
t_subtree ts_subtree_last_external_token(t_subtree tree);
t_mutable_subtree ts_subtree_make_mut(t_subtree_pool *pool, t_subtree self);
t_subtree ts_subtree_new_error(t_subtree_pool *pool, t_i32 lookahead_char, t_length padding, t_length size, t_u32 bytes_scanned, t_state_id parse_state, const t_language *language);
t_subtree ts_subtree_new_error_node(t_subtree_array *children, bool extra, const t_language *language);
t_subtree ts_subtree_new_leaf(t_subtree_pool *pool, t_symbol symbol, t_length padding, t_length size, t_u32 lookahead_bytes, t_state_id parse_state, bool has_external_tokens, bool depends_on_column, bool is_keyword, const t_language *language);
t_subtree ts_subtree_new_missing_leaf(t_subtree_pool *pool, t_symbol symbol, t_length padding, t_u32 lookahead_bytes, const t_language *language);
t_mutable_subtree ts_subtree_new_node(t_symbol symbol, t_subtree_array *children, unsigned production_id, const t_language *language);
void ts_subtree_pool_delete(t_subtree_pool *self);
t_subtree_pool ts_subtree_pool_new(t_u32 capacity);
void ts_subtree_release(t_subtree_pool *pool, t_subtree self);
void ts_subtree_retain(t_subtree self);
void ts_subtree_set_symbol(t_mutable_subtree *self, t_symbol symbol, const t_language *language);
char *ts_subtree_string(t_subtree self, t_symbol alias_symbol, bool alias_is_named, const t_language *language, bool include_all);
void ts_subtree_summarize_children(t_mutable_subtree self, const t_language *language);
t_first_tree *ts_tree_copy(const t_first_tree *self);
// START PROBABLY DELETE WORTHY
t_tree_cursor ts_tree_cursor_copy(const t_tree_cursor *_cursor);
t_u32 ts_tree_cursor_current_depth(const t_tree_cursor *_self);
t_u32 ts_tree_cursor_current_descendant_index(const t_tree_cursor *_self);
t_field_id ts_tree_cursor_current_field_id(const t_tree_cursor *_self);
const char *ts_tree_cursor_current_field_name(const t_tree_cursor *_self);
t_parse_node ts_tree_cursor_current_node(const t_tree_cursor *_self);
void ts_tree_cursor_current_status(const t_tree_cursor *_self, t_field_id *field_id, bool *has_later_siblings, bool *has_later_named_siblings, bool *can_have_later_siblings_with_this_field, t_symbol *supertypes, unsigned *supertype_count);
void ts_tree_cursor_delete(t_tree_cursor *_self);
void ts_tree_cursor_goto_descendant(t_tree_cursor *_self, t_u32 goal_descendant_index);
bool ts_tree_cursor_goto_first_child(t_tree_cursor *self);
t_i64 ts_tree_cursor_goto_first_child_for_byte(t_tree_cursor *self, t_u32 goal_byte);
t_i64 ts_tree_cursor_goto_first_child_for_point(t_tree_cursor *self, t_point goal_point);
t_tree_cursor_step ts_tree_cursor_goto_first_child_internal(t_tree_cursor *_self);
bool ts_tree_cursor_goto_last_child(t_tree_cursor *self);
t_tree_cursor_step ts_tree_cursor_goto_last_child_internal(t_tree_cursor *_self);
bool ts_tree_cursor_goto_next_sibling(t_tree_cursor *self);
t_tree_cursor_step ts_tree_cursor_goto_next_sibling_internal(t_tree_cursor *_self);
bool ts_tree_cursor_goto_parent(t_tree_cursor *_self);
bool ts_tree_cursor_goto_previous_sibling(t_tree_cursor *self);
t_tree_cursor_step ts_tree_cursor_goto_previous_sibling_internal(t_tree_cursor *_self);
t_tree_cursor_step ts_tree_cursor_goto_sibling_internal(t_tree_cursor *_self, bool (*advance)(t_cursor_child_iterator *, t_tree_cursor_entry *, bool *));
void ts_tree_cursor_init(t_tree_cursor *self, t_parse_node node);
t_tree_cursor ts_tree_cursor_new(t_parse_node node);
t_parse_node ts_tree_cursor_parent_node(const t_tree_cursor *_self);
void ts_tree_cursor_reset(t_tree_cursor *_self, t_parse_node node);
void ts_tree_cursor_reset_to(t_tree_cursor *_dst, const t_tree_cursor *_src);
// END PROBABLY DELETE WORTHY
void ts_tree_delete(t_first_tree *self);
void ts_tree_edit(t_first_tree *self, const t_input_edit *edit);
t_parse_range *ts_tree_get_changed_ranges(const t_first_tree *old_tree, const t_first_tree *new_tree, t_u32 *length);
t_parse_range *ts_tree_included_ranges(const t_first_tree *self, t_u32 *length);
const t_language *ts_tree_language(const t_first_tree *self);
t_first_tree *ts_tree_new(t_subtree root, const t_language *language, const t_parse_range *included_ranges, unsigned included_range_count);
void ts_tree_print_dot_graph(const t_first_tree *self, int file_descriptor);
t_parse_node ts_tree_root_node(const t_first_tree *self);
t_parse_node ts_tree_root_node_with_offset(const t_first_tree *self, t_u32 offset_bytes, t_point offset_extent);
#endif /* COMBINED_H */

98
parser/src/create_language.c
View file

@ -1,98 +0,0 @@
/* ************************************************************************** */
/* */
/* ::: :::::::: */
/* create_language.c :+: :+: :+: */
/* +:+ +:+ +:+ */
/* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */
/* +#+#+#+#+#+ +#+ */
/* Created: 2024/04/25 16:13:52 by maiboyer #+# #+# */
/* Updated: 2024/06/09 21:46:14 by maiboyer ### ########.fr */
/* */
/* ************************************************************************** */
#include "../parse_types.h"
#include "../static/headers/constants.h"
#include "../static/headers/symbols.h"
bool lex_keywords_main(t_lexer *lexer, t_state_id state);
bool lex_normal_main(t_lexer *lexer, t_state_id state);
bool tree_sitter_sh_external_scanner_scan(void *ctx, t_lexer *lexer, const bool *ret);
const bool *create_external_scanner_states(void);
const char *const *create_field_names(void);
const char *const *create_symbols_names(void);
const t_field_map_entry *create_field_map_entries(void);
const t_field_map_slice *create_field_map_slices(void);
const t_lex_modes *create_lex_modes(void);
const t_parse_action_entry *create_parse_actions_entries(void);
const t_state_id *create_primary_state_ids(void);
const t_symbol *create_alias_sequences(void);
const t_symbol *create_external_scanner_symbol_map(void);
const t_symbol *create_non_terminal_alias_map(void);
const t_symbol *create_unique_symbols_map(void);
const t_symbol_metadata *create_symbols_metadata(void);
const uint16_t *create_parse_table(void);
const uint16_t *create_small_parse_table(void);
const uint32_t *create_small_parse_table_map(void);
uint32_t tree_sitter_sh_external_scanner_serialize(void *ctx, char *s);
void tree_sitter_sh_external_scanner_deserialize(void *ctx, const char *s, uint32_t val);
void tree_sitter_sh_external_scanner_destroy(void *ctx);
void *tree_sitter_sh_external_scanner_create(void);
static t_scanner init_scanner(void)
{
return ((t_scanner){
create_external_scanner_states(),
create_external_scanner_symbol_map(),
tree_sitter_sh_external_scanner_create,
tree_sitter_sh_external_scanner_destroy,
tree_sitter_sh_external_scanner_scan,
tree_sitter_sh_external_scanner_serialize,
tree_sitter_sh_external_scanner_deserialize,
});
}
static void init_language(t_language *language)
{
language->parse_table = create_parse_table();
language->small_parse_table = create_small_parse_table();
language->small_parse_table_map = create_small_parse_table_map();
language->parse_actions = create_parse_actions_entries();
language->symbol_names = create_symbols_names();
language->field_names = create_field_names();
language->field_map_slices = create_field_map_slices();
language->field_map_entries = create_field_map_entries();
language->symbol_metadata = create_symbols_metadata();
language->public_symbol_map = create_unique_symbols_map();
language->alias_map = create_non_terminal_alias_map();
language->alias_sequences = create_alias_sequences();
language->lex_modes = create_lex_modes();
language->primary_state_ids = create_primary_state_ids();
language->lex_fn = lex_normal_main;
language->keyword_lex_fn = lex_keywords_main;
language->keyword_capture_token = sym_word;
language->external_scanner = init_scanner();
}
const t_language *tree_sitter_bash(void)
{
static bool init = false;
static t_language language = {
.version = LANGUAGE_VERSION,
.symbol_count = SYMBOL_COUNT,
.alias_count = ALIAS_COUNT,
.token_count = TOKEN_COUNT,
.external_token_count = EXTERNAL_TOKEN_COUNT,
.state_count = STATE_COUNT,
.large_state_count = LARGE_STATE_COUNT,
.production_id_count = PRODUCTION_ID_COUNT,
.field_count = FIELD_COUNT,
.max_alias_sequence_length = MAX_ALIAS_SEQUENCE_LENGTH,
};
if (!init)
{
init_language(&language);
init = true;
}
return ((t_language *)&language);
}

711
parser/src/funcs.c
View file

@ -1,711 +0,0 @@
/* ************************************************************************** */
/* */
/* ::: :::::::: */
/* funcs.c :+: :+: :+: */
/* +:+ +:+ +:+ */
/* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */
/* +#+#+#+#+#+ +#+ */
/* Created: 2024/05/30 19:21:01 by maiboyer #+# #+# */
/* Updated: 2024/05/30 19:22:53 by maiboyer ### ########.fr */
/* */
/* ************************************************************************** */
#include "./api.h"
size_t atomic_load(const volatile size_t *p)
{
return (*p);
}
uint32_t atomic_inc(volatile uint32_t *p)
{
return (++(*p));
}
uint32_t atomic_dec(volatile uint32_t *p)
{
return (--(*p));
}
bool ts_language_is_symbol_external(const t_language *self, t_symbol symbol)
{
return 0 < symbol && symbol < self->external_token_count + 1;
}
const t_parse_action *ts_language_actions(const t_language *self, t_state_id state, t_symbol symbol, uint32_t *count)
{
t_table_entry entry;
ts_language_table_entry(self, state, symbol, &entry);
*count = entry.action_count;
return entry.actions;
}
bool ts_language_has_reduce_action(const t_language *self, t_state_id state, t_symbol symbol)
{
t_table_entry entry;
ts_language_table_entry(self, state, symbol, &entry);
return entry.action_count > 0 && entry.actions[0].type == TSParseActionTypeReduce;
}
uint16_t ts_language_lookup(const t_language *self, t_state_id state, t_symbol symbol)
{
if (state >= self->large_state_count)
{
uint32_t index = self->small_parse_table_map[state - self->large_state_count];
const uint16_t *data = &self->small_parse_table[index];
uint16_t group_count = *(data++);
for (unsigned i = 0; i < group_count; i++)
{
uint16_t section_value = *(data++);
uint16_t symbol_count = *(data++);
for (unsigned j = 0; j < symbol_count; j++)
{
if (*(data++) == symbol)
return section_value;
}
}
return 0;
}
else
{
return self->parse_table[state * self->symbol_count + symbol];
}
}
bool ts_language_has_actions(const t_language *self, t_state_id state, t_symbol symbol)
{
return ts_language_lookup(self, state, symbol) != 0;
}
t_lookahead_iterator ts_language_lookaheads(const t_language *self, t_state_id state)
{
bool is_small_state = state >= self->large_state_count;
const uint16_t *data;
const uint16_t *group_end = NULL;
uint16_t group_count = 0;
if (is_small_state)
{
uint32_t index = self->small_parse_table_map[state - self->large_state_count];
data = &self->small_parse_table[index];
group_end = data + 1;
group_count = *data;
}
else
{
data = &self->parse_table[state * self->symbol_count] - 1;
}
return (t_lookahead_iterator){
.language = self,
.data = data,
.group_end = group_end,
.group_count = group_count,
.is_small_state = is_small_state,
.symbol = UINT16_MAX,
.next_state = 0,
};
}
bool ts_lookahead_iterator__next(t_lookahead_iterator *self)
{
// For small parse states, valid symbols are listed explicitly,
// grouped by their value. There's no need to look up the actions
// again until moving to the next group.
if (self->is_small_state)
{
self->data++;
if (self->data == self->group_end)
{
if (self->group_count == 0)
return false;
self->group_count--;
self->table_value = *(self->data++);
unsigned symbol_count = *(self->data++);
self->group_end = self->data + symbol_count;
self->symbol = *self->data;
}
else
{
self->symbol = *self->data;
return true;
}
}
// For large parse states, iterate through every symbol until one
// is found that has valid actions.
else
{
do
{
self->data++;
self->symbol++;
if (self->symbol >= self->language->symbol_count)
return false;
self->table_value = *self->data;
} while (!self->table_value);
}
// Depending on if the symbols is terminal or non-terminal, the table value
// either represents a list of actions or a successor state.
if (self->symbol < self->language->token_count)
{
const t_parse_action_entry *entry = &self->language->parse_actions[self->table_value];
self->action_count = entry->entry.count;
self->actions = (const t_parse_action *)(entry + 1);
self->next_state = 0;
}
else
{
self->action_count = 0;
self->next_state = self->table_value;
}
return true;
}
bool ts_language_state_is_primary(const t_language *self, t_state_id state)
{
if (self->version >= LANGUAGE_VERSION_WITH_PRIMARY_STATES)
{
return state == self->primary_state_ids[state];
}
else
{
return true;
}
}
const bool *ts_language_enabled_external_tokens(const t_language *self, unsigned external_scanner_state)
{
if (external_scanner_state == 0)
{
return NULL;
}
else
{
return self->external_scanner.states + self->external_token_count * external_scanner_state;
}
}
const t_symbol *ts_language_alias_sequence(const t_language *self, uint32_t production_id)
{
return production_id ? &self->alias_sequences[production_id * self->max_alias_sequence_length] : NULL;
}
t_symbol ts_language_alias_at(const t_language *self, uint32_t production_id, uint32_t child_index)
{
return production_id ? self->alias_sequences[production_id * self->max_alias_sequence_length + child_index] : 0;
}
void ts_language_field_map(const t_language *self, uint32_t production_id, const t_field_map_entry **start, const t_field_map_entry **end)
{
if (self->field_count == 0)
{
*start = NULL;
*end = NULL;
return;
}
t_field_map_slice slice = self->field_map_slices[production_id];
*start = &self->field_map_entries[slice.index];
*end = &self->field_map_entries[slice.index] + slice.length;
}
void ts_language_aliases_for_symbol(const t_language *self, t_symbol original_symbol, const t_symbol **start, const t_symbol **end)
{
*start = &self->public_symbol_map[original_symbol];
*end = *start + 1;
unsigned idx = 0;
for (;;)
{
t_symbol symbol = self->alias_map[idx++];
if (symbol == 0 || symbol > original_symbol)
break;
uint16_t count = self->alias_map[idx++];
if (symbol == original_symbol)
{
*start = &self->alias_map[idx];
*end = &self->alias_map[idx + count];
break;
}
idx += count;
}
}
bool length_is_undefined(t_length length)
{
return length.bytes == 0 && length.extent.column != 0;
}
t_length length_min(t_length len1, t_length len2)
{
return (len1.bytes < len2.bytes) ? len1 : len2;
}
t_length length_add(t_length len1, t_length len2)
{
t_length result;
result.bytes = len1.bytes + len2.bytes;
result.extent = point_add(len1.extent, len2.extent);
return result;
}
t_length length_sub(t_length len1, t_length len2)
{
t_length result;
result.bytes = len1.bytes - len2.bytes;
result.extent = point_sub(len1.extent, len2.extent);
return result;
}
t_length length_zero(void)
{
t_length result = {0, {0, 0}};
return result;
}
t_length length_saturating_sub(t_length len1, t_length len2)
{
if (len1.bytes > len2.bytes)
{
return length_sub(len1, len2);
}
else
{
return length_zero();
}
}
bool set_contains(t_char_range *ranges, uint32_t len, int32_t lookahead)
{
uint32_t index = 0;
uint32_t size = len - index;
while (size > 1)
{
uint32_t half_size = size / 2;
uint32_t mid_index = index + half_size;
t_char_range *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end)
{
return true;
}
else if (lookahead > range->end)
{
index = mid_index;
}
size -= half_size;
}
t_char_range *range = &ranges[index];
return (lookahead >= range->start && lookahead <= range->end);
}
t_point point__new(unsigned row, unsigned column)
{
t_point result = {row, column};
return result;
}
t_point point_add(t_point a, t_point b)
{
if (b.row > 0)
return point__new(a.row + b.row, b.column);
else
return point__new(a.row, a.column + b.column);
}
t_point point_sub(t_point a, t_point b)
{
if (a.row > b.row)
return point__new(a.row - b.row, a.column);
else
return point__new(0, a.column - b.column);
}
bool point_lte(t_point a, t_point b)
{
return (a.row < b.row) || (a.row == b.row && a.column <= b.column);
}
bool point_lt(t_point a, t_point b)
{
return (a.row < b.row) || (a.row == b.row && a.column < b.column);
}
bool point_gt(t_point a, t_point b)
{
return (a.row > b.row) || (a.row == b.row && a.column > b.column);
}
bool point_gte(t_point a, t_point b)
{
return (a.row > b.row) || (a.row == b.row && a.column >= b.column);
}
bool point_eq(t_point a, t_point b)
{
return a.row == b.row && a.column == b.column;
}
t_point point_min(t_point a, t_point b)
{
if (a.row < b.row || (a.row == b.row && a.column < b.column))
return a;
else
return b;
}
t_point point_max(t_point a, t_point b)
{
if (a.row > b.row || (a.row == b.row && a.column > b.column))
return a;
else
return b;
}
void ts_reduce_action_set_add(t_reduce_action_set *self, t_reduce_action new_action)
{
for (uint32_t i = 0; i < self->size; i++)
{
t_reduce_action action = self->contents[i];
if (action.symbol == new_action.symbol && action.count == new_action.count)
return;
}
array_push(self, new_action);
}
t_reusable_node reusable_node_new(void)
{
return (t_reusable_node){array_new(), NULL_SUBTREE};
}
void reusable_node_clear(t_reusable_node *self)
{
array_clear(&self->stack);
self->last_external_token = NULL_SUBTREE;
}
t_subtree reusable_node_tree(t_reusable_node *self)
{
return self->stack.size > 0 ? self->stack.contents[self->stack.size - 1].tree : NULL_SUBTREE;
}
uint32_t reusable_node_byte_offset(t_reusable_node *self)
{
return self->stack.size > 0 ? self->stack.contents[self->stack.size - 1].byte_offset : UINT32_MAX;
}
void reusable_node_delete(t_reusable_node *self)
{
array_delete(&self->stack);
}
void reusable_node_advance(t_reusable_node *self)
{
t_stack_entry last_entry = *array_back(&self->stack);
uint32_t byte_offset = last_entry.byte_offset + ts_subtree_total_bytes(last_entry.tree);
if (ts_subtree_has_external_tokens(last_entry.tree))
{
self->last_external_token = ts_subtree_last_external_token(last_entry.tree);
}
t_subtree tree;
uint32_t next_index;
do
{
t_stack_entry popped_entry = array_pop(&self->stack);
next_index = popped_entry.child_index + 1;
if (self->stack.size == 0)
return;
tree = array_back(&self->stack)->tree;
} while (ts_subtree_child_count(tree) <= next_index);
array_push(&self->stack, ((t_stack_entry){
.tree = ts_subtree_children(tree)[next_index],
.child_index = next_index,
.byte_offset = byte_offset,
}));
}
bool reusable_node_descend(t_reusable_node *self)
{
t_stack_entry last_entry = *array_back(&self->stack);
if (ts_subtree_child_count(last_entry.tree) > 0)
{
array_push(&self->stack, ((t_stack_entry){
.tree = ts_subtree_children(last_entry.tree)[0],
.child_index = 0,
.byte_offset = last_entry.byte_offset,
}));
return true;
}
else
{
return false;
}
}
void reusable_node_advance_past_leaf(t_reusable_node *self)
{
while (reusable_node_descend(self))
{
}
reusable_node_advance(self);
}
void reusable_node_reset(t_reusable_node *self, t_subtree tree)
{
reusable_node_clear(self);
array_push(&self->stack, ((t_stack_entry){
.tree = tree,
.child_index = 0,
.byte_offset = 0,
}));
// Never reuse the root node, because it has a non-standard internal
// structure due to transformations that are applied when it is accepted:
// adding the EOF child and any extra children.
if (!reusable_node_descend(self))
{
reusable_node_clear(self);
}
}
#define SUBTREE_GET(self, name) ((self).data.is_inline ? (self).data.name : (self).ptr->name)
t_symbol ts_subtree_symbol(t_subtree self)
{
return SUBTREE_GET(self, symbol);
}
bool ts_subtree_visible(t_subtree self)
{
return SUBTREE_GET(self, visible);
}
bool ts_subtree_named(t_subtree self)
{
return SUBTREE_GET(self, named);
}
bool ts_subtree_extra(t_subtree self)
{
return SUBTREE_GET(self, extra);
}
bool ts_subtree_has_changes(t_subtree self)
{
return SUBTREE_GET(self, has_changes);
}
bool ts_subtree_missing(t_subtree self)
{
return SUBTREE_GET(self, is_missing);
}
bool ts_subtree_is_keyword(t_subtree self)
{
return SUBTREE_GET(self, is_keyword);
}
t_state_id ts_subtree_parse_state(t_subtree self)
{
return SUBTREE_GET(self, parse_state);
}
uint32_t ts_subtree_lookahead_bytes(t_subtree self)
{
return SUBTREE_GET(self, lookahead_bytes);
}
size_t ts_subtree_alloc_size(uint32_t child_count)
{
return child_count * sizeof(t_subtree) + sizeof(t_subtree_heap_data);
}
void ts_subtree_set_extra(t_mutable_subtree *self, bool is_extra)
{
if (self->data.is_inline)
{
self->data.extra = is_extra;
}
else
{
self->ptr->extra = is_extra;
}
}
t_symbol ts_subtree_leaf_symbol(t_subtree self)
{
if (self.data.is_inline)
return self.data.symbol;
if (self.ptr->child_count == 0)
return self.ptr->symbol;
return self.ptr->first_leaf.symbol;
}
t_state_id ts_subtree_leaf_parse_state(t_subtree self)
{
if (self.data.is_inline)
return self.data.parse_state;
if (self.ptr->child_count == 0)
return self.ptr->parse_state;
return self.ptr->first_leaf.parse_state;
}
t_length ts_subtree_padding(t_subtree self)
{
if (self.data.is_inline)
{
t_length result = {self.data.padding_bytes, {self.data.padding_rows, self.data.padding_columns}};
return result;
}
else
{
return self.ptr->padding;
}
}
t_length ts_subtree_size(t_subtree self)
{
if (self.data.is_inline)
{
t_length result = {self.data.size_bytes, {0, self.data.size_bytes}};
return result;
}
else
{
return self.ptr->size;
}
}
t_length ts_subtree_total_size(t_subtree self)
{
return length_add(ts_subtree_padding(self), ts_subtree_size(self));
}
uint32_t ts_subtree_total_bytes(t_subtree self)
{
return ts_subtree_total_size(self).bytes;
}
uint32_t ts_subtree_child_count(t_subtree self)
{
return self.data.is_inline ? 0 : self.ptr->child_count;
}
uint32_t ts_subtree_repeat_depth(t_subtree self)
{
return self.data.is_inline ? 0 : self.ptr->repeat_depth;
}
uint32_t ts_subtree_is_repetition(t_subtree self)
{
return self.data.is_inline ? 0 : !self.ptr->named && !self.ptr->visible && self.ptr->child_count != 0;
}
uint32_t ts_subtree_visible_descendant_count(t_subtree self)
{
return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->visible_descendant_count;
}
uint32_t ts_subtree_visible_child_count(t_subtree self)
{
if (ts_subtree_child_count(self) > 0)
{
return self.ptr->visible_child_count;
}
else
{
return 0;
}
}
uint32_t ts_subtree_error_cost(t_subtree self)
{
if (ts_subtree_missing(self))
{
return ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY;
}
else
{
return self.data.is_inline ? 0 : self.ptr->error_cost;
}
}
int32_t ts_subtree_dynamic_precedence(t_subtree self)
{
return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->dynamic_precedence;
}
uint16_t ts_subtree_production_id(t_subtree self)
{
if (ts_subtree_child_count(self) > 0)
{
return self.ptr->production_id;
}
else
{
return 0;
}
}
bool ts_subtree_fragile_left(t_subtree self)
{
return self.data.is_inline ? false : self.ptr->fragile_left;
}
bool ts_subtree_fragile_right(t_subtree self)
{
return self.data.is_inline ? false : self.ptr->fragile_right;
}
bool ts_subtree_has_external_tokens(t_subtree self)
{
return self.data.is_inline ? false : self.ptr->has_external_tokens;
}
bool ts_subtree_has_external_scanner_state_change(t_subtree self)
{
return self.data.is_inline ? false : self.ptr->has_external_scanner_state_change;
}
bool ts_subtree_depends_on_column(t_subtree self)
{
return self.data.is_inline ? false : self.ptr->depends_on_column;
}
bool ts_subtree_is_fragile(t_subtree self)
{
return self.data.is_inline ? false : (self.ptr->fragile_left || self.ptr->fragile_right);
}
bool ts_subtree_is_error(t_subtree self)
{
return ts_subtree_symbol(self) == ts_builtin_sym_error;
}
bool ts_subtree_is_eof(t_subtree self)
{
return ts_subtree_symbol(self) == ts_builtin_sym_end;
}
t_subtree ts_subtree_from_mut(t_mutable_subtree self)
{
t_subtree result;
result.data = self.data;
return result;
}
t_mutable_subtree ts_subtree_to_mut_unsafe(t_subtree self)
{
t_mutable_subtree result;
result.data = self.data;
return result;
}
t_subtree ts_tree_cursor_current_subtree(const t_tree_cursor *_self)
{
const t_tree_cursor *self = (const t_tree_cursor *)_self;
t_tree_cursor_entry *last_entry = array_back(&self->stack);
return *last_entry->subtree;
}

173
parser/src/funcs.h
View file

@ -1,173 +0,0 @@
#ifndef FUNCS_H
#define FUNCS_H
#include "./api_structs.h"
bool length_is_undefined(t_length length);
bool point_eq(t_point a, t_point b);
bool point_gt(t_point a, t_point b);
bool point_gte(t_point a, t_point b);
bool point_lt(t_point a, t_point b);
bool point_lte(t_point a, t_point b);
bool reusable_node_descend(t_reusable_node *self);
bool set_contains(t_char_range *ranges, uint32_t len, int32_t lookahead);
bool ts_external_scanner_state_eq(const t_external_scanner_state *self, const char *, uint32_t);
bool ts_language_has_actions(const t_language *self, t_state_id state, t_symbol symbol);
bool ts_language_has_reduce_action(const t_language *self, t_state_id state, t_symbol symbol);
bool ts_language_is_symbol_external(const t_language *self, t_symbol symbol);
bool ts_language_state_is_primary(const t_language *self, t_state_id state);
bool ts_lexer_set_included_ranges(t_lexer *self, const t_parse_range *ranges, uint32_t count);
bool ts_lookahead_iterator__next(t_lookahead_iterator *self);
bool ts_range_array_intersects(const t_range_array *self, uint32_t start_index, uint32_t start_byte, uint32_t end_byte);
bool ts_stack_can_merge(t_stack *, t_stack_version, t_stack_version);
bool ts_stack_has_advanced_since_error(const t_stack *, t_stack_version);
bool ts_stack_is_active(const t_stack *, t_stack_version);
bool ts_stack_is_halted(const t_stack *, t_stack_version);
bool ts_stack_is_paused(const t_stack *, t_stack_version);
bool ts_stack_merge(t_stack *, t_stack_version, t_stack_version);
bool ts_subtree_depends_on_column(t_subtree self);
bool ts_subtree_external_scanner_state_eq(t_subtree, t_subtree);
bool ts_subtree_extra(t_subtree self);
bool ts_subtree_fragile_left(t_subtree self);
bool ts_subtree_fragile_right(t_subtree self);
bool ts_subtree_has_changes(t_subtree self);
bool ts_subtree_has_external_scanner_state_change(t_subtree self);
bool ts_subtree_has_external_tokens(t_subtree self);
bool ts_subtree_has_external_tokens(t_subtree self);
bool ts_subtree_is_eof(t_subtree self);
bool ts_subtree_is_error(t_subtree self);
bool ts_subtree_is_fragile(t_subtree self);
bool ts_subtree_is_keyword(t_subtree self);
bool ts_subtree_missing(t_subtree self);
bool ts_subtree_named(t_subtree self);
bool ts_subtree_visible(t_subtree self);
char *ts_subtree_string(t_subtree, t_symbol, bool, const t_language *, bool include_all);
const bool *ts_language_enabled_external_tokens(const t_language *self, uint32_t external_scanner_state);
const char *ts_external_scanner_state_data(const t_external_scanner_state *);
const t_external_scanner_state *ts_subtree_external_scanner_state(t_subtree self);
const t_parse_action *ts_language_actions(const t_language *self, t_state_id state, t_symbol symbol, uint32_t *count);
const t_symbol *ts_language_alias_sequence(const t_language *self, uint32_t production_id);
int ts_stack_dynamic_precedence(t_stack *, t_stack_version);
int ts_subtree_compare(t_subtree, t_subtree, t_subtree_pool *);
int32_t ts_subtree_dynamic_precedence(t_subtree self);
size_t atomic_load(const volatile size_t *p);
size_t ts_subtree_alloc_size(uint32_t child_count);
t_first_tree *ts_tree_new(t_subtree root, const t_language *language, const t_parse_range *, uint32_t);
t_length length_add(t_length len1, t_length len2);
t_length length_min(t_length len1, t_length len2);
t_length length_saturating_sub(t_length len1, t_length len2);
t_length length_sub(t_length len1, t_length len2);
t_length length_zero(void);
t_length ts_stack_position(const t_stack *, t_stack_version);
t_length ts_subtree_padding(t_subtree self);
t_length ts_subtree_size(t_subtree self);
t_length ts_subtree_total_size(t_subtree self);
t_lookahead_iterator ts_language_lookaheads(const t_language *self, t_state_id state);
t_mutable_subtree ts_subtree_make_mut(t_subtree_pool *, t_subtree);
t_mutable_subtree ts_subtree_new_node(t_symbol, t_subtree_array *, uint32_t, const t_language *);
t_mutable_subtree ts_subtree_to_mut_unsafe(t_subtree self);
t_parse_node ts_node_new(const t_first_tree *, const t_subtree *, t_length, t_symbol);
t_parse_node ts_tree_cursor_parent_node(const t_tree_cursor *);
t_parse_range *ts_lexer_included_ranges(const t_lexer *self, uint32_t *count);
t_point point__new(uint32_t row, uint32_t column);
t_point point_add(t_point a, t_point b);
t_point point_max(t_point a, t_point b);
t_point point_min(t_point a, t_point b);
t_point point_sub(t_point a, t_point b);
t_reusable_node reusable_node_new(void);
t_stack *ts_stack_new(t_subtree_pool *);
t_stack_slice_array ts_stack_pop_all(t_stack *, t_stack_version);
t_stack_slice_array ts_stack_pop_count(t_stack *, t_stack_version, uint32_t count);
t_stack_slice_array ts_stack_pop_pending(t_stack *, t_stack_version);
t_stack_summary *ts_stack_get_summary(t_stack *, t_stack_version);
t_stack_version ts_stack_copy_version(t_stack *, t_stack_version);
t_state_id ts_language_next_state(const t_language *self, t_state_id state, t_symbol symbol);
t_state_id ts_stack_state(const t_stack *, t_stack_version);
t_state_id ts_subtree_leaf_parse_state(t_subtree self);
t_state_id ts_subtree_parse_state(t_subtree self);
t_subtree reusable_node_tree(t_reusable_node *self);
t_subtree ts_stack_last_external_token(const t_stack *, t_stack_version);
t_subtree ts_stack_resume(t_stack *, t_stack_version);
t_subtree ts_subtree_edit(t_subtree, const t_input_edit *edit, t_subtree_pool *);
t_subtree ts_subtree_from_mut(t_mutable_subtree self);
t_subtree ts_subtree_last_external_token(t_subtree);
t_subtree ts_subtree_new_error(t_subtree_pool *, int32_t, t_length, t_length, uint32_t, t_state_id, const t_language *);
t_subtree ts_subtree_new_error_node(t_subtree_array *, bool, const t_language *);
t_subtree ts_subtree_new_leaf(t_subtree_pool *, t_symbol, t_length, t_length, uint32_t, t_state_id, bool, bool, bool, const t_language *);
t_subtree ts_subtree_new_missing_leaf(t_subtree_pool *, t_symbol, t_length, uint32_t, const t_language *);
t_subtree ts_tree_cursor_current_subtree(const t_tree_cursor *_self);
t_subtree_array ts_stack_pop_error(t_stack *, t_stack_version);
t_subtree_pool ts_subtree_pool_new(uint32_t capacity);
t_symbol ts_language_alias_at(const t_language *self, uint32_t production_id, uint32_t child_index);
t_symbol ts_language_public_symbol(const t_language *, t_symbol);
t_symbol ts_subtree_leaf_symbol(t_subtree self);
t_symbol ts_subtree_symbol(t_subtree self);
t_symbol_metadata ts_language_symbol_metadata(const t_language *, t_symbol);
t_tree_cursor_step ts_tree_cursor_goto_first_child_internal(t_tree_cursor *);
t_tree_cursor_step ts_tree_cursor_goto_next_sibling_internal(t_tree_cursor *);
uint16_t ts_language_lookup(const t_language *self, t_state_id state, t_symbol symbol);
uint16_t ts_subtree_production_id(t_subtree self);
uint32_t atomic_dec(volatile uint32_t *p);
uint32_t atomic_inc(volatile uint32_t *p);
uint32_t reusable_node_byte_offset(t_reusable_node *self);
uint32_t ts_stack_version_count(const t_stack *);
uint32_t ts_subtree_child_count(t_subtree self);
uint32_t ts_subtree_child_count(t_subtree self);
uint32_t ts_subtree_error_cost(t_subtree self);
uint32_t ts_subtree_is_repetition(t_subtree self);
uint32_t ts_subtree_lookahead_bytes(t_subtree self);
uint32_t ts_subtree_repeat_depth(t_subtree self);
uint32_t ts_subtree_total_bytes(t_subtree self);
uint32_t ts_subtree_total_bytes(t_subtree self);
uint32_t ts_subtree_visible_child_count(t_subtree self);
uint32_t ts_subtree_visible_descendant_count(t_subtree self);
uint32_t ts_stack_error_cost(const t_stack *, t_stack_version version);
uint32_t ts_stack_node_count_since_error(const t_stack *, t_stack_version);
uint32_t ts_subtree_get_changed_ranges(const t_subtree *old_tree, const t_subtree *new_tree, t_tree_cursor *cursor1, t_tree_cursor *cursor2, const t_language *language, const t_range_array *included_range_differences, t_parse_range **ranges);
void reusable_node_advance(t_reusable_node *self);
void reusable_node_advance_past_leaf(t_reusable_node *self);
void reusable_node_clear(t_reusable_node *self);
void reusable_node_delete(t_reusable_node *self);
void reusable_node_reset(t_reusable_node *self, t_subtree tree);
void ts_external_scanner_state_delete(t_external_scanner_state *self);
void ts_external_scanner_state_init(t_external_scanner_state *, const char *, uint32_t);
void ts_language_aliases_for_symbol(const t_language *self, t_symbol original_symbol, const t_symbol **start, const t_symbol **end);
void ts_language_field_map(const t_language *self, uint32_t production_id, const t_field_map_entry **start, const t_field_map_entry **end);
void ts_language_table_entry(const t_language *, t_state_id, t_symbol, t_table_entry *);
void ts_lexer_advance_to_end(t_lexer *lexer);
void ts_lexer_delete(t_lexer *lexer);
void ts_lexer_finish(t_lexer *lexer, uint32_t *);
void ts_lexer_init(t_lexer *lexer);
void ts_lexer_mark_end(t_lexer *lexer);
void ts_lexer_reset(t_lexer *lexer, t_length);
void ts_lexer_set_input(t_lexer *lexer, t_parse_input);
void ts_lexer_start(t_lexer *lexer);
void ts_range_array_get_changed_ranges(const t_parse_range *old_ranges, uint32_t old_range_count, const t_parse_range *new_ranges, uint32_t new_range_count, t_range_array *differences);
void ts_reduce_action_set_add(t_reduce_action_set *self, t_reduce_action new_action);
void ts_stack_clear(t_stack *);
void ts_stack_delete(t_stack *);
void ts_stack_halt(t_stack *, t_stack_version);
void ts_stack_pause(t_stack *, t_stack_version, t_subtree);
void ts_stack_push(t_stack *, t_stack_version, t_subtree, bool, t_state_id);
void ts_stack_record_summary(t_stack *, t_stack_version, uint32_t max_depth);
void ts_stack_remove_version(t_stack *, t_stack_version);
void ts_stack_renumber_version(t_stack *, t_stack_version, t_stack_version);
void ts_stack_set_last_external_token(t_stack *, t_stack_version, t_subtree);
void ts_stack_swap_versions(t_stack *, t_stack_version, t_stack_version);
void ts_subtree_array_clear(t_subtree_pool *, t_subtree_array *);
void ts_subtree_array_copy(t_subtree_array, t_subtree_array *);
void ts_subtree_array_delete(t_subtree_pool *, t_subtree_array *);
void ts_subtree_array_remove_trailing_extras(t_subtree_array *, t_subtree_array *);
void ts_subtree_array_reverse(t_subtree_array *);
void ts_subtree_balance(t_subtree, t_subtree_pool *, const t_language *);
void ts_subtree_pool_delete(t_subtree_pool *);
void ts_subtree_release(t_subtree_pool *, t_subtree);
void ts_subtree_retain(t_subtree);
void ts_subtree_set_extra(t_mutable_subtree *self, bool is_extra);
void ts_subtree_set_symbol(t_mutable_subtree *, t_symbol, const t_language *);
void ts_subtree_summarize(t_mutable_subtree, const t_subtree *, uint32_t, const t_language *);
void ts_subtree_summarize_children(t_mutable_subtree, const t_language *);
void ts_tree_cursor_current_status(const t_tree_cursor *, t_field_id *, bool *, bool *, bool *, t_symbol *, uint32_t *);
void ts_tree_cursor_init(t_tree_cursor *, t_parse_node);
#endif // FUNCS_H

1244
parser/src/scanner.c
View file

File diff suppressed because it is too large Load diff

543
parser/src/structs.h
View file

@ -1,543 +0,0 @@
#ifndef STRUCTS_H
#define STRUCTS_H
#include "./api.h"
typedef unsigned t_stack_action;
typedef struct s_analysis_state t_analysis_state;
typedef struct s_analysis_state_entry t_analysis_state_entry;
typedef struct s_analysis_subgraph t_analysis_subgraph;
typedef struct s_analysis_subgraph_node t_analysis_subgraph_node;
typedef struct s_capture_list_pool t_capture_list_pool;
typedef struct s_cursor_child_iterator t_cursor_child_iterator;
typedef struct s_edit t_edit;
typedef struct s_edit_entry t_edit_entry;
typedef struct s_error_status t_error_status;
typedef struct s_first_parser t_first_parser;
typedef struct s_iterator t_iterator;
typedef struct s_node_child_iterator t_node_child_iterator;
typedef struct s_parse_query t_parse_query;
typedef struct s_pattern_entry t_pattern_entry;
typedef struct s_query_analysis t_query_analysis;
typedef struct s_query_cursor t_query_cursor;
typedef struct s_query_pattern t_query_pattern;
typedef struct s_query_state t_query_state;
typedef struct s_query_step t_query_step;
typedef struct s_slice t_slice;
typedef struct s_stack t_stack;
typedef struct s_stack_head t_stack_head;
typedef struct s_stack_iterator t_stack_iterator;
typedef struct s_stack_link t_stack_link;
typedef struct s_stack_node t_stack_node;
typedef struct s_state_predecessor_map t_state_predecessor_map;
typedef struct s_step_offset t_step_offset;
typedef struct s_stream t_stream;
typedef struct s_string_input t_string_input;
typedef struct s_summarize_stack_session t_summarize_stack_session;
typedef struct s_symbol_table t_symbol_table;
typedef struct s_token_cache t_token_cache;
typedef t_stack_action (*t_stack_callback)(void *, const t_stack_iterator *);
typedef uint32_t (*t_unicode_decode_function)(const uint8_t *chunk,
uint32_t size,
int32_t *codepoint);
typedef Array(t_analysis_state *) t_analysis_state_set;
typedef Array(t_analysis_subgraph) t_analysis_subgraph_array;
typedef Array(t_query_capture) t_capture_list;
typedef Array(t_stack_node *) t_stack_node_array;
typedef Array(uint8_t) t_capture_quantifiers;
typedef enum e_stack_status t_stack_status;
typedef enum e_error_comparaison t_error_comparaison;
typedef enum e_iterator_comparison t_iterator_comparison;
struct s_iterator
{
t_tree_cursor cursor;
const t_language *language;
unsigned visible_depth;
bool in_padding;
};
enum e_iterator_comparison
{
IteratorDiffers,
IteratorMayDiffer,
IteratorMatches,
};
struct s_node_child_iterator
{
t_subtree parent;
const t_first_tree *tree;
t_length position;
uint32_t child_index;
uint32_t structural_child_index;
const t_symbol *alias_sequence;
};
struct s_token_cache
{
t_subtree token;
t_subtree last_external_token;
uint32_t byte_index;
};
struct s_first_parser
{
t_lexer lexer;
t_stack *stack;
t_subtree_pool tree_pool;
const t_language *language;
t_reduce_action_set reduce_actions;
t_subtree finished_tree;
t_subtree_array trailing_extras;
t_subtree_array trailing_extras2;
t_subtree_array scratch_trees;
t_token_cache token_cache;
t_reusable_node reusable_node;
void *external_scanner_payload;
t_parser_clock end_clock;
t_parser_duration timeout_duration;
unsigned accept_count;
unsigned operation_count;
const volatile size_t *cancellation_flag;
t_subtree old_tree;
t_range_array included_range_differences;
unsigned included_range_difference_index;
bool has_scanner_error;
};
struct s_error_status
{
unsigned cost;
unsigned node_count;
int dynamic_precedence;
bool is_in_error;
};
enum e_error_comparaison
{
ErrorComparisonTakeLeft,
ErrorComparisonPreferLeft,
ErrorComparisonNone,
ErrorComparisonPreferRight,
ErrorComparisonTakeRight,
};
struct s_string_input
{
const char *string;
uint32_t length;
};
/*
* t_stream - A sequence of unicode characters derived from a UTF8 string.
* This struct is used in parsing queries from S-expressions.
*/
struct s_stream
{
const char *input;
const char *start;
const char *end;
int32_t next;
uint8_t next_size;
};
/*
* t_query_step - A step in the process of matching a query. Each node within
* a query S-expression corresponds to one of these steps. An entire pattern
* is represented as a sequence of these steps. The basic properties of a
* node are represented by these fields:
* - `symbol` - The grammar symbol to match. A zero value represents the
* wildcard symbol, '_'.
* - `field` - The field name to match. A zero value means that a field name
* was not specified.
* - `capture_ids` - An array of integers representing the names of captures
* associated with this node in the pattern, terminated by a `NONE` value.
* - `depth` - The depth where this node occurs in the pattern. The root node
* of the pattern has depth zero.
* - `negated_field_list_id` - An id representing a set of fields that must
* not be present on a node matching this step.
*
* Steps have some additional fields in order to handle the `.` (or "anchor")
* operator, which forbids additional child nodes:
* - `is_immediate` - Indicates that the node matching this step cannot be
* preceded by other sibling nodes that weren't specified in the pattern.
* - `is_last_child` - Indicates that the node matching this step cannot have
* any subsequent named siblings.
*
* For simple patterns, steps are matched in sequential order. But in order to
* handle alternative/repeated/optional sub-patterns, query steps are not always
* structured as a linear sequence; they sometimes need to split and merge. This
* is done using the following fields:
* - `alternative_index` - The index of a different query step that serves as
* an alternative to this step. A `NONE` value represents no alternative.
* When a query state reaches a step with an alternative index, the state
* is duplicated, with one copy remaining at the original step, and one copy
* moving to the alternative step. The alternative may have its own
* alternative step, so this splitting is an iterative process.
* - `is_dead_end` - Indicates that this state cannot be passed directly, and
* exists only in order to redirect to an alternative index, with no
* splitting.
* - `is_pass_through` - Indicates that state has no matching logic of its own,
* and exists only to split a state. One copy of the state advances
* immediately to the next step, and one moves to the alternative step.
* - `alternative_is_immediate` - Indicates that this step's alternative step
* should be treated as if `is_immediate` is true.
*
* Steps also store some derived state that summarizes how they relate to other
* steps within the same pattern. This is used to optimize the matching process:
* - `contains_captures` - Indicates that this step or one of its child steps
* has a non-empty `capture_ids` list.
* - `parent_pattern_guaranteed` - Indicates that if this step is reached, then
* it and all of its subsequent sibling steps within the same parent pattern
* are guaranteed to match.
* - `root_pattern_guaranteed` - Similar to `parent_pattern_guaranteed`, but
* for the entire top-level pattern. When iterating through a query's
* captures using `ts_query_cursor_next_capture`, this field is used to
* detect that a capture can safely be returned from a match that has not
* even completed yet.
*/
struct s_query_step
{
t_symbol symbol;
t_symbol supertype_symbol;
t_field_id field;
uint16_t capture_ids[MAX_STEP_CAPTURE_COUNT];
uint16_t depth;
uint16_t alternative_index;
uint16_t negated_field_list_id;
bool is_named : 1;
bool is_immediate : 1;
bool is_last_child : 1;
bool is_pass_through : 1;
bool is_dead_end : 1;
bool alternative_is_immediate : 1;
bool contains_captures : 1;
bool root_pattern_guaranteed : 1;
bool parent_pattern_guaranteed : 1;
};
/*
* t_slice - A slice of an external array. Within a query, capture names,
* literal string values, and predicate step information are stored in three
* contiguous arrays. Individual captures, string values, and predicates are
* represented as slices of these three arrays.
*/
struct s_slice
{
uint32_t offset;
uint32_t length;
};
/*
* t_symbol_table - a two-way mapping of strings to ids.
*/
struct s_symbol_table
{
Array(char) characters;
Array(t_slice) slices;
};
/**
* CaptureQuantififers - a data structure holding the quantifiers of pattern
* captures.
*/
/*
* t_pattern_entry - Information about the starting point for matching a
* particular pattern. These entries are stored in a 'pattern map' - a sorted
* array that makes it possible to efficiently lookup patterns based on the
* symbol for their first step. The entry consists of the following fields:
* - `pattern_index` - the index of the pattern within the query
* - `step_index` - the index of the pattern's first step in the shared `steps`
* array
* - `is_rooted` - whether or not the pattern has a single root node. This
* property affects decisions about whether or not to start the pattern for
* nodes outside of a QueryCursor's range restriction.
*/
struct s_pattern_entry
{
uint16_t step_index;
uint16_t pattern_index;
bool is_rooted;
};
struct s_query_pattern
{
t_slice steps;
t_slice predicate_steps;
uint32_t start_byte;
bool is_non_local;
};
struct s_step_offset
{
uint32_t byte_offset;
uint16_t step_index;
};
/*
* t_query_state - The state of an in-progress match of a particular pattern
* in a query. While executing, a `t_query_cursor` must keep track of a number
* of possible in-progress matches. Each of those possible matches is
* represented as one of these states. Fields:
* - `id` - A numeric id that is exposed to the public API. This allows the
* caller to remove a given match, preventing any more of its captures
* from being returned.
* - `start_depth` - The depth in the tree where the first step of the state's
* pattern was matched.
* - `pattern_index` - The pattern that the state is matching.
* - `consumed_capture_count` - The number of captures from this match that
* have already been returned.
* - `capture_list_id` - A numeric id that can be used to retrieve the state's
* list of captures from the `t_capture_list_pool`.
* - `seeking_immediate_match` - A flag that indicates that the state's next
* step must be matched by the very next sibling. This is used when
* processing repetitions.
* - `has_in_progress_alternatives` - A flag that indicates that there is are
* other states that have the same captures as this state, but are at
* different steps in their pattern. This means that in order to obey the
* 'longest-match' rule, this state should not be returned as a match until
* it is clear that there can be no other alternative match with more
* captures.
*/
struct s_query_state
{
uint32_t id;
uint32_t capture_list_id;
uint16_t start_depth;
uint16_t step_index;
uint16_t pattern_index;
uint16_t consumed_capture_count : 12;
bool seeking_immediate_match : 1;
bool has_in_progress_alternatives : 1;
bool dead : 1;
bool needs_parent : 1;
};
/*
* t_capture_list_pool - A collection of *lists* of captures. Each query state
* needs to maintain its own list of captures. To avoid repeated allocations,
* this struct maintains a fixed set of capture lists, and keeps track of which
* ones are currently in use by a query state.
*/
struct s_capture_list_pool
{
Array(t_capture_list) list;
t_capture_list empty_list;
// The maximum number of capture lists that we are allowed to allocate. We
// never allow `list` to allocate more entries than this, dropping pending
// matches if needed to stay under the limit.
uint32_t max_capture_list_count;
// The number of capture lists allocated in `list` that are not currently in
// use. We reuse those existing-but-unused capture lists before trying to
// allocate any new ones. We use an invalid value (UINT32_MAX) for a capture
// list's length to indicate that it's not in use.
uint32_t free_capture_list_count;
};
/*
* t_analysis_state - The state needed for walking the parse table when
* analyzing a query pattern, to determine at which steps the pattern might fail
* to match.
*/
struct s_analysis_state_entry
{
t_state_id parse_state;
t_symbol parent_symbol;
uint16_t child_index;
t_field_id field_id : 15;
bool done : 1;
};
struct s_analysis_state
{
t_analysis_state_entry stack[MAX_ANALYSIS_STATE_DEPTH];
uint16_t depth;
uint16_t step_index;
t_symbol root_symbol;
};
struct s_query_analysis
{
t_analysis_state_set states;
t_analysis_state_set next_states;
t_analysis_state_set deeper_states;
t_analysis_state_set state_pool;
Array(uint16_t) final_step_indices;
Array(t_symbol) finished_parent_symbols;
bool did_abort;
};
/*
* t_analysis_subgraph - A subset of the states in the parse table that are used
* in constructing nodes with a certain symbol. Each state is accompanied by
* some information about the possible node that could be produced in
* downstream states.
*/
struct s_analysis_subgraph_node
{
t_state_id state;
uint16_t production_id;
uint8_t child_index : 7;
bool done : 1;
};
struct s_analysis_subgraph
{
t_symbol symbol;
Array(t_state_id) start_states;
Array(t_analysis_subgraph_node) nodes;
};
/*
* t_state_predecessor_map - A map that stores the predecessors of each parse
* state. This is used during query analysis to determine which parse states can
* lead to which reduce actions.
*/
struct s_state_predecessor_map
{
t_state_id *contents;
};
/*
* t_parse_query - A tree query, compiled from a string of S-expressions. The
* query itself is immutable. The mutable state used in the process of executing
* the query is stored in a `t_query_cursor`.
*/
struct s_parse_query
{
t_symbol_table captures;
t_symbol_table predicate_values;
Array(t_capture_quantifiers) capture_quantifiers;
Array(t_query_step) steps;
Array(t_pattern_entry) pattern_map;
Array(t_query_predicate_step) predicate_steps;
Array(t_query_pattern) patterns;
Array(t_step_offset) step_offsets;
Array(t_field_id) negated_fields;
Array(char) string_buffer;
Array(t_symbol) repeat_symbols_with_rootless_patterns;
const t_language *language;
uint16_t wildcard_root_pattern_count;
};
/*
* t_query_cursor - A stateful struct used to execute a query on a tree.
*/
struct s_query_cursor
{
const t_parse_query *query;
t_tree_cursor cursor;
Array(t_query_state) states;
Array(t_query_state) finished_states;
t_capture_list_pool capture_list_pool;
uint32_t depth;
uint32_t max_start_depth;
uint32_t start_byte;
uint32_t end_byte;
t_point start_point;
t_point end_point;
uint32_t next_state_id;
bool on_visible_node;
bool ascending;
bool halted;
bool did_exceed_match_limit;
};
struct s_stack_link
{
t_stack_node *node;
t_subtree subtree;
bool is_pending;
};
struct s_stack_node
{
t_state_id state;
t_length position;
t_stack_link links[MAX_LINK_COUNT];
short unsigned int link_count;
uint32_t ref_count;
unsigned error_cost;
unsigned node_count;
int dynamic_precedence;
};
struct s_stack_iterator
{
t_stack_node *node;
t_subtree_array subtrees;
uint32_t subtree_count;
bool is_pending;
};
enum e_stack_status
{
StackStatusActive,
StackStatusPaused,
StackStatusHalted,
};
struct s_stack_head
{
t_stack_node *node;
t_stack_summary *summary;
unsigned node_count_at_last_error;
t_subtree last_external_token;
t_subtree lookahead_when_paused;
t_stack_status status;
};
struct s_stack
{
Array(t_stack_head) heads;
t_stack_slice_array slices;
Array(t_stack_iterator) iterators;
t_stack_node_array node_pool;
t_stack_node *base_node;
t_subtree_pool *subtree_pool;
};
enum e_stack_action
{
StackActionNone,
StackActionStop = 1,
StackActionPop = 2,
};
struct s_summarize_stack_session
{
t_stack_summary *summary;
unsigned max_depth;
};
struct s_edit
{
t_length start;
t_length old_end;
t_length new_end;
};
struct s_edit_entry
{
t_subtree *tree;
t_edit edit;
};
struct s_cursor_child_iterator
{
t_subtree parent;
const t_first_tree *tree;
t_length position;
uint32_t child_index;
uint32_t structural_child_index;
uint32_t descendant_index;
const t_symbol *alias_sequence;
};
#endif // STRUCTS_H