raphael/minishell
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

working parser tm

Browse source
This commit is contained in:
Maieul BOYER 2024-06-30 18:32:35 +02:00
parent 8d192a0abc
commit 2d88b6af25
No known key found for this signature in database
51 changed files with 33 additions and 31734 deletions
Download patch file Download diff file Expand all files Collapse all files

65
parser/src/api.h
View file

@ -1,65 +0,0 @@
#ifndef TREE_SITTER_ARRAY_H_
#define TREE_SITTER_ARRAY_H_
#include "me/char/char.h"
#include "me/mem/mem.h"
#include <assert.h>
#include <limits.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "./api_structs.h"
#include "./array.h"
#include "./funcs.h"
#define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1)
#define LANGUAGE_VERSION_WITH_PRIMARY_STATES 14
#define LANGUAGE_VERSION_USABLE_VIA_WASM 13
#define ERROR_STATE 0
#define ERROR_COST_PER_RECOVERY 500
#define ERROR_COST_PER_MISSING_TREE 110
#define ERROR_COST_PER_SKIPPED_TREE 100
#define ERROR_COST_PER_SKIPPED_LINE 30
#define ERROR_COST_PER_SKIPPED_CHAR 1
#define MAX_STEP_CAPTURE_COUNT 3
#define MAX_NEGATED_FIELD_COUNT 8
#define MAX_STATE_PREDECESSOR_COUNT 256
#define MAX_ANALYSIS_STATE_DEPTH 8
#define MAX_ANALYSIS_ITERATION_COUNT 256
#define MAX_LINK_COUNT 8
#define MAX_NODE_POOL_SIZE 50
#define MAX_ITERATOR_COUNT 64
#define TS_MAX_INLINE_TREE_LENGTH UINT8_MAX
#define TS_MAX_TREE_POOL_SIZE 32
#define ts_builtin_sym_error ((t_symbol)-1)
#define ts_builtin_sym_end 0
#define POINT_ZERO ((t_point){0, 0})
#define POINT_MAX ((t_point){UINT32_MAX, UINT32_MAX})
#define TS_TREE_STATE_NONE USHRT_MAX
#define NULL_SUBTREE ((t_subtree){.ptr = NULL})
#define STACK_VERSION_NONE ((t_stack_version)-1)
#define TS_DECODE_ERROR (-1)
#if true
# undef malloc
# undef calloc
# undef realloc
# undef free
# define malloc(s) mem_alloc((s))
# define calloc(s, l) mem_alloc_array((s), (l))
# define realloc(p, t) mem_realloc((p), (t))
# define free(p) mem_free((p))
#endif
// Get a subtree's children, which are allocated immediately before the
// tree's own heap data.
#define ts_subtree_children(self) ((self).data.is_inline ? NULL : (t_subtree *)((self).ptr) - (self).ptr->child_count)
static const t_length LENGTH_UNDEFINED = {0, {0, 1}};
static const t_length LENGTH_MAX = {UINT32_MAX, {UINT32_MAX, UINT32_MAX}};
#endif // TREE_SITTER_TREE_H_

546
parser/src/api_structs.h
View file

@ -1,546 +0,0 @@
#ifndef API_STRUCTS_H
#define API_STRUCTS_H
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
#include "./array.h"
#include "me/types.h"
#include <stdint.h>
typedef uint16_t t_field_id;
typedef uint16_t t_state_id;
typedef uint16_t t_symbol;
typedef uint64_t t_parser_clock;
typedef uint64_t t_parser_duration;
typedef uint32_t t_stack_version;
typedef union u_parse_action_entry t_parse_action_entry;
typedef union u_subtree t_subtree;
typedef union u_mutable_subtree t_mutable_subtree;
typedef union u_parse_action t_parse_action;
typedef struct s_char_range t_char_range;
typedef struct s_external_scanner_state t_external_scanner_state;
typedef struct s_field_map_entry t_field_map_entry;
typedef struct s_field_map_slice t_field_map_slice;
typedef struct s_first_parser t_first_parser;
typedef struct s_first_tree t_first_tree;
typedef struct s_input_edit t_input_edit;
typedef struct s_language t_language;
typedef struct s_length t_length;
typedef struct s_lex_mode t_lex_mode;
typedef struct s_lexer t_lexer;
typedef struct s_lexer_data t_lexer_data;
typedef struct s_lookahead_iterator t_lookahead_iterator;
typedef struct s_parse_input t_parse_input;
typedef struct s_parse_logger t_parse_logger;
typedef struct s_parse_node t_parse_node;
typedef struct s_parse_query t_parse_query;
typedef struct s_parse_query_cursor t_parse_query_cursor;
typedef struct s_parse_query_error t_parse_query_error;
typedef struct s_parse_query_error_cost t_parse_query_error_cost;
typedef struct s_parse_range t_parse_range;
typedef struct s_parse_state t_parse_state;
typedef struct s_point t_point;
typedef struct s_query_capture t_query_capture;
typedef struct s_query_cursor t_query_cursor;
typedef struct s_query_match t_query_match;
typedef struct s_query_predicate_step t_query_predicate_step;
typedef struct s_reduce_action t_reduce_action;
typedef struct s_reusable_node t_reusable_node;
typedef struct s_stack t_stack;
typedef struct s_stack_entry t_stack_entry;
typedef struct s_stack_slice t_stack_slice;
typedef struct s_stack_summary_entry t_stack_summary_entry;
typedef struct s_subtree_heap_data t_subtree_heap_data;
typedef struct s_subtree_inline_data t_subtree_inline_data;
typedef struct s_subtree_pool t_subtree_pool;
typedef struct s_symbol_metadata t_symbol_metadata;
typedef struct s_table_entry t_table_entry;
typedef struct s_tree_cursor t_tree_cursor;
typedef struct s_tree_cursor_entry t_tree_cursor_entry;
typedef enum e_input_encoding t_input_encoding;
typedef enum e_log_type t_log_type;
typedef enum e_parse_action_type t_parse_action_type;
typedef enum e_quantifier t_quantifier;
typedef enum e_query_error t_query_error;
typedef enum e_query_predicate_step_type t_query_predicate_step_type;
typedef enum e_symbol_type t_symbol_type;
typedef Array(t_parse_range) t_range_array;
typedef Array(t_subtree) t_subtree_array;
typedef Array(t_mutable_subtree) t_mutable_subtree_array;
typedef Array(t_reduce_action) t_reduce_action_set;
typedef Array(t_stack_slice) t_stack_slice_array;
typedef Array(t_stack_summary_entry) t_stack_summary;
typedef void (*StackIterateCallback)(void *, t_state_id, uint32_t);
struct s_point
{
uint32_t row;
uint32_t column;
};
struct s_length
{
uint32_t bytes;
t_point extent;
};
struct s_stack_slice
{
t_subtree_array subtrees;
t_stack_version version;
};
struct s_stack_summary_entry
{
t_length position;
unsigned depth;
t_state_id state;
};
enum e_input_encoding
{
TSInputEncodingUTF8,
TSInputEncodingUTF16,
};
enum e_symbol_type
{
TSSymbolTypeRegular,
TSSymbolTypeAnonymous,
TSSymbolTypeAuxiliary,
};
struct s_parse_range
{
t_point start_point;
t_point end_point;
uint32_t start_byte;
uint32_t end_byte;
};
struct s_parse_input
{
void *payload;
const char *(*read)(void *payload, uint32_t byte_index, t_point position, uint32_t *bytes_read);
t_input_encoding encoding;
};
enum e_log_type
{
TSLogTypeParse,
TSLogTypeLex,
};
struct s_parse_logger
{
void *payload;
void (*log)(void *payload, t_log_type log_type, const char *buffer);
};
struct s_input_edit
{
uint32_t start_byte;
uint32_t old_end_byte;
uint32_t new_end_byte;
t_point start_point;
t_point old_end_point;
t_point new_end_point;
};
struct s_parse_node
{
uint32_t context[4];
const void *id;
const t_first_tree *tree;
};
struct s_tree_cursor_entry
{
const t_subtree *subtree;
t_length position;
uint32_t child_index;
uint32_t structural_child_index;
uint32_t descendant_index;
};
struct s_tree_cursor
{
const t_first_tree *tree;
Array(t_tree_cursor_entry) stack;
t_symbol root_alias_symbol;
};
struct s_query_capture
{
t_parse_node node;
uint32_t index;
};
enum e_quantifier
{
TSQuantifierZero = 0, // must match the array initialization value
TSQuantifierZeroOrOne,
TSQuantifierZeroOrMore,
TSQuantifierOne,
TSQuantifierOneOrMore,
};
struct s_query_match
{
uint32_t id;
uint16_t pattern_index;
uint16_t capture_count;
const t_query_capture *captures;
};
enum e_query_predicate_step_type
{
TSQueryPredicateStepTypeDone,
TSQueryPredicateStepTypeCapture,
TSQueryPredicateStepTypeString,
};
struct s_query_predicate_step
{
t_query_predicate_step_type type;
uint32_t value_id;
};
enum e_query_error
{
TSQueryErrorNone = 0,
TSQueryErrorSyntax,
TSQueryErrorNodeType,
TSQueryErrorField,
TSQueryErrorCapture,
TSQueryErrorStructure,
TSQueryErrorLanguage,
};
struct s_parent_cache_entry
{
const t_subtree *child;
const t_subtree *parent;
t_length position;
t_symbol alias_symbol;
};
typedef enum e_tree_cursor_step
{
TreeCursorStepNone,
TreeCursorStepHidden,
TreeCursorStepVisible,
} t_tree_cursor_step;
// The serialized state of an external scanner.
//
// Every time an external token subtree is created after a call to an
// external scanner, the scanner's `serialize` function is called to
// retrieve a serialized copy of its state. The bytes are then copied
// onto the subtree itself so that the scanner's state can later be
// restored using its `deserialize` function.
//
// Small byte arrays are stored inline, and long ones are allocated
// separately on the heap.
struct s_external_scanner_state
{
// TODO: extract this anonymous struct
union {
char *long_data;
char short_data[24];
};
uint32_t length;
};
struct s_subtree_inline_data
{
bool is_inline : 1;
bool visible : 1;
bool named : 1;
bool extra : 1;
bool has_changes : 1;
bool is_missing : 1;
bool is_keyword : 1;
uint8_t symbol;
uint16_t parse_state;
uint8_t padding_columns;
uint8_t padding_rows : 4;
uint8_t lookahead_bytes : 4;
uint8_t padding_bytes;
uint8_t size_bytes;
};
struct s_subtree_heap_data
{
volatile uint32_t ref_count;
t_length padding;
t_length size;
uint32_t lookahead_bytes;
uint32_t error_cost;
uint32_t child_count;
t_symbol symbol;
t_state_id parse_state;
bool visible : 1;
bool named : 1;
bool extra : 1;
bool fragile_left : 1;
bool fragile_right : 1;
bool has_changes : 1;
bool has_external_tokens : 1;
bool has_external_scanner_state_change : 1;
bool depends_on_column : 1;
bool is_missing : 1;
bool is_keyword : 1;
// TODO: extract these anonymous struct
union {
// Non-terminal subtrees (`child_count > 0`)
struct
{
uint32_t visible_child_count;
uint32_t named_child_count;
uint32_t visible_descendant_count;
int32_t dynamic_precedence;
uint16_t repeat_depth;
uint16_t production_id;
struct
{
t_symbol symbol;
t_state_id parse_state;
} first_leaf;
};
// External terminal subtrees (`child_count == 0 &&
// has_external_tokens`)
t_external_scanner_state external_scanner_state;
// Error terminal subtrees (`child_count == 0 && symbol ==
// ts_builtin_sym_error`)
int32_t lookahead_char;
};
};
// The fundamental building block of a syntax tree.
union u_subtree {
t_subtree_inline_data data;
const t_subtree_heap_data *ptr;
};
// Like t_subtree, but mutable.
union u_mutable_subtree {
t_subtree_inline_data data;
t_subtree_heap_data *ptr;
};
struct s_subtree_pool
{
t_mutable_subtree_array free_trees;
t_mutable_subtree_array tree_stack;
};
union u_parse_action {
// TODO: extract this anonymous struct
struct
{
uint8_t type;
t_state_id state;
bool extra;
bool repetition;
} shift;
// TODO: extract this anonymous struct
struct
{
uint8_t type;
uint8_t child_count;
t_symbol symbol;
int16_t dynamic_precedence;
uint16_t production_id;
} reduce;
uint8_t type;
};
struct s_table_entry
{
const t_parse_action *actions;
uint32_t action_count;
bool is_reusable;
};
struct s_lookahead_iterator
{
const t_language *language;
const uint16_t *data;
const uint16_t *group_end;
t_state_id state;
uint16_t table_value;
uint16_t section_index;
uint16_t group_count;
bool is_small_state;
const t_parse_action *actions;
t_symbol symbol;
t_state_id next_state;
uint16_t action_count;
};
struct s_symbol_metadata
{
bool visible;
bool named;
bool supertype;
};
enum e_parse_action_type
{
TSParseActionTypeShift,
TSParseActionTypeReduce,
TSParseActionTypeAccept,
TSParseActionTypeRecover,
};
union u_parse_action_entry {
t_parse_action action;
// TODO: extract this anonymous struct
struct
{
uint8_t count;
bool reusable;
} entry;
};
struct s_field_map_entry
{
t_field_id field_id;
uint8_t child_index;
bool inherited;
};
struct s_field_map_slice
{
uint16_t index;
uint16_t length;
};
struct s_lexer_data
{
int32_t lookahead;
t_symbol result_symbol;
void (*advance)(t_lexer_data *, bool);
void (*mark_end)(t_lexer_data *);
uint32_t (*get_column)(t_lexer_data *);
bool (*is_at_included_range_start)(const t_lexer_data *);
bool (*eof)(const t_lexer_data *);
};
struct s_lex_mode
{
uint16_t lex_state;
uint16_t external_lex_state;
};
struct s_char_range
{
int32_t start;
int32_t end;
};
struct s_language
{
uint32_t version;
uint32_t symbol_count;
uint32_t alias_count;
uint32_t token_count;
uint32_t external_token_count;
uint32_t state_count;
uint32_t large_state_count;
uint32_t production_id_count;
uint32_t field_count;
uint16_t max_alias_sequence_length;
const uint16_t *parse_table;
const uint16_t *small_parse_table;
const uint32_t *small_parse_table_map;
const t_parse_action_entry *parse_actions;
const char *const *symbol_names;
const char *const *field_names;
const t_field_map_slice *field_map_slices;
const t_field_map_entry *field_map_entries;
const t_symbol_metadata *symbol_metadata;
const t_symbol *public_symbol_map;
const uint16_t *alias_map;
const t_symbol *alias_sequences;
const t_lex_mode *lex_modes;
bool (*lex_fn)(t_lexer_data *, t_state_id);
bool (*keyword_lex_fn)(t_lexer_data *, t_state_id);
t_symbol keyword_capture_token;
// TODO: extract this anonymous struct
struct
{
const bool *states;
const t_symbol *symbol_map;
void *(*create)(void);
void (*destroy)(void *);
bool (*scan)(void *, t_lexer_data *, const bool *symbol_whitelist);
uint32_t (*serialize)(void *, char *);
void (*deserialize)(void *, const char *, uint32_t);
} external_scanner;
const t_state_id *primary_state_ids;
};
struct s_lexer
{
t_lexer_data data;
t_length current_position;
t_length token_start_position;
t_length token_end_position;
t_parse_range *included_ranges;
const char *chunk;
t_parse_input input;
t_parse_logger logger;
uint32_t included_range_count;
uint32_t current_included_range_index;
uint32_t chunk_start;
uint32_t chunk_size;
uint32_t lookahead_size;
bool did_get_column;
char debug_buffer[TREE_SITTER_SERIALIZATION_BUFFER_SIZE];
};
struct s_reduce_action
{
uint32_t count;
t_symbol symbol;
int32_t dynamic_precedence;
uint16_t production_id;
};
struct s_stack_entry
{
t_subtree tree;
uint32_t child_index;
uint32_t byte_offset;
};
struct s_reusable_node
{
Array(t_stack_entry) stack;
t_subtree last_external_token;
};
struct s_first_tree
{
t_subtree root;
const t_language *language;
t_parse_range *included_ranges;
uint32_t included_range_count;
};
#endif // API_STRUCTS_H

283
parser/src/array.h
View file

@ -1,283 +0,0 @@
#ifndef ARRAY_H
#define ARRAY_H
#include <assert.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#if true
# undef malloc
# undef calloc
# undef realloc
# undef free
# define malloc(s) mem_alloc((s))
# define calloc(s, l) mem_alloc_array((s), (l))
# define realloc(p, t) mem_realloc((p), (t))
# define free(p) mem_free((p))
#endif
#define Array(T) \
struct \
{ \
T *contents; \
uint32_t size; \
uint32_t capacity; \
}
#ifndef inline
# define inline __inline__
#endif
/// Initialize an array.
#define array_init(self) ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
/// Create an empty array.
#define array_new() \
{ \
NULL, 0, 0 \
}
/// Get a pointer to the element at a given `index` in the array.
#define array_get(self, _index) (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
/// Get a pointer to the first element in the array.
#define array_front(self) array_get(self, 0)
/// Get a pointer to the last element in the array.
#define array_back(self) array_get(self, (self)->size - 1)
/// Clear the array, setting its size to zero. Note that this does not free any
/// memory allocated for the array's contents.
#define array_clear(self) ((self)->size = 0)
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
/// less than the array's current capacity, this function has no effect.
#define array_reserve(self, new_capacity) _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
/// Free any memory allocated for this array. Note that this does not free any
/// memory allocated for the array's contents.
#define array_delete(self) _array__delete((Array *)(self))
/// Push a new `element` onto the end of the array.
#define array_push(self, element) (_array__grow((Array *)(self), 1, array_elem_size(self)), (self)->contents[(self)->size++] = (element))
/// Increase the array's size by `count` elements.
/// New elements are zero-initialized.
#define array_grow_by(self, count) \
do \
{ \
if ((count) == 0) \
break; \
_array__grow((Array *)(self), count, array_elem_size(self)); \
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
(self)->size += (count); \
} while (0)
/// Append all elements from one array to the end of another.
#define array_push_all(self, other) array_extend((self), (other)->size, (other)->contents)
/// Append `count` elements to the end of the array, reading their values from
/// the `contents` pointer.
#define array_extend(self, count, contents) _array__splice((Array *)(self), array_elem_size(self), (self)->size, 0, count, contents)
/// Remove `old_count` elements from the array starting at the given `index`. At
/// the same index, insert `new_count` new elements, reading their values from
/// the `new_contents` pointer.
#define array_splice(self, _index, old_count, new_count, new_contents) \
_array__splice((Array *)(self), array_elem_size(self), _index, old_count, new_count, new_contents)
/// Insert one `element` into the array at the given `index`.
#define array_insert(self, _index, element) _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
/// Remove one element from the array at the given `index`.
#define array_erase(self, _index) _array__erase((Array *)(self), array_elem_size(self), _index)
/// Pop the last element off the array, returning the element by value.
#define array_pop(self) ((self)->contents[--(self)->size])
/// Assign the contents of one array to another, reallocating if necessary.
#define array_assign(self, other) _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
/// Swap one array with another
#define array_swap(self, other) _array__swap((Array *)(self), (Array *)(other))
/// Get the size of the array contents
#define array_elem_size(self) (sizeof *(self)->contents)
/// Search a sorted array for a given `needle` value, using the given `compare`
/// callback to determine the order.
///
/// If an existing element is found to be equal to `needle`, then the `index`
/// out-parameter is set to the existing value's index, and the `exists`
/// out-parameter is set to true. Otherwise, `index` is set to an index where
/// `needle` should be inserted in order to preserve the sorting, and `exists`
/// is set to false.
#define array_search_sorted_with(self, compare, needle, _index, _exists) _array__search_sorted(self, 0, compare, , needle, _index, _exists)
/// Helper macro for the `_sorted_by` routines below. This takes the left
/// (existing) parameter by reference in order to work with the generic sorting
/// function above.
#define _compare_int(a, b) ((int)*(a) - (int)(b))
/// Search a sorted array for a given `needle` value, using integer comparisons
/// of a given struct field (specified with a leading dot) to determine the
/// order.
///
/// See also `array_search_sorted_with`.
#define array_search_sorted_by(self, field, needle, _index, _exists) _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
/// Insert a given `value` into a sorted array, using the given `compare`
/// callback to determine the order.
#define array_insert_sorted_with(self, compare, value) \
do \
{ \
unsigned _index, _exists; \
array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
if (!_exists) \
array_insert(self, _index, value); \
} while (0)
/// Insert a given `value` into a sorted array, using integer comparisons of
/// a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_by`.
#define array_insert_sorted_by(self, field, value) \
do \
{ \
unsigned _index, _exists; \
array_search_sorted_by(self, field, (value)field, &_index, &_exists); \
if (!_exists) \
array_insert(self, _index, value); \
} while (0)
typedef Array(void) Array;
/// This is not what you're looking for, see `array_delete`.
static inline void _array__delete(Array *self)
{
if (self->contents)
{
free(self->contents);
self->contents = NULL;
self->size = 0;
self->capacity = 0;
}
}
/// This is not what you're looking for, see `array_erase`.
static inline void _array__erase(Array *self, size_t element_size, uint32_t index)
{
assert(index < self->size);
char *contents = (char *)self->contents;
memmove(contents + index * element_size, contents + (index + 1) * element_size, (self->size - index - 1) * element_size);
self->size--;
}
/// This is not what you're looking for, see `array_reserve`.
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity)
{
if (new_capacity > self->capacity)
{
if (self->contents)
{
self->contents = realloc(self->contents, new_capacity * element_size);
}
else
{
self->contents = malloc(new_capacity * element_size);
}
self->capacity = new_capacity;
}
}
/// This is not what you're looking for, see `array_assign`.
static inline void _array__assign(Array *self, const Array *other, size_t element_size)
{
_array__reserve(self, element_size, other->size);
self->size = other->size;
memcpy(self->contents, other->contents, self->size * element_size);
}
/// This is not what you're looking for, see `array_swap`.
static inline void _array__swap(Array *self, Array *other)
{
Array swap = *other;
*other = *self;
*self = swap;
}
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
static inline void _array__grow(Array *self, uint32_t count, size_t element_size)
{
uint32_t new_size = self->size + count;
if (new_size > self->capacity)
{
uint32_t new_capacity = self->capacity * 2;
if (new_capacity < 8)
new_capacity = 8;
if (new_capacity < new_size)
new_capacity = new_size;
_array__reserve(self, element_size, new_capacity);
}
}
/// This is not what you're looking for, see `array_splice`.
static inline void _array__splice(Array *self, size_t element_size, uint32_t index, uint32_t old_count, uint32_t new_count, const void *elements)
{
uint32_t new_size = self->size + new_count - old_count;
uint32_t old_end = index + old_count;
uint32_t new_end = index + new_count;
assert(old_end <= self->size);
_array__reserve(self, element_size, new_size);
char *contents = (char *)self->contents;
if (self->size > old_end)
{
memmove(contents + new_end * element_size, contents + old_end * element_size, (self->size - old_end) * element_size);
}
if (new_count > 0)
{
if (elements)
{
memcpy((contents + index * element_size), elements, new_count * element_size);
}
else
{
memset((contents + index * element_size), 0, new_count * element_size);
}
}
self->size += new_count - old_count;
}
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
/// This is not what you're looking for, see `array_search_sorted_with` or
/// `array_search_sorted_by`.
#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
do \
{ \
*(_index) = start; \
*(_exists) = false; \
uint32_t size = (self)->size - *(_index); \
if (size == 0) \
break; \
int comparison; \
while (size > 1) \
{ \
uint32_t half_size = size / 2; \
uint32_t mid_index = *(_index) + half_size; \
comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
if (comparison <= 0) \
*(_index) = mid_index; \
size -= half_size; \
} \
comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
if (comparison == 0) \
*(_exists) = true; \
else if (comparison < 0) \
*(_index) += 1; \
} while (0)
#endif // ARRAY_H

10456
parser/src/combined.c
View file

File diff suppressed because it is too large Load diff

272
parser/src/combined.h
View file

@ -1,272 +0,0 @@
/* ************************************************************************** */
/* */
/* ::: :::::::: */
/* combined.h :+: :+: :+: */
/* +:+ +:+ +:+ */
/* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */
/* +#+#+#+#+#+ +#+ */
/* Created: 2024/06/18 17:01:23 by maiboyer #+# #+# */
/* Updated: 2024/06/18 17:12:45 by maiboyer ### ########.fr */
/* */
/* ************************************************************************** */
#ifndef COMBINED_H
#define COMBINED_H
#include "./api.h"
#include "./structs.h"
t_u32 ascii_decode(const t_u8 *chunk, t_u32 size, t_i32 *codepoint);
int _ts_dup(int file_descriptor);
t_external_scanner_state ts_external_scanner_state_copy(const t_external_scanner_state *self);
const char *ts_external_scanner_state_data(const t_external_scanner_state *self);
void ts_external_scanner_state_delete(t_external_scanner_state *self);
bool ts_external_scanner_state_eq(const t_external_scanner_state *self, const char *buffer, unsigned length);
void ts_external_scanner_state_init(t_external_scanner_state *self, const char *data, unsigned length);
const t_language *ts_language_copy(const t_language *self);
void ts_language_delete(const t_language *self);
t_u32 ts_language_field_count(const t_language *self);
t_field_id ts_language_field_id_for_name(const t_language *self, const char *name, t_u32 name_length);
const char *ts_language_field_name_for_id(const t_language *self, t_field_id id);
t_state_id ts_language_next_state(const t_language *self, t_state_id state, t_symbol symbol);
t_symbol ts_language_public_symbol(const t_language *self, t_symbol symbol);
t_u32 ts_language_state_count(const t_language *self);
t_u32 ts_language_symbol_count(const t_language *self);
t_symbol ts_language_symbol_for_name(const t_language *self, const char *string, t_u32 length, bool is_named);
t_symbol_metadata ts_language_symbol_metadata(const t_language *self, t_symbol symbol);
const char *ts_language_symbol_name(const t_language *self, t_symbol symbol);
t_symbol_type ts_language_symbol_type(const t_language *self, t_symbol symbol);
void ts_language_table_entry(const t_language *self, t_state_id state, t_symbol symbol, t_table_entry *result);
t_u32 ts_language_version(const t_language *self);
void ts_lexer_advance_to_end(t_lexer *self);
void ts_lexer_delete(t_lexer *self);
void ts_lexer_finish(t_lexer *self, t_u32 *lookahead_end_byte);
t_parse_range *ts_lexer_included_ranges(const t_lexer *self, t_u32 *count);
void ts_lexer_init(t_lexer *self);
void ts_lexer_mark_end(t_lexer *self);
void ts_lexer_reset(t_lexer *self, t_length position);
bool ts_lexer_set_included_ranges(t_lexer *self, const t_parse_range *ranges, t_u32 count);
void ts_lexer_set_input(t_lexer *self, t_parse_input input);
void ts_lexer_start(t_lexer *self);
// START PROBABLY DELETE WORTHY
t_symbol ts_lookahead_iterator_current_symbol(const t_lookahead_iterator *self);
const char *ts_lookahead_iterator_current_symbol_name(const t_lookahead_iterator *self);
void ts_lookahead_iterator_delete(t_lookahead_iterator *self);
const t_language *ts_lookahead_iterator_language(const t_lookahead_iterator *self);
t_lookahead_iterator *ts_lookahead_iterator_new(const t_language *self, t_state_id state);
bool ts_lookahead_iterator_next(t_lookahead_iterator *self);
bool ts_lookahead_iterator_reset(t_lookahead_iterator *self, const t_language *language, t_state_id state);
bool ts_lookahead_iterator_reset_state(t_lookahead_iterator *self, t_state_id state);
// END PROBABLY DELETE WORTHY
t_parse_node ts_node_child(t_parse_node self, t_u32 child_index);
t_parse_node ts_node_child_by_field_id(t_parse_node self, t_field_id field_id);
t_parse_node ts_node_child_by_field_name(t_parse_node self, const char *name, t_u32 name_length);
t_parse_node ts_node_child_containing_descendant(t_parse_node self, t_parse_node subnode);
t_u32 ts_node_child_count(t_parse_node self);
t_u32 ts_node_descendant_count(t_parse_node self);
t_parse_node ts_node_descendant_for_byte_range(t_parse_node self, t_u32 start, t_u32 end);
t_parse_node ts_node_descendant_for_point_range(t_parse_node self, t_point start, t_point end);
void ts_node_edit(t_parse_node *self, const t_input_edit *edit);
t_u32 ts_node_end_byte(t_parse_node self);
t_point ts_node_end_point(t_parse_node self);
bool ts_node_eq(t_parse_node self, t_parse_node other);
t_field_id ts_node_field_id_for_child(t_parse_node self, t_u32 child_index);
const char *ts_node_field_name_for_child(t_parse_node self, t_u32 child_index);
t_parse_node ts_node_first_child_for_byte(t_parse_node self, t_u32 byte);
t_parse_node ts_node_first_named_child_for_byte(t_parse_node self, t_u32 byte);
t_symbol ts_node_grammar_symbol(t_parse_node self);
const char *ts_node_grammar_type(t_parse_node self);
bool ts_node_has_changes(t_parse_node self);
bool ts_node_has_error(t_parse_node self);
bool ts_node_is_error(t_parse_node self);
bool ts_node_is_extra(t_parse_node self);
bool ts_node_is_missing(t_parse_node self);
bool ts_node_is_named(t_parse_node self);
bool ts_node_is_null(t_parse_node self);
const t_language *ts_node_language(t_parse_node self);
t_parse_node ts_node_named_child(t_parse_node self, t_u32 child_index);
t_u32 ts_node_named_child_count(t_parse_node self);
t_parse_node ts_node_named_descendant_for_byte_range(t_parse_node self, t_u32 start, t_u32 end);
t_parse_node ts_node_named_descendant_for_point_range(t_parse_node self, t_point start, t_point end);
t_parse_node ts_node_new(const t_first_tree *tree, const t_subtree *subtree, t_length position, t_symbol alias);
t_parse_node ts_node_next_named_sibling(t_parse_node self);
t_state_id ts_node_next_parse_state(t_parse_node self);
t_parse_node ts_node_next_sibling(t_parse_node self);
t_parse_node ts_node_parent(t_parse_node self);
t_state_id ts_node_parse_state(t_parse_node self);
t_parse_node ts_node_prev_named_sibling(t_parse_node self);
t_parse_node ts_node_prev_sibling(t_parse_node self);
t_u32 ts_node_start_byte(t_parse_node self);
t_point ts_node_start_point(t_parse_node self);
char *ts_node_string(t_parse_node self);
t_symbol ts_node_symbol(t_parse_node self);
const char *ts_node_type(t_parse_node self);
const size_t *ts_parser_cancellation_flag(const t_first_parser *self);
void ts_parser_delete(t_first_parser *self);
const t_parse_range *ts_parser_included_ranges(const t_first_parser *self, t_u32 *count);
const t_language *ts_parser_language(const t_first_parser *self);
t_parse_logger ts_parser_logger(const t_first_parser *self);
t_first_parser *ts_parser_new(void);
t_first_tree *ts_parser_parse(t_first_parser *self, const t_first_tree *old_tree, t_parse_input input);
t_first_tree *ts_parser_parse_string(t_first_parser *self, const t_first_tree *old_tree, const char *string, t_u32 length);
t_first_tree *ts_parser_parse_string_encoding(t_first_parser *self, const t_first_tree *old_tree, const char *string, t_u32 length, t_input_encoding encoding);
void ts_parser_print_dot_graphs(t_first_parser *self, int fd);
void ts_parser_reset(t_first_parser *self);
void ts_parser_set_cancellation_flag(t_first_parser *self, const size_t *flag);
bool ts_parser_set_included_ranges(t_first_parser *self, const t_parse_range *ranges, t_u32 count);
bool ts_parser_set_language(t_first_parser *self, const t_language *language);
void ts_parser_set_logger(t_first_parser *self, t_parse_logger logger);
void ts_parser_set_timeout_micros(t_first_parser *self, t_u64 timeout_micros);
t_u64 ts_parser_timeout_micros(const t_first_parser *self);
// START PROBABLY DELETE WORTHY
bool ts_query__step_is_fallible(const t_parse_query *self, t_u16 step_index);
t_u32 ts_query_capture_count(const t_parse_query *self);
const char *ts_query_capture_name_for_id(const t_parse_query *self, t_u32 index, t_u32 *length);
t_quantifier ts_query_capture_quantifier_for_id(const t_parse_query *self, t_u32 pattern_index, t_u32 capture_index);
void ts_query_cursor__compare_captures(t_query_cursor *self, t_query_state *left_state, t_query_state *right_state, bool *left_contains_right, bool *right_contains_left);
int ts_query_cursor__compare_nodes(t_parse_node left, t_parse_node right);
void ts_query_cursor_delete(t_query_cursor *self);
bool ts_query_cursor_did_exceed_match_limit(const t_query_cursor *self);
void ts_query_cursor_exec(t_query_cursor *self, const t_parse_query *query, t_parse_node node);
t_u32 ts_query_cursor_match_limit(const t_query_cursor *self);
t_query_cursor *ts_query_cursor_new(void);
bool ts_query_cursor_next_capture(t_query_cursor *self, t_query_match *match, t_u32 *capture_index);
bool ts_query_cursor_next_match(t_query_cursor *self, t_query_match *match);
void ts_query_cursor_remove_match(t_query_cursor *self, t_u32 match_id);
void ts_query_cursor_set_byte_range(t_query_cursor *self, t_u32 start_byte, t_u32 end_byte);
void ts_query_cursor_set_match_limit(t_query_cursor *self, t_u32 limit);
void ts_query_cursor_set_max_start_depth(t_query_cursor *self, t_u32 max_start_depth);
void ts_query_cursor_set_point_range(t_query_cursor *self, t_point start_point, t_point end_point);
void ts_query_delete(t_parse_query *self);
void ts_query_disable_capture(t_parse_query *self, const char *name, t_u32 length);
void ts_query_disable_pattern(t_parse_query *self, t_u32 pattern_index);
bool ts_query_is_pattern_guaranteed_at_step(const t_parse_query *self, t_u32 byte_offset);
bool ts_query_is_pattern_non_local(const t_parse_query *self, t_u32 pattern_index);
bool ts_query_is_pattern_rooted(const t_parse_query *self, t_u32 pattern_index);
t_parse_query *ts_query_new(const t_language *language, const char *source, t_u32 source_len, t_u32 *error_offset, t_query_error *error_type);
t_u32 ts_query_pattern_count(const t_parse_query *self);
const t_query_predicate_step *ts_query_predicates_for_pattern(const t_parse_query *self, t_u32 pattern_index, t_u32 *step_count);
t_u32 ts_query_start_byte_for_pattern(const t_parse_query *self, t_u32 pattern_index);
t_u32 ts_query_string_count(const t_parse_query *self);
const char *ts_query_string_value_for_id(const t_parse_query *self, t_u32 index, t_u32 *length);
// END PROBABLY DELETE WORTHY
void ts_range_array_get_changed_ranges(const t_parse_range *old_ranges, unsigned old_range_count, const t_parse_range *new_ranges, unsigned new_range_count, t_range_array *differences);
bool ts_range_array_intersects(const t_range_array *self, unsigned start_index, t_u32 start_byte, t_u32 end_byte);
bool ts_stack_can_merge(t_stack *self, t_stack_version version1, t_stack_version version2);
void ts_stack_clear(t_stack *self);
t_stack_version ts_stack_copy_version(t_stack *self, t_stack_version version);
void ts_stack_delete(t_stack *self);
int ts_stack_dynamic_precedence(t_stack *self, t_stack_version version);
unsigned ts_stack_error_cost(const t_stack *self, t_stack_version version);
t_stack_summary *ts_stack_get_summary(t_stack *self, t_stack_version version);
void ts_stack_halt(t_stack *self, t_stack_version version);
bool ts_stack_has_advanced_since_error(const t_stack *self, t_stack_version version);
bool ts_stack_is_active(const t_stack *self, t_stack_version version);
bool ts_stack_is_halted(const t_stack *self, t_stack_version version);
bool ts_stack_is_paused(const t_stack *self, t_stack_version version);
t_subtree ts_stack_last_external_token(const t_stack *self, t_stack_version version);
bool ts_stack_merge(t_stack *self, t_stack_version version1, t_stack_version version2);
t_stack *ts_stack_new(t_subtree_pool *subtree_pool);
unsigned ts_stack_node_count_since_error(const t_stack *self, t_stack_version version);
void ts_stack_pause(t_stack *self, t_stack_version version, t_subtree lookahead);
t_stack_slice_array ts_stack_pop_all(t_stack *self, t_stack_version version);
t_stack_slice_array ts_stack_pop_count(t_stack *self, t_stack_version version, t_u32 count);
t_subtree_array ts_stack_pop_error(t_stack *self, t_stack_version version);
t_stack_slice_array ts_stack_pop_pending(t_stack *self, t_stack_version version);
t_length ts_stack_position(const t_stack *self, t_stack_version version);
bool ts_stack_print_dot_graph(t_stack *self, const t_language *language, void *f);
void ts_stack_push(t_stack *self, t_stack_version version, t_subtree subtree, bool pending, t_state_id state);
void ts_stack_record_summary(t_stack *self, t_stack_version version, unsigned max_depth);
void ts_stack_remove_version(t_stack *self, t_stack_version version);
void ts_stack_renumber_version(t_stack *self, t_stack_version v1, t_stack_version v2);
t_subtree ts_stack_resume(t_stack *self, t_stack_version version);
void ts_stack_set_last_external_token(t_stack *self, t_stack_version version, t_subtree token);
t_state_id ts_stack_state(const t_stack *self, t_stack_version version);
void ts_stack_swap_versions(t_stack *self, t_stack_version v1, t_stack_version v2);
t_u32 ts_stack_version_count(const t_stack *self);
void ts_subtree__print_dot_graph(const t_subtree *self, t_u32 start_offset, const t_language *language, t_symbol alias_symbol, void *f);
void ts_subtree_array_clear(t_subtree_pool *pool, t_subtree_array *self);
void ts_subtree_array_copy(t_subtree_array self, t_subtree_array *dest);
void ts_subtree_array_delete(t_subtree_pool *pool, t_subtree_array *self);
void ts_subtree_array_remove_trailing_extras(t_subtree_array *self, t_subtree_array *destination);
void ts_subtree_array_reverse(t_subtree_array *self);
void ts_subtree_balance(t_subtree self, t_subtree_pool *pool, const t_language *language);
t_mutable_subtree ts_subtree_clone(t_subtree self);
int ts_subtree_compare(t_subtree left, t_subtree right, t_subtree_pool *pool);
t_subtree ts_subtree_edit(t_subtree self, const t_input_edit *input_edit, t_subtree_pool *pool);
const t_external_scanner_state *ts_subtree_external_scanner_state(t_subtree self);
bool ts_subtree_external_scanner_state_eq(t_subtree self, t_subtree other);
unsigned ts_subtree_get_changed_ranges(const t_subtree *old_tree, const t_subtree *new_tree, t_tree_cursor *cursor1, t_tree_cursor *cursor2, const t_language *language, const t_range_array *included_range_differences, t_parse_range **ranges);
t_subtree ts_subtree_last_external_token(t_subtree tree);
t_mutable_subtree ts_subtree_make_mut(t_subtree_pool *pool, t_subtree self);
t_subtree ts_subtree_new_error(t_subtree_pool *pool, t_i32 lookahead_char, t_length padding, t_length size, t_u32 bytes_scanned, t_state_id parse_state, const t_language *language);
t_subtree ts_subtree_new_error_node(t_subtree_array *children, bool extra, const t_language *language);
t_subtree ts_subtree_new_leaf(t_subtree_pool *pool, t_symbol symbol, t_length padding, t_length size, t_u32 lookahead_bytes, t_state_id parse_state, bool has_external_tokens, bool depends_on_column, bool is_keyword, const t_language *language);
t_subtree ts_subtree_new_missing_leaf(t_subtree_pool *pool, t_symbol symbol, t_length padding, t_u32 lookahead_bytes, const t_language *language);
t_mutable_subtree ts_subtree_new_node(t_symbol symbol, t_subtree_array *children, unsigned production_id, const t_language *language);
void ts_subtree_pool_delete(t_subtree_pool *self);
t_subtree_pool ts_subtree_pool_new(t_u32 capacity);
void ts_subtree_release(t_subtree_pool *pool, t_subtree self);
void ts_subtree_retain(t_subtree self);
void ts_subtree_set_symbol(t_mutable_subtree *self, t_symbol symbol, const t_language *language);
char *ts_subtree_string(t_subtree self, t_symbol alias_symbol, bool alias_is_named, const t_language *language, bool include_all);
void ts_subtree_summarize_children(t_mutable_subtree self, const t_language *language);
t_first_tree *ts_tree_copy(const t_first_tree *self);
// START PROBABLY DELETE WORTHY
t_tree_cursor ts_tree_cursor_copy(const t_tree_cursor *_cursor);
t_u32 ts_tree_cursor_current_depth(const t_tree_cursor *_self);
t_u32 ts_tree_cursor_current_descendant_index(const t_tree_cursor *_self);
t_field_id ts_tree_cursor_current_field_id(const t_tree_cursor *_self);
const char *ts_tree_cursor_current_field_name(const t_tree_cursor *_self);
t_parse_node ts_tree_cursor_current_node(const t_tree_cursor *_self);
void ts_tree_cursor_current_status(const t_tree_cursor *_self, t_field_id *field_id, bool *has_later_siblings, bool *has_later_named_siblings, bool *can_have_later_siblings_with_this_field, t_symbol *supertypes, unsigned *supertype_count);
void ts_tree_cursor_delete(t_tree_cursor *_self);
void ts_tree_cursor_goto_descendant(t_tree_cursor *_self, t_u32 goal_descendant_index);
bool ts_tree_cursor_goto_first_child(t_tree_cursor *self);
t_i64 ts_tree_cursor_goto_first_child_for_byte(t_tree_cursor *self, t_u32 goal_byte);
t_i64 ts_tree_cursor_goto_first_child_for_point(t_tree_cursor *self, t_point goal_point);
t_tree_cursor_step ts_tree_cursor_goto_first_child_internal(t_tree_cursor *_self);
bool ts_tree_cursor_goto_last_child(t_tree_cursor *self);
t_tree_cursor_step ts_tree_cursor_goto_last_child_internal(t_tree_cursor *_self);
bool ts_tree_cursor_goto_next_sibling(t_tree_cursor *self);
t_tree_cursor_step ts_tree_cursor_goto_next_sibling_internal(t_tree_cursor *_self);
bool ts_tree_cursor_goto_parent(t_tree_cursor *_self);
bool ts_tree_cursor_goto_previous_sibling(t_tree_cursor *self);
t_tree_cursor_step ts_tree_cursor_goto_previous_sibling_internal(t_tree_cursor *_self);
t_tree_cursor_step ts_tree_cursor_goto_sibling_internal(t_tree_cursor *_self, bool (*advance)(t_cursor_child_iterator *, t_tree_cursor_entry *, bool *));
void ts_tree_cursor_init(t_tree_cursor *self, t_parse_node node);
t_tree_cursor ts_tree_cursor_new(t_parse_node node);
t_parse_node ts_tree_cursor_parent_node(const t_tree_cursor *_self);
void ts_tree_cursor_reset(t_tree_cursor *_self, t_parse_node node);
void ts_tree_cursor_reset_to(t_tree_cursor *_dst, const t_tree_cursor *_src);
// END PROBABLY DELETE WORTHY
void ts_tree_delete(t_first_tree *self);
void ts_tree_edit(t_first_tree *self, const t_input_edit *edit);
t_parse_range *ts_tree_get_changed_ranges(const t_first_tree *old_tree, const t_first_tree *new_tree, t_u32 *length);
t_parse_range *ts_tree_included_ranges(const t_first_tree *self, t_u32 *length);
const t_language *ts_tree_language(const t_first_tree *self);
t_first_tree *ts_tree_new(t_subtree root, const t_language *language, const t_parse_range *included_ranges, unsigned included_range_count);
void ts_tree_print_dot_graph(const t_first_tree *self, int file_descriptor);
t_parse_node ts_tree_root_node(const t_first_tree *self);
t_parse_node ts_tree_root_node_with_offset(const t_first_tree *self, t_u32 offset_bytes, t_point offset_extent);
#endif /* COMBINED_H */

98
parser/src/create_language.c
View file

@ -1,98 +0,0 @@
/* ************************************************************************** */
/* */
/* ::: :::::::: */
/* create_language.c :+: :+: :+: */
/* +:+ +:+ +:+ */
/* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */
/* +#+#+#+#+#+ +#+ */
/* Created: 2024/04/25 16:13:52 by maiboyer #+# #+# */
/* Updated: 2024/06/09 21:46:14 by maiboyer ### ########.fr */
/* */
/* ************************************************************************** */
#include "../parse_types.h"
#include "../static/headers/constants.h"
#include "../static/headers/symbols.h"
bool lex_keywords_main(t_lexer *lexer, t_state_id state);
bool lex_normal_main(t_lexer *lexer, t_state_id state);
bool tree_sitter_sh_external_scanner_scan(void *ctx, t_lexer *lexer, const bool *ret);
const bool *create_external_scanner_states(void);
const char *const *create_field_names(void);
const char *const *create_symbols_names(void);
const t_field_map_entry *create_field_map_entries(void);
const t_field_map_slice *create_field_map_slices(void);
const t_lex_modes *create_lex_modes(void);
const t_parse_action_entry *create_parse_actions_entries(void);
const t_state_id *create_primary_state_ids(void);
const t_symbol *create_alias_sequences(void);
const t_symbol *create_external_scanner_symbol_map(void);
const t_symbol *create_non_terminal_alias_map(void);
const t_symbol *create_unique_symbols_map(void);
const t_symbol_metadata *create_symbols_metadata(void);
const uint16_t *create_parse_table(void);
const uint16_t *create_small_parse_table(void);
const uint32_t *create_small_parse_table_map(void);
uint32_t tree_sitter_sh_external_scanner_serialize(void *ctx, char *s);
void tree_sitter_sh_external_scanner_deserialize(void *ctx, const char *s, uint32_t val);
void tree_sitter_sh_external_scanner_destroy(void *ctx);
void *tree_sitter_sh_external_scanner_create(void);
static t_scanner init_scanner(void)
{
return ((t_scanner){
create_external_scanner_states(),
create_external_scanner_symbol_map(),
tree_sitter_sh_external_scanner_create,
tree_sitter_sh_external_scanner_destroy,
tree_sitter_sh_external_scanner_scan,
tree_sitter_sh_external_scanner_serialize,
tree_sitter_sh_external_scanner_deserialize,
});
}
static void init_language(t_language *language)
{
language->parse_table = create_parse_table();
language->small_parse_table = create_small_parse_table();
language->small_parse_table_map = create_small_parse_table_map();
language->parse_actions = create_parse_actions_entries();
language->symbol_names = create_symbols_names();
language->field_names = create_field_names();
language->field_map_slices = create_field_map_slices();
language->field_map_entries = create_field_map_entries();
language->symbol_metadata = create_symbols_metadata();
language->public_symbol_map = create_unique_symbols_map();
language->alias_map = create_non_terminal_alias_map();
language->alias_sequences = create_alias_sequences();
language->lex_modes = create_lex_modes();
language->primary_state_ids = create_primary_state_ids();
language->lex_fn = lex_normal_main;
language->keyword_lex_fn = lex_keywords_main;
language->keyword_capture_token = sym_word;
language->external_scanner = init_scanner();
}
const t_language *tree_sitter_bash(void)
{
static bool init = false;
static t_language language = {
.version = LANGUAGE_VERSION,
.symbol_count = SYMBOL_COUNT,
.alias_count = ALIAS_COUNT,
.token_count = TOKEN_COUNT,
.external_token_count = EXTERNAL_TOKEN_COUNT,
.state_count = STATE_COUNT,
.large_state_count = LARGE_STATE_COUNT,
.production_id_count = PRODUCTION_ID_COUNT,
.field_count = FIELD_COUNT,
.max_alias_sequence_length = MAX_ALIAS_SEQUENCE_LENGTH,
};
if (!init)
{
init_language(&language);
init = true;
}
return ((t_language *)&language);
}

711
parser/src/funcs.c
View file

@ -1,711 +0,0 @@
/* ************************************************************************** */
/* */
/* ::: :::::::: */
/* funcs.c :+: :+: :+: */
/* +:+ +:+ +:+ */
/* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */
/* +#+#+#+#+#+ +#+ */
/* Created: 2024/05/30 19:21:01 by maiboyer #+# #+# */
/* Updated: 2024/05/30 19:22:53 by maiboyer ### ########.fr */
/* */
/* ************************************************************************** */
#include "./api.h"
size_t atomic_load(const volatile size_t *p)
{
return (*p);
}
uint32_t atomic_inc(volatile uint32_t *p)
{
return (++(*p));
}
uint32_t atomic_dec(volatile uint32_t *p)
{
return (--(*p));
}
bool ts_language_is_symbol_external(const t_language *self, t_symbol symbol)
{
return 0 < symbol && symbol < self->external_token_count + 1;
}
const t_parse_action *ts_language_actions(const t_language *self, t_state_id state, t_symbol symbol, uint32_t *count)
{
t_table_entry entry;
ts_language_table_entry(self, state, symbol, &entry);
*count = entry.action_count;
return entry.actions;
}
bool ts_language_has_reduce_action(const t_language *self, t_state_id state, t_symbol symbol)
{
t_table_entry entry;
ts_language_table_entry(self, state, symbol, &entry);
return entry.action_count > 0 && entry.actions[0].type == TSParseActionTypeReduce;
}
uint16_t ts_language_lookup(const t_language *self, t_state_id state, t_symbol symbol)
{
if (state >= self->large_state_count)
{
uint32_t index = self->small_parse_table_map[state - self->large_state_count];
const uint16_t *data = &self->small_parse_table[index];
uint16_t group_count = *(data++);
for (unsigned i = 0; i < group_count; i++)
{
uint16_t section_value = *(data++);
uint16_t symbol_count = *(data++);
for (unsigned j = 0; j < symbol_count; j++)
{
if (*(data++) == symbol)
return section_value;
}
}
return 0;
}
else
{
return self->parse_table[state * self->symbol_count + symbol];
}
}
bool ts_language_has_actions(const t_language *self, t_state_id state, t_symbol symbol)
{
return ts_language_lookup(self, state, symbol) != 0;
}
t_lookahead_iterator ts_language_lookaheads(const t_language *self, t_state_id state)
{
bool is_small_state = state >= self->large_state_count;
const uint16_t *data;
const uint16_t *group_end = NULL;
uint16_t group_count = 0;
if (is_small_state)
{
uint32_t index = self->small_parse_table_map[state - self->large_state_count];
data = &self->small_parse_table[index];
group_end = data + 1;
group_count = *data;
}
else
{
data = &self->parse_table[state * self->symbol_count] - 1;
}
return (t_lookahead_iterator){
.language = self,
.data = data,
.group_end = group_end,
.group_count = group_count,
.is_small_state = is_small_state,
.symbol = UINT16_MAX,
.next_state = 0,
};
}
bool ts_lookahead_iterator__next(t_lookahead_iterator *self)
{
// For small parse states, valid symbols are listed explicitly,
// grouped by their value. There's no need to look up the actions
// again until moving to the next group.
if (self->is_small_state)
{
self->data++;
if (self->data == self->group_end)
{
if (self->group_count == 0)
return false;
self->group_count--;
self->table_value = *(self->data++);
unsigned symbol_count = *(self->data++);
self->group_end = self->data + symbol_count;
self->symbol = *self->data;
}
else
{
self->symbol = *self->data;
return true;
}
}
// For large parse states, iterate through every symbol until one
// is found that has valid actions.
else
{
do
{
self->data++;
self->symbol++;
if (self->symbol >= self->language->symbol_count)
return false;
self->table_value = *self->data;
} while (!self->table_value);
}
// Depending on if the symbols is terminal or non-terminal, the table value
// either represents a list of actions or a successor state.
if (self->symbol < self->language->token_count)
{
const t_parse_action_entry *entry = &self->language->parse_actions[self->table_value];
self->action_count = entry->entry.count;
self->actions = (const t_parse_action *)(entry + 1);
self->next_state = 0;
}
else
{
self->action_count = 0;
self->next_state = self->table_value;
}
return true;
}
bool ts_language_state_is_primary(const t_language *self, t_state_id state)
{
if (self->version >= LANGUAGE_VERSION_WITH_PRIMARY_STATES)
{
return state == self->primary_state_ids[state];
}
else
{
return true;
}
}
const bool *ts_language_enabled_external_tokens(const t_language *self, unsigned external_scanner_state)
{
if (external_scanner_state == 0)
{
return NULL;
}
else
{
return self->external_scanner.states + self->external_token_count * external_scanner_state;
}
}
const t_symbol *ts_language_alias_sequence(const t_language *self, uint32_t production_id)
{
return production_id ? &self->alias_sequences[production_id * self->max_alias_sequence_length] : NULL;
}
t_symbol ts_language_alias_at(const t_language *self, uint32_t production_id, uint32_t child_index)
{
return production_id ? self->alias_sequences[production_id * self->max_alias_sequence_length + child_index] : 0;
}
void ts_language_field_map(const t_language *self, uint32_t production_id, const t_field_map_entry **start, const t_field_map_entry **end)
{
if (self->field_count == 0)
{
*start = NULL;
*end = NULL;
return;
}
t_field_map_slice slice = self->field_map_slices[production_id];
*start = &self->field_map_entries[slice.index];
*end = &self->field_map_entries[slice.index] + slice.length;
}
void ts_language_aliases_for_symbol(const t_language *self, t_symbol original_symbol, const t_symbol **start, const t_symbol **end)
{
*start = &self->public_symbol_map[original_symbol];
*end = *start + 1;
unsigned idx = 0;
for (;;)
{
t_symbol symbol = self->alias_map[idx++];
if (symbol == 0 || symbol > original_symbol)
break;
uint16_t count = self->alias_map[idx++];
if (symbol == original_symbol)
{
*start = &self->alias_map[idx];
*end = &self->alias_map[idx + count];
break;
}
idx += count;
}
}
bool length_is_undefined(t_length length)
{
return length.bytes == 0 && length.extent.column != 0;
}
t_length length_min(t_length len1, t_length len2)
{
return (len1.bytes < len2.bytes) ? len1 : len2;
}
t_length length_add(t_length len1, t_length len2)
{
t_length result;
result.bytes = len1.bytes + len2.bytes;
result.extent = point_add(len1.extent, len2.extent);
return result;
}
t_length length_sub(t_length len1, t_length len2)
{
t_length result;
result.bytes = len1.bytes - len2.bytes;
result.extent = point_sub(len1.extent, len2.extent);
return result;
}
t_length length_zero(void)
{
t_length result = {0, {0, 0}};
return result;
}
t_length length_saturating_sub(t_length len1, t_length len2)
{
if (len1.bytes > len2.bytes)
{
return length_sub(len1, len2);
}
else
{
return length_zero();
}
}
bool set_contains(t_char_range *ranges, uint32_t len, int32_t lookahead)
{
uint32_t index = 0;
uint32_t size = len - index;
while (size > 1)
{
uint32_t half_size = size / 2;
uint32_t mid_index = index + half_size;
t_char_range *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end)
{
return true;
}
else if (lookahead > range->end)
{
index = mid_index;
}
size -= half_size;
}
t_char_range *range = &ranges[index];
return (lookahead >= range->start && lookahead <= range->end);
}
t_point point__new(unsigned row, unsigned column)
{
t_point result = {row, column};
return result;
}
t_point point_add(t_point a, t_point b)
{
if (b.row > 0)
return point__new(a.row + b.row, b.column);
else
return point__new(a.row, a.column + b.column);
}
t_point point_sub(t_point a, t_point b)
{
if (a.row > b.row)
return point__new(a.row - b.row, a.column);
else
return point__new(0, a.column - b.column);
}
bool point_lte(t_point a, t_point b)
{
return (a.row < b.row) || (a.row == b.row && a.column <= b.column);
}
bool point_lt(t_point a, t_point b)
{
return (a.row < b.row) || (a.row == b.row && a.column < b.column);
}
bool point_gt(t_point a, t_point b)
{
return (a.row > b.row) || (a.row == b.row && a.column > b.column);
}
bool point_gte(t_point a, t_point b)
{
return (a.row > b.row) || (a.row == b.row && a.column >= b.column);
}
bool point_eq(t_point a, t_point b)
{
return a.row == b.row && a.column == b.column;
}
t_point point_min(t_point a, t_point b)
{
if (a.row < b.row || (a.row == b.row && a.column < b.column))
return a;
else
return b;
}
t_point point_max(t_point a, t_point b)
{
if (a.row > b.row || (a.row == b.row && a.column > b.column))
return a;
else
return b;
}
void ts_reduce_action_set_add(t_reduce_action_set *self, t_reduce_action new_action)
{
for (uint32_t i = 0; i < self->size; i++)
{
t_reduce_action action = self->contents[i];
if (action.symbol == new_action.symbol && action.count == new_action.count)
return;
}
array_push(self, new_action);
}
t_reusable_node reusable_node_new(void)
{
return (t_reusable_node){array_new(), NULL_SUBTREE};
}
void reusable_node_clear(t_reusable_node *self)
{
array_clear(&self->stack);
self->last_external_token = NULL_SUBTREE;
}
t_subtree reusable_node_tree(t_reusable_node *self)
{
return self->stack.size > 0 ? self->stack.contents[self->stack.size - 1].tree : NULL_SUBTREE;
}
uint32_t reusable_node_byte_offset(t_reusable_node *self)
{
return self->stack.size > 0 ? self->stack.contents[self->stack.size - 1].byte_offset : UINT32_MAX;
}
void reusable_node_delete(t_reusable_node *self)
{
array_delete(&self->stack);
}
void reusable_node_advance(t_reusable_node *self)
{
t_stack_entry last_entry = *array_back(&self->stack);
uint32_t byte_offset = last_entry.byte_offset + ts_subtree_total_bytes(last_entry.tree);
if (ts_subtree_has_external_tokens(last_entry.tree))
{
self->last_external_token = ts_subtree_last_external_token(last_entry.tree);
}
t_subtree tree;
uint32_t next_index;
do
{
t_stack_entry popped_entry = array_pop(&self->stack);
next_index = popped_entry.child_index + 1;
if (self->stack.size == 0)
return;
tree = array_back(&self->stack)->tree;
} while (ts_subtree_child_count(tree) <= next_index);
array_push(&self->stack, ((t_stack_entry){
.tree = ts_subtree_children(tree)[next_index],
.child_index = next_index,
.byte_offset = byte_offset,
}));
}
bool reusable_node_descend(t_reusable_node *self)
{
t_stack_entry last_entry = *array_back(&self->stack);
if (ts_subtree_child_count(last_entry.tree) > 0)
{
array_push(&self->stack, ((t_stack_entry){
.tree = ts_subtree_children(last_entry.tree)[0],
.child_index = 0,
.byte_offset = last_entry.byte_offset,
}));
return true;
}
else
{
return false;
}
}
void reusable_node_advance_past_leaf(t_reusable_node *self)
{
while (reusable_node_descend(self))
{
}
reusable_node_advance(self);
}
void reusable_node_reset(t_reusable_node *self, t_subtree tree)
{
reusable_node_clear(self);
array_push(&self->stack, ((t_stack_entry){
.tree = tree,
.child_index = 0,
.byte_offset = 0,
}));
// Never reuse the root node, because it has a non-standard internal
// structure due to transformations that are applied when it is accepted:
// adding the EOF child and any extra children.
if (!reusable_node_descend(self))
{
reusable_node_clear(self);
}
}
#define SUBTREE_GET(self, name) ((self).data.is_inline ? (self).data.name : (self).ptr->name)
t_symbol ts_subtree_symbol(t_subtree self)
{
return SUBTREE_GET(self, symbol);
}
bool ts_subtree_visible(t_subtree self)
{
return SUBTREE_GET(self, visible);
}
bool ts_subtree_named(t_subtree self)
{
return SUBTREE_GET(self, named);
}
bool ts_subtree_extra(t_subtree self)
{
return SUBTREE_GET(self, extra);
}
bool ts_subtree_has_changes(t_subtree self)
{
return SUBTREE_GET(self, has_changes);
}
bool ts_subtree_missing(t_subtree self)
{
return SUBTREE_GET(self, is_missing);
}
bool ts_subtree_is_keyword(t_subtree self)
{
return SUBTREE_GET(self, is_keyword);
}
t_state_id ts_subtree_parse_state(t_subtree self)
{
return SUBTREE_GET(self, parse_state);
}
uint32_t ts_subtree_lookahead_bytes(t_subtree self)
{
return SUBTREE_GET(self, lookahead_bytes);
}
size_t ts_subtree_alloc_size(uint32_t child_count)
{
return child_count * sizeof(t_subtree) + sizeof(t_subtree_heap_data);
}
void ts_subtree_set_extra(t_mutable_subtree *self, bool is_extra)
{
if (self->data.is_inline)
{
self->data.extra = is_extra;
}
else
{
self->ptr->extra = is_extra;
}
}
t_symbol ts_subtree_leaf_symbol(t_subtree self)
{
if (self.data.is_inline)
return self.data.symbol;
if (self.ptr->child_count == 0)
return self.ptr->symbol;
return self.ptr->first_leaf.symbol;
}
t_state_id ts_subtree_leaf_parse_state(t_subtree self)
{
if (self.data.is_inline)
return self.data.parse_state;
if (self.ptr->child_count == 0)
return self.ptr->parse_state;
return self.ptr->first_leaf.parse_state;
}
t_length ts_subtree_padding(t_subtree self)
{
if (self.data.is_inline)
{
t_length result = {self.data.padding_bytes, {self.data.padding_rows, self.data.padding_columns}};
return result;
}
else
{
return self.ptr->padding;
}
}
t_length ts_subtree_size(t_subtree self)
{
if (self.data.is_inline)
{
t_length result = {self.data.size_bytes, {0, self.data.size_bytes}};
return result;
}
else
{
return self.ptr->size;
}
}
t_length ts_subtree_total_size(t_subtree self)
{
return length_add(ts_subtree_padding(self), ts_subtree_size(self));
}
uint32_t ts_subtree_total_bytes(t_subtree self)
{
return ts_subtree_total_size(self).bytes;
}
uint32_t ts_subtree_child_count(t_subtree self)
{
return self.data.is_inline ? 0 : self.ptr->child_count;
}
uint32_t ts_subtree_repeat_depth(t_subtree self)
{
return self.data.is_inline ? 0 : self.ptr->repeat_depth;
}
uint32_t ts_subtree_is_repetition(t_subtree self)
{
return self.data.is_inline ? 0 : !self.ptr->named && !self.ptr->visible && self.ptr->child_count != 0;
}
uint32_t ts_subtree_visible_descendant_count(t_subtree self)
{
return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->visible_descendant_count;
}
uint32_t ts_subtree_visible_child_count(t_subtree self)
{
if (ts_subtree_child_count(self) > 0)
{
return self.ptr->visible_child_count;
}
else
{
return 0;
}
}
uint32_t ts_subtree_error_cost(t_subtree self)
{
if (ts_subtree_missing(self))
{
return ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY;
}
else
{
return self.data.is_inline ? 0 : self.ptr->error_cost;
}
}
int32_t ts_subtree_dynamic_precedence(t_subtree self)
{
return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->dynamic_precedence;
}
uint16_t ts_subtree_production_id(t_subtree self)
{
if (ts_subtree_child_count(self) > 0)
{
return self.ptr->production_id;
}
else
{
return 0;
}
}
bool ts_subtree_fragile_left(t_subtree self)
{
return self.data.is_inline ? false : self.ptr->fragile_left;
}
bool ts_subtree_fragile_right(t_subtree self)
{
return self.data.is_inline ? false : self.ptr->fragile_right;
}
bool ts_subtree_has_external_tokens(t_subtree self)
{
return self.data.is_inline ? false : self.ptr->has_external_tokens;
}
bool ts_subtree_has_external_scanner_state_change(t_subtree self)
{
return self.data.is_inline ? false : self.ptr->has_external_scanner_state_change;
}
bool ts_subtree_depends_on_column(t_subtree self)
{
return self.data.is_inline ? false : self.ptr->depends_on_column;
}
bool ts_subtree_is_fragile(t_subtree self)
{
return self.data.is_inline ? false : (self.ptr->fragile_left || self.ptr->fragile_right);
}
bool ts_subtree_is_error(t_subtree self)
{
return ts_subtree_symbol(self) == ts_builtin_sym_error;
}
bool ts_subtree_is_eof(t_subtree self)
{
return ts_subtree_symbol(self) == ts_builtin_sym_end;
}
t_subtree ts_subtree_from_mut(t_mutable_subtree self)
{
t_subtree result;
result.data = self.data;
return result;
}
t_mutable_subtree ts_subtree_to_mut_unsafe(t_subtree self)
{
t_mutable_subtree result;
result.data = self.data;
return result;
}
t_subtree ts_tree_cursor_current_subtree(const t_tree_cursor *_self)
{
const t_tree_cursor *self = (const t_tree_cursor *)_self;
t_tree_cursor_entry *last_entry = array_back(&self->stack);
return *last_entry->subtree;
}

173
parser/src/funcs.h
View file

@ -1,173 +0,0 @@
#ifndef FUNCS_H
#define FUNCS_H
#include "./api_structs.h"
bool length_is_undefined(t_length length);
bool point_eq(t_point a, t_point b);
bool point_gt(t_point a, t_point b);
bool point_gte(t_point a, t_point b);
bool point_lt(t_point a, t_point b);
bool point_lte(t_point a, t_point b);
bool reusable_node_descend(t_reusable_node *self);
bool set_contains(t_char_range *ranges, uint32_t len, int32_t lookahead);
bool ts_external_scanner_state_eq(const t_external_scanner_state *self, const char *, uint32_t);
bool ts_language_has_actions(const t_language *self, t_state_id state, t_symbol symbol);
bool ts_language_has_reduce_action(const t_language *self, t_state_id state, t_symbol symbol);
bool ts_language_is_symbol_external(const t_language *self, t_symbol symbol);
bool ts_language_state_is_primary(const t_language *self, t_state_id state);
bool ts_lexer_set_included_ranges(t_lexer *self, const t_parse_range *ranges, uint32_t count);
bool ts_lookahead_iterator__next(t_lookahead_iterator *self);
bool ts_range_array_intersects(const t_range_array *self, uint32_t start_index, uint32_t start_byte, uint32_t end_byte);
bool ts_stack_can_merge(t_stack *, t_stack_version, t_stack_version);
bool ts_stack_has_advanced_since_error(const t_stack *, t_stack_version);
bool ts_stack_is_active(const t_stack *, t_stack_version);
bool ts_stack_is_halted(const t_stack *, t_stack_version);
bool ts_stack_is_paused(const t_stack *, t_stack_version);
bool ts_stack_merge(t_stack *, t_stack_version, t_stack_version);
bool ts_subtree_depends_on_column(t_subtree self);
bool ts_subtree_external_scanner_state_eq(t_subtree, t_subtree);
bool ts_subtree_extra(t_subtree self);
bool ts_subtree_fragile_left(t_subtree self);
bool ts_subtree_fragile_right(t_subtree self);
bool ts_subtree_has_changes(t_subtree self);
bool ts_subtree_has_external_scanner_state_change(t_subtree self);
bool ts_subtree_has_external_tokens(t_subtree self);
bool ts_subtree_has_external_tokens(t_subtree self);
bool ts_subtree_is_eof(t_subtree self);
bool ts_subtree_is_error(t_subtree self);
bool ts_subtree_is_fragile(t_subtree self);
bool ts_subtree_is_keyword(t_subtree self);
bool ts_subtree_missing(t_subtree self);
bool ts_subtree_named(t_subtree self);
bool ts_subtree_visible(t_subtree self);
char *ts_subtree_string(t_subtree, t_symbol, bool, const t_language *, bool include_all);
const bool *ts_language_enabled_external_tokens(const t_language *self, uint32_t external_scanner_state);
const char *ts_external_scanner_state_data(const t_external_scanner_state *);
const t_external_scanner_state *ts_subtree_external_scanner_state(t_subtree self);
const t_parse_action *ts_language_actions(const t_language *self, t_state_id state, t_symbol symbol, uint32_t *count);
const t_symbol *ts_language_alias_sequence(const t_language *self, uint32_t production_id);
int ts_stack_dynamic_precedence(t_stack *, t_stack_version);
int ts_subtree_compare(t_subtree, t_subtree, t_subtree_pool *);
int32_t ts_subtree_dynamic_precedence(t_subtree self);
size_t atomic_load(const volatile size_t *p);
size_t ts_subtree_alloc_size(uint32_t child_count);
t_first_tree *ts_tree_new(t_subtree root, const t_language *language, const t_parse_range *, uint32_t);
t_length length_add(t_length len1, t_length len2);
t_length length_min(t_length len1, t_length len2);
t_length length_saturating_sub(t_length len1, t_length len2);
t_length length_sub(t_length len1, t_length len2);
t_length length_zero(void);
t_length ts_stack_position(const t_stack *, t_stack_version);
t_length ts_subtree_padding(t_subtree self);
t_length ts_subtree_size(t_subtree self);
t_length ts_subtree_total_size(t_subtree self);
t_lookahead_iterator ts_language_lookaheads(const t_language *self, t_state_id state);
t_mutable_subtree ts_subtree_make_mut(t_subtree_pool *, t_subtree);
t_mutable_subtree ts_subtree_new_node(t_symbol, t_subtree_array *, uint32_t, const t_language *);
t_mutable_subtree ts_subtree_to_mut_unsafe(t_subtree self);
t_parse_node ts_node_new(const t_first_tree *, const t_subtree *, t_length, t_symbol);
t_parse_node ts_tree_cursor_parent_node(const t_tree_cursor *);
t_parse_range *ts_lexer_included_ranges(const t_lexer *self, uint32_t *count);
t_point point__new(uint32_t row, uint32_t column);
t_point point_add(t_point a, t_point b);
t_point point_max(t_point a, t_point b);
t_point point_min(t_point a, t_point b);
t_point point_sub(t_point a, t_point b);
t_reusable_node reusable_node_new(void);
t_stack *ts_stack_new(t_subtree_pool *);
t_stack_slice_array ts_stack_pop_all(t_stack *, t_stack_version);
t_stack_slice_array ts_stack_pop_count(t_stack *, t_stack_version, uint32_t count);
t_stack_slice_array ts_stack_pop_pending(t_stack *, t_stack_version);
t_stack_summary *ts_stack_get_summary(t_stack *, t_stack_version);
t_stack_version ts_stack_copy_version(t_stack *, t_stack_version);
t_state_id ts_language_next_state(const t_language *self, t_state_id state, t_symbol symbol);
t_state_id ts_stack_state(const t_stack *, t_stack_version);
t_state_id ts_subtree_leaf_parse_state(t_subtree self);
t_state_id ts_subtree_parse_state(t_subtree self);
t_subtree reusable_node_tree(t_reusable_node *self);
t_subtree ts_stack_last_external_token(const t_stack *, t_stack_version);
t_subtree ts_stack_resume(t_stack *, t_stack_version);
t_subtree ts_subtree_edit(t_subtree, const t_input_edit *edit, t_subtree_pool *);
t_subtree ts_subtree_from_mut(t_mutable_subtree self);
t_subtree ts_subtree_last_external_token(t_subtree);
t_subtree ts_subtree_new_error(t_subtree_pool *, int32_t, t_length, t_length, uint32_t, t_state_id, const t_language *);
t_subtree ts_subtree_new_error_node(t_subtree_array *, bool, const t_language *);
t_subtree ts_subtree_new_leaf(t_subtree_pool *, t_symbol, t_length, t_length, uint32_t, t_state_id, bool, bool, bool, const t_language *);
t_subtree ts_subtree_new_missing_leaf(t_subtree_pool *, t_symbol, t_length, uint32_t, const t_language *);
t_subtree ts_tree_cursor_current_subtree(const t_tree_cursor *_self);
t_subtree_array ts_stack_pop_error(t_stack *, t_stack_version);
t_subtree_pool ts_subtree_pool_new(uint32_t capacity);
t_symbol ts_language_alias_at(const t_language *self, uint32_t production_id, uint32_t child_index);
t_symbol ts_language_public_symbol(const t_language *, t_symbol);
t_symbol ts_subtree_leaf_symbol(t_subtree self);
t_symbol ts_subtree_symbol(t_subtree self);
t_symbol_metadata ts_language_symbol_metadata(const t_language *, t_symbol);
t_tree_cursor_step ts_tree_cursor_goto_first_child_internal(t_tree_cursor *);
t_tree_cursor_step ts_tree_cursor_goto_next_sibling_internal(t_tree_cursor *);
uint16_t ts_language_lookup(const t_language *self, t_state_id state, t_symbol symbol);
uint16_t ts_subtree_production_id(t_subtree self);
uint32_t atomic_dec(volatile uint32_t *p);
uint32_t atomic_inc(volatile uint32_t *p);
uint32_t reusable_node_byte_offset(t_reusable_node *self);
uint32_t ts_stack_version_count(const t_stack *);
uint32_t ts_subtree_child_count(t_subtree self);
uint32_t ts_subtree_child_count(t_subtree self);
uint32_t ts_subtree_error_cost(t_subtree self);
uint32_t ts_subtree_is_repetition(t_subtree self);
uint32_t ts_subtree_lookahead_bytes(t_subtree self);
uint32_t ts_subtree_repeat_depth(t_subtree self);
uint32_t ts_subtree_total_bytes(t_subtree self);
uint32_t ts_subtree_total_bytes(t_subtree self);
uint32_t ts_subtree_visible_child_count(t_subtree self);
uint32_t ts_subtree_visible_descendant_count(t_subtree self);
uint32_t ts_stack_error_cost(const t_stack *, t_stack_version version);
uint32_t ts_stack_node_count_since_error(const t_stack *, t_stack_version);
uint32_t ts_subtree_get_changed_ranges(const t_subtree *old_tree, const t_subtree *new_tree, t_tree_cursor *cursor1, t_tree_cursor *cursor2, const t_language *language, const t_range_array *included_range_differences, t_parse_range **ranges);
void reusable_node_advance(t_reusable_node *self);
void reusable_node_advance_past_leaf(t_reusable_node *self);
void reusable_node_clear(t_reusable_node *self);
void reusable_node_delete(t_reusable_node *self);
void reusable_node_reset(t_reusable_node *self, t_subtree tree);
void ts_external_scanner_state_delete(t_external_scanner_state *self);
void ts_external_scanner_state_init(t_external_scanner_state *, const char *, uint32_t);
void ts_language_aliases_for_symbol(const t_language *self, t_symbol original_symbol, const t_symbol **start, const t_symbol **end);
void ts_language_field_map(const t_language *self, uint32_t production_id, const t_field_map_entry **start, const t_field_map_entry **end);
void ts_language_table_entry(const t_language *, t_state_id, t_symbol, t_table_entry *);
void ts_lexer_advance_to_end(t_lexer *lexer);
void ts_lexer_delete(t_lexer *lexer);
void ts_lexer_finish(t_lexer *lexer, uint32_t *);
void ts_lexer_init(t_lexer *lexer);
void ts_lexer_mark_end(t_lexer *lexer);
void ts_lexer_reset(t_lexer *lexer, t_length);
void ts_lexer_set_input(t_lexer *lexer, t_parse_input);
void ts_lexer_start(t_lexer *lexer);
void ts_range_array_get_changed_ranges(const t_parse_range *old_ranges, uint32_t old_range_count, const t_parse_range *new_ranges, uint32_t new_range_count, t_range_array *differences);
void ts_reduce_action_set_add(t_reduce_action_set *self, t_reduce_action new_action);
void ts_stack_clear(t_stack *);
void ts_stack_delete(t_stack *);
void ts_stack_halt(t_stack *, t_stack_version);
void ts_stack_pause(t_stack *, t_stack_version, t_subtree);
void ts_stack_push(t_stack *, t_stack_version, t_subtree, bool, t_state_id);
void ts_stack_record_summary(t_stack *, t_stack_version, uint32_t max_depth);
void ts_stack_remove_version(t_stack *, t_stack_version);
void ts_stack_renumber_version(t_stack *, t_stack_version, t_stack_version);
void ts_stack_set_last_external_token(t_stack *, t_stack_version, t_subtree);
void ts_stack_swap_versions(t_stack *, t_stack_version, t_stack_version);
void ts_subtree_array_clear(t_subtree_pool *, t_subtree_array *);
void ts_subtree_array_copy(t_subtree_array, t_subtree_array *);
void ts_subtree_array_delete(t_subtree_pool *, t_subtree_array *);
void ts_subtree_array_remove_trailing_extras(t_subtree_array *, t_subtree_array *);
void ts_subtree_array_reverse(t_subtree_array *);
void ts_subtree_balance(t_subtree, t_subtree_pool *, const t_language *);
void ts_subtree_pool_delete(t_subtree_pool *);
void ts_subtree_release(t_subtree_pool *, t_subtree);
void ts_subtree_retain(t_subtree);
void ts_subtree_set_extra(t_mutable_subtree *self, bool is_extra);
void ts_subtree_set_symbol(t_mutable_subtree *, t_symbol, const t_language *);
void ts_subtree_summarize(t_mutable_subtree, const t_subtree *, uint32_t, const t_language *);
void ts_subtree_summarize_children(t_mutable_subtree, const t_language *);
void ts_tree_cursor_current_status(const t_tree_cursor *, t_field_id *, bool *, bool *, bool *, t_symbol *, uint32_t *);
void ts_tree_cursor_init(t_tree_cursor *, t_parse_node);
#endif // FUNCS_H

1244
parser/src/scanner.c
View file

File diff suppressed because it is too large Load diff

543
parser/src/structs.h
View file

@ -1,543 +0,0 @@
#ifndef STRUCTS_H
#define STRUCTS_H
#include "./api.h"
typedef unsigned t_stack_action;
typedef struct s_analysis_state t_analysis_state;
typedef struct s_analysis_state_entry t_analysis_state_entry;
typedef struct s_analysis_subgraph t_analysis_subgraph;
typedef struct s_analysis_subgraph_node t_analysis_subgraph_node;
typedef struct s_capture_list_pool t_capture_list_pool;
typedef struct s_cursor_child_iterator t_cursor_child_iterator;
typedef struct s_edit t_edit;
typedef struct s_edit_entry t_edit_entry;
typedef struct s_error_status t_error_status;
typedef struct s_first_parser t_first_parser;
typedef struct s_iterator t_iterator;
typedef struct s_node_child_iterator t_node_child_iterator;
typedef struct s_parse_query t_parse_query;
typedef struct s_pattern_entry t_pattern_entry;
typedef struct s_query_analysis t_query_analysis;
typedef struct s_query_cursor t_query_cursor;
typedef struct s_query_pattern t_query_pattern;
typedef struct s_query_state t_query_state;
typedef struct s_query_step t_query_step;
typedef struct s_slice t_slice;
typedef struct s_stack t_stack;
typedef struct s_stack_head t_stack_head;
typedef struct s_stack_iterator t_stack_iterator;
typedef struct s_stack_link t_stack_link;
typedef struct s_stack_node t_stack_node;
typedef struct s_state_predecessor_map t_state_predecessor_map;
typedef struct s_step_offset t_step_offset;
typedef struct s_stream t_stream;
typedef struct s_string_input t_string_input;
typedef struct s_summarize_stack_session t_summarize_stack_session;
typedef struct s_symbol_table t_symbol_table;
typedef struct s_token_cache t_token_cache;
typedef t_stack_action (*t_stack_callback)(void *, const t_stack_iterator *);
typedef uint32_t (*t_unicode_decode_function)(const uint8_t *chunk,
uint32_t size,
int32_t *codepoint);
typedef Array(t_analysis_state *) t_analysis_state_set;
typedef Array(t_analysis_subgraph) t_analysis_subgraph_array;
typedef Array(t_query_capture) t_capture_list;
typedef Array(t_stack_node *) t_stack_node_array;
typedef Array(uint8_t) t_capture_quantifiers;
typedef enum e_stack_status t_stack_status;
typedef enum e_error_comparaison t_error_comparaison;
typedef enum e_iterator_comparison t_iterator_comparison;
struct s_iterator
{
t_tree_cursor cursor;
const t_language *language;
unsigned visible_depth;
bool in_padding;
};
enum e_iterator_comparison
{
IteratorDiffers,
IteratorMayDiffer,
IteratorMatches,
};
struct s_node_child_iterator
{
t_subtree parent;
const t_first_tree *tree;
t_length position;
uint32_t child_index;
uint32_t structural_child_index;
const t_symbol *alias_sequence;
};
struct s_token_cache
{
t_subtree token;
t_subtree last_external_token;
uint32_t byte_index;
};
struct s_first_parser
{
t_lexer lexer;
t_stack *stack;
t_subtree_pool tree_pool;
const t_language *language;
t_reduce_action_set reduce_actions;
t_subtree finished_tree;
t_subtree_array trailing_extras;
t_subtree_array trailing_extras2;
t_subtree_array scratch_trees;
t_token_cache token_cache;
t_reusable_node reusable_node;
void *external_scanner_payload;
t_parser_clock end_clock;
t_parser_duration timeout_duration;
unsigned accept_count;
unsigned operation_count;
const volatile size_t *cancellation_flag;
t_subtree old_tree;
t_range_array included_range_differences;
unsigned included_range_difference_index;
bool has_scanner_error;
};
struct s_error_status
{
unsigned cost;
unsigned node_count;
int dynamic_precedence;
bool is_in_error;
};
enum e_error_comparaison
{
ErrorComparisonTakeLeft,
ErrorComparisonPreferLeft,
ErrorComparisonNone,
ErrorComparisonPreferRight,
ErrorComparisonTakeRight,
};
struct s_string_input
{
const char *string;
uint32_t length;
};
/*
* t_stream - A sequence of unicode characters derived from a UTF8 string.
* This struct is used in parsing queries from S-expressions.
*/
struct s_stream
{
const char *input;
const char *start;
const char *end;
int32_t next;
uint8_t next_size;
};
/*
* t_query_step - A step in the process of matching a query. Each node within
* a query S-expression corresponds to one of these steps. An entire pattern
* is represented as a sequence of these steps. The basic properties of a
* node are represented by these fields:
* - `symbol` - The grammar symbol to match. A zero value represents the
* wildcard symbol, '_'.
* - `field` - The field name to match. A zero value means that a field name
* was not specified.
* - `capture_ids` - An array of integers representing the names of captures
* associated with this node in the pattern, terminated by a `NONE` value.
* - `depth` - The depth where this node occurs in the pattern. The root node
* of the pattern has depth zero.
* - `negated_field_list_id` - An id representing a set of fields that must
* not be present on a node matching this step.
*
* Steps have some additional fields in order to handle the `.` (or "anchor")
* operator, which forbids additional child nodes:
* - `is_immediate` - Indicates that the node matching this step cannot be
* preceded by other sibling nodes that weren't specified in the pattern.
* - `is_last_child` - Indicates that the node matching this step cannot have
* any subsequent named siblings.
*
* For simple patterns, steps are matched in sequential order. But in order to
* handle alternative/repeated/optional sub-patterns, query steps are not always
* structured as a linear sequence; they sometimes need to split and merge. This
* is done using the following fields:
* - `alternative_index` - The index of a different query step that serves as
* an alternative to this step. A `NONE` value represents no alternative.
* When a query state reaches a step with an alternative index, the state
* is duplicated, with one copy remaining at the original step, and one copy
* moving to the alternative step. The alternative may have its own
* alternative step, so this splitting is an iterative process.
* - `is_dead_end` - Indicates that this state cannot be passed directly, and
* exists only in order to redirect to an alternative index, with no
* splitting.
* - `is_pass_through` - Indicates that state has no matching logic of its own,
* and exists only to split a state. One copy of the state advances
* immediately to the next step, and one moves to the alternative step.
* - `alternative_is_immediate` - Indicates that this step's alternative step
* should be treated as if `is_immediate` is true.
*
* Steps also store some derived state that summarizes how they relate to other
* steps within the same pattern. This is used to optimize the matching process:
* - `contains_captures` - Indicates that this step or one of its child steps
* has a non-empty `capture_ids` list.
* - `parent_pattern_guaranteed` - Indicates that if this step is reached, then
* it and all of its subsequent sibling steps within the same parent pattern
* are guaranteed to match.
* - `root_pattern_guaranteed` - Similar to `parent_pattern_guaranteed`, but
* for the entire top-level pattern. When iterating through a query's
* captures using `ts_query_cursor_next_capture`, this field is used to
* detect that a capture can safely be returned from a match that has not
* even completed yet.
*/
struct s_query_step
{
t_symbol symbol;
t_symbol supertype_symbol;
t_field_id field;
uint16_t capture_ids[MAX_STEP_CAPTURE_COUNT];
uint16_t depth;
uint16_t alternative_index;
uint16_t negated_field_list_id;
bool is_named : 1;
bool is_immediate : 1;
bool is_last_child : 1;
bool is_pass_through : 1;
bool is_dead_end : 1;
bool alternative_is_immediate : 1;
bool contains_captures : 1;
bool root_pattern_guaranteed : 1;
bool parent_pattern_guaranteed : 1;
};
/*
* t_slice - A slice of an external array. Within a query, capture names,
* literal string values, and predicate step information are stored in three
* contiguous arrays. Individual captures, string values, and predicates are
* represented as slices of these three arrays.
*/
struct s_slice
{
uint32_t offset;
uint32_t length;
};
/*
* t_symbol_table - a two-way mapping of strings to ids.
*/
struct s_symbol_table
{
Array(char) characters;
Array(t_slice) slices;
};
/**
* CaptureQuantififers - a data structure holding the quantifiers of pattern
* captures.
*/
/*
* t_pattern_entry - Information about the starting point for matching a
* particular pattern. These entries are stored in a 'pattern map' - a sorted
* array that makes it possible to efficiently lookup patterns based on the
* symbol for their first step. The entry consists of the following fields:
* - `pattern_index` - the index of the pattern within the query
* - `step_index` - the index of the pattern's first step in the shared `steps`
* array
* - `is_rooted` - whether or not the pattern has a single root node. This
* property affects decisions about whether or not to start the pattern for
* nodes outside of a QueryCursor's range restriction.
*/
struct s_pattern_entry
{
uint16_t step_index;
uint16_t pattern_index;
bool is_rooted;
};
struct s_query_pattern
{
t_slice steps;
t_slice predicate_steps;
uint32_t start_byte;
bool is_non_local;
};
struct s_step_offset
{
uint32_t byte_offset;
uint16_t step_index;
};
/*
* t_query_state - The state of an in-progress match of a particular pattern
* in a query. While executing, a `t_query_cursor` must keep track of a number
* of possible in-progress matches. Each of those possible matches is
* represented as one of these states. Fields:
* - `id` - A numeric id that is exposed to the public API. This allows the
* caller to remove a given match, preventing any more of its captures
* from being returned.
* - `start_depth` - The depth in the tree where the first step of the state's
* pattern was matched.
* - `pattern_index` - The pattern that the state is matching.
* - `consumed_capture_count` - The number of captures from this match that
* have already been returned.
* - `capture_list_id` - A numeric id that can be used to retrieve the state's
* list of captures from the `t_capture_list_pool`.
* - `seeking_immediate_match` - A flag that indicates that the state's next
* step must be matched by the very next sibling. This is used when
* processing repetitions.
* - `has_in_progress_alternatives` - A flag that indicates that there is are
* other states that have the same captures as this state, but are at
* different steps in their pattern. This means that in order to obey the
* 'longest-match' rule, this state should not be returned as a match until
* it is clear that there can be no other alternative match with more
* captures.
*/
struct s_query_state
{
uint32_t id;
uint32_t capture_list_id;
uint16_t start_depth;
uint16_t step_index;
uint16_t pattern_index;
uint16_t consumed_capture_count : 12;
bool seeking_immediate_match : 1;
bool has_in_progress_alternatives : 1;
bool dead : 1;
bool needs_parent : 1;
};
/*
* t_capture_list_pool - A collection of *lists* of captures. Each query state
* needs to maintain its own list of captures. To avoid repeated allocations,
* this struct maintains a fixed set of capture lists, and keeps track of which
* ones are currently in use by a query state.
*/
struct s_capture_list_pool
{
Array(t_capture_list) list;
t_capture_list empty_list;
// The maximum number of capture lists that we are allowed to allocate. We
// never allow `list` to allocate more entries than this, dropping pending
// matches if needed to stay under the limit.
uint32_t max_capture_list_count;
// The number of capture lists allocated in `list` that are not currently in
// use. We reuse those existing-but-unused capture lists before trying to
// allocate any new ones. We use an invalid value (UINT32_MAX) for a capture
// list's length to indicate that it's not in use.
uint32_t free_capture_list_count;
};
/*
* t_analysis_state - The state needed for walking the parse table when
* analyzing a query pattern, to determine at which steps the pattern might fail
* to match.
*/
struct s_analysis_state_entry
{
t_state_id parse_state;
t_symbol parent_symbol;
uint16_t child_index;
t_field_id field_id : 15;
bool done : 1;
};
struct s_analysis_state
{
t_analysis_state_entry stack[MAX_ANALYSIS_STATE_DEPTH];
uint16_t depth;
uint16_t step_index;
t_symbol root_symbol;
};
struct s_query_analysis
{
t_analysis_state_set states;
t_analysis_state_set next_states;
t_analysis_state_set deeper_states;
t_analysis_state_set state_pool;
Array(uint16_t) final_step_indices;
Array(t_symbol) finished_parent_symbols;
bool did_abort;
};
/*
* t_analysis_subgraph - A subset of the states in the parse table that are used
* in constructing nodes with a certain symbol. Each state is accompanied by
* some information about the possible node that could be produced in
* downstream states.
*/
struct s_analysis_subgraph_node
{
t_state_id state;
uint16_t production_id;
uint8_t child_index : 7;
bool done : 1;
};
struct s_analysis_subgraph
{
t_symbol symbol;
Array(t_state_id) start_states;
Array(t_analysis_subgraph_node) nodes;
};
/*
* t_state_predecessor_map - A map that stores the predecessors of each parse
* state. This is used during query analysis to determine which parse states can
* lead to which reduce actions.
*/
struct s_state_predecessor_map
{
t_state_id *contents;
};
/*
* t_parse_query - A tree query, compiled from a string of S-expressions. The
* query itself is immutable. The mutable state used in the process of executing
* the query is stored in a `t_query_cursor`.
*/
struct s_parse_query
{
t_symbol_table captures;
t_symbol_table predicate_values;
Array(t_capture_quantifiers) capture_quantifiers;
Array(t_query_step) steps;
Array(t_pattern_entry) pattern_map;
Array(t_query_predicate_step) predicate_steps;
Array(t_query_pattern) patterns;
Array(t_step_offset) step_offsets;
Array(t_field_id) negated_fields;
Array(char) string_buffer;
Array(t_symbol) repeat_symbols_with_rootless_patterns;
const t_language *language;
uint16_t wildcard_root_pattern_count;
};
/*
* t_query_cursor - A stateful struct used to execute a query on a tree.
*/
struct s_query_cursor
{
const t_parse_query *query;
t_tree_cursor cursor;
Array(t_query_state) states;
Array(t_query_state) finished_states;
t_capture_list_pool capture_list_pool;
uint32_t depth;
uint32_t max_start_depth;
uint32_t start_byte;
uint32_t end_byte;
t_point start_point;
t_point end_point;
uint32_t next_state_id;
bool on_visible_node;
bool ascending;
bool halted;
bool did_exceed_match_limit;
};
struct s_stack_link
{
t_stack_node *node;
t_subtree subtree;
bool is_pending;
};
struct s_stack_node
{
t_state_id state;
t_length position;
t_stack_link links[MAX_LINK_COUNT];
short unsigned int link_count;
uint32_t ref_count;
unsigned error_cost;
unsigned node_count;
int dynamic_precedence;
};
struct s_stack_iterator
{
t_stack_node *node;
t_subtree_array subtrees;
uint32_t subtree_count;
bool is_pending;
};
enum e_stack_status
{
StackStatusActive,
StackStatusPaused,
StackStatusHalted,
};
struct s_stack_head
{
t_stack_node *node;
t_stack_summary *summary;
unsigned node_count_at_last_error;
t_subtree last_external_token;
t_subtree lookahead_when_paused;
t_stack_status status;
};
struct s_stack
{
Array(t_stack_head) heads;
t_stack_slice_array slices;
Array(t_stack_iterator) iterators;
t_stack_node_array node_pool;
t_stack_node *base_node;
t_subtree_pool *subtree_pool;
};
enum e_stack_action
{
StackActionNone,
StackActionStop = 1,
StackActionPop = 2,
};
struct s_summarize_stack_session
{
t_stack_summary *summary;
unsigned max_depth;
};
struct s_edit
{
t_length start;
t_length old_end;
t_length new_end;
};
struct s_edit_entry
{
t_subtree *tree;
t_edit edit;
};
struct s_cursor_child_iterator
{
t_subtree parent;
const t_first_tree *tree;
t_length position;
uint32_t child_index;
uint32_t structural_child_index;
uint32_t descendant_index;
const t_symbol *alias_sequence;
};
#endif // STRUCTS_H