raphael/minishell
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

started to rename struct and stuff

Browse source
This commit is contained in:
Maieul BOYER 2024-04-29 16:57:49 +02:00
parent ff4b0c471f
commit 54cefca53f
No known key found for this signature in database
25 changed files with 2413 additions and 2431 deletions
Download patch file Download diff file Expand all files Collapse all files

460
parser/includes/api.h
View file

File diff suppressed because it is too large Load diff

18
parser/includes/parser_raw_types.h
View file

@ -1,18 +0,0 @@
/* ************************************************************************** */
/* */
/* ::: :::::::: */
/* parser_raw_types.h :+: :+: :+: */
/* +:+ +:+ +:+ */
/* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */
/* +#+#+#+#+#+ +#+ */
/* Created: 2024/04/24 17:44:47 by maiboyer #+# #+# */
/* Updated: 2024/04/24 17:44:48 by maiboyer ### ########.fr */
/* */
/* ************************************************************************** */
#ifndef PARSER_RAW_TYPES_H
#define PARSER_RAW_TYPES_H
#endif /* PARSER_RAW_TYPES_H */

6
parser/includes/types/types_lexer.h
View file

@ -6,7 +6,7 @@
/* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */ /* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */
/* +#+#+#+#+#+ +#+ */ /* +#+#+#+#+#+ +#+ */
/* Created: 2024/04/29 14:45:45 by maiboyer #+# #+# */ /* Created: 2024/04/29 14:45:45 by maiboyer #+# #+# */
/* Updated: 2024/04/29 14:46:48 by maiboyer ### ########.fr */ /* Updated: 2024/04/29 16:24:16 by maiboyer ### ########.fr */
/* */ /* */
/* ************************************************************************** */ /* ************************************************************************** */
@ -18,11 +18,11 @@
typedef struct s_lexer typedef struct s_lexer
{ {
t_u32 lookahead; t_i32 lookahead;
t_symbol result_symbol; t_symbol result_symbol;
void (*advance)(struct s_lexer *, bool); void (*advance)(struct s_lexer *, bool);
void (*mark_end)(struct s_lexer *); void (*mark_end)(struct s_lexer *);
t_u32 (*get_column)(struct s_lexer *); t_i32 (*get_column)(struct s_lexer *);
bool (*is_at_included_range_start)(const struct s_lexer *); bool (*is_at_included_range_start)(const struct s_lexer *);
bool (*eof)(const struct s_lexer *); bool (*eof)(const struct s_lexer *);
} t_lexer; } t_lexer;

11
parser/includes/types/types_symbol.h
View file

@ -6,15 +6,18 @@
/* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */ /* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */
/* +#+#+#+#+#+ +#+ */ /* +#+#+#+#+#+ +#+ */
/* Created: 2024/04/29 14:43:33 by maiboyer #+# #+# */ /* Created: 2024/04/29 14:43:33 by maiboyer #+# #+# */
/* Updated: 2024/04/29 14:44:50 by maiboyer ### ########.fr */ /* Updated: 2024/04/29 16:41:28 by maiboyer ### ########.fr */
/* */ /* */
/* ************************************************************************** */ /* ************************************************************************** */
#ifndef TYPES_SYMBOL_H #ifndef TYPES_SYMBOL_H
# define TYPES_SYMBOL_H #define TYPES_SYMBOL_H
# include "me/types.h" #include "me/types.h"
typedef t_u16 t_symbol; typedef t_u16 t_symbol;
#define ts_builtin_sym_end (t_symbol)0
#define ts_builtin_sym_error (t_symbol)(-1)
#endif /* TYPES_SYMBOL_H */ #endif /* TYPES_SYMBOL_H */

143
parser/parse_types.h
View file

@ -6,40 +6,37 @@
/* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */ /* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */
/* +#+#+#+#+#+ +#+ */ /* +#+#+#+#+#+ +#+ */
/* Created: 2024/04/24 23:01:45 by maiboyer #+# #+# */ /* Created: 2024/04/24 23:01:45 by maiboyer #+# #+# */
/* Updated: 2024/04/29 15:26:51 by maiboyer ### ########.fr */ /* Updated: 2024/04/29 16:41:40 by maiboyer ### ########.fr */
/* */ /* */
/* ************************************************************************** */ /* ************************************************************************** */
#ifndef PARSETYPES_H #ifndef PARSE_TYPES_H
#define PARSETYPES_H # define PARSE_TYPES_H
#include <stdbool.h> # include <stdbool.h>
#include <stdint.h> # include <stdint.h>
#define ts_builtin_sym_end 0 # include "me/types.h"
#define ts_builtin_sym_error -1 # include "parser/types/types_char_range.h"
# include "parser/types/types_field_id.h"
# include "parser/types/types_field_map_entry.h"
# include "parser/types/types_field_map_slice.h"
# include "parser/types/types_language.h"
# include "parser/types/types_lex_modes.h"
# include "parser/types/types_lexer.h"
# include "parser/types/types_lexer_state.h"
# include "parser/types/types_parse_action_entry.h"
# include "parser/types/types_parse_action_type.h"
# include "parser/types/types_parse_actions.h"
# include "parser/types/types_parser_range.h"
# include "parser/types/types_point.h"
# include "parser/types/types_scanner.h"
# include "parser/types/types_state_id.h"
# include "parser/types/types_symbol.h"
# include "parser/types/types_symbol_metadata.h"
#include "me/types.h" static inline bool lex_skip(t_state_id state_value, t_lexer *lexer,
#include "parser/types/types_field_id.h" t_lexer_state *s)
#include "parser/types/types_field_map_entry.h"
#include "parser/types/types_field_map_slice.h"
#include "parser/types/types_language.h"
#include "parser/types/types_lex_modes.h"
#include "parser/types/types_lexer.h"
#include "parser/types/types_lexer_state.h"
#include "parser/types/types_parse_action_entry.h"
#include "parser/types/types_parse_action_type.h"
#include "parser/types/types_parse_actions.h"
#include "parser/types/types_parser_range.h"
#include "parser/types/types_point.h"
#include "parser/types/types_scanner.h"
#include "parser/types/types_state_id.h"
#include "parser/types/types_symbol.h"
#include "parser/types/types_symbol_metadata.h"
#include "parser/types/types_char_range.h"
static inline bool lex_skip(t_state_id state_value, t_lexer *lexer,
t_lexer_state *s)
{ {
(void)(lexer); (void)(lexer);
s->skip = true; s->skip = true;
@ -47,16 +44,16 @@ static inline bool lex_skip(t_state_id state_value, t_lexer *lexer,
return (true); return (true);
}; };
static inline bool lex_advance(t_state_id state_value, t_lexer *lexer, static inline bool lex_advance(t_state_id state_value, t_lexer *lexer,
t_lexer_state *s) t_lexer_state *s)
{ {
(void)(lexer); (void)(lexer);
s->state = state_value; s->state = state_value;
return (true); return (true);
}; };
static inline bool lex_accept_token(t_symbol symbol_value, t_lexer *lexer, static inline bool lex_accept_token(t_symbol symbol_value, t_lexer *lexer,
t_lexer_state *s) t_lexer_state *s)
{ {
s->result = true; s->result = true;
lexer->result_symbol = symbol_value; lexer->result_symbol = symbol_value;
@ -64,15 +61,15 @@ static inline bool lex_accept_token(t_symbol symbol_value, t_lexer *lexer,
return (true); return (true);
}; };
static inline bool lex_end_state(t_lexer *lexer, t_lexer_state *s) static inline bool lex_end_state(t_lexer *lexer, t_lexer_state *s)
{ {
(void)(lexer); (void)(lexer);
(void)(s); (void)(s);
return (false); return (false);
}; };
static inline t_field_map_entry fmap_entry(t_field_id field_id, static inline t_field_map_entry fmap_entry(t_field_id field_id,
uint8_t child_index, bool inherited) uint8_t child_index, bool inherited)
{ {
return ((t_field_map_entry){ return ((t_field_map_entry){
.field_id = field_id, .field_id = field_id,
@ -81,7 +78,7 @@ static inline t_field_map_entry fmap_entry(t_field_id field_id,
}); });
}; };
static inline t_field_map_slice fmap_slice(t_u16 index, t_u16 length) static inline t_field_map_slice fmap_slice(t_u16 index, t_u16 length)
{ {
return ((t_field_map_slice){ return ((t_field_map_slice){
.index = index, .index = index,
@ -89,8 +86,8 @@ static inline t_field_map_slice fmap_slice(t_u16 index, t_u16 length)
}); });
}; };
static inline t_symbol_metadata sym_metadata(bool visible, bool named, static inline t_symbol_metadata sym_metadata(bool visible, bool named,
bool supertype) bool supertype)
{ {
return ((t_symbol_metadata){ return ((t_symbol_metadata){
.visible = visible, .visible = visible,
@ -99,13 +96,13 @@ static inline t_symbol_metadata sym_metadata(bool visible, bool named,
}); });
}; };
static inline t_parse_action_entry entry(uint8_t count, bool reusable) static inline t_parse_action_entry entry(uint8_t count, bool reusable)
{ {
return ((t_parse_action_entry){ return ((t_parse_action_entry){.entry = {.count = count,
.entry = {.count = count, .reusable = reusable}}); .reusable = reusable}});
}; };
static inline t_parse_action_entry shift(t_state_id state_value) static inline t_parse_action_entry shift(t_state_id state_value)
{ {
return ((t_parse_action_entry){{.shift = { return ((t_parse_action_entry){{.shift = {
.type = ActionTypeShift, .type = ActionTypeShift,
@ -113,57 +110,55 @@ static inline t_parse_action_entry shift(t_state_id state_value)
}}}); }}});
}; };
static inline t_parse_action_entry shift_repeat(t_state_id state_value) static inline t_parse_action_entry shift_repeat(t_state_id state_value)
{ {
return ((t_parse_action_entry){{.shift = {.type = ActionTypeShift, return ((t_parse_action_entry){{.shift = {.type = ActionTypeShift,
.state = (state_value), .state = (state_value), .repetition = true}}});
.repetition = true}}});
}; };
static inline t_parse_action_entry shift_extra(void) static inline t_parse_action_entry shift_extra(void)
{ {
return ((t_parse_action_entry){ return ((t_parse_action_entry){{.shift = {.type = ActionTypeShift,
{.shift = {.type = ActionTypeShift, .extra = true}}}); .extra = true}}});
}; };
static inline t_parse_action_entry reduce( static inline t_parse_action_entry reduce(
t_symbol symbol, uint8_t child_count, int16_t dynamic_precedence, t_symbol symbol, uint8_t child_count, int16_t dynamic_precedence,
t_u16 production_id) t_u16 production_id)
{ {
return ( return ((t_parse_action_entry){{.reduce = {
(t_parse_action_entry){{.reduce = { .type = ActionTypeReduce,
.type = ActionTypeReduce, .child_count = child_count,
.child_count = child_count, .symbol = symbol,
.symbol = symbol, .dynamic_precedence = dynamic_precedence,
.dynamic_precedence = dynamic_precedence, .production_id = production_id,
.production_id = production_id, }}});
}}});
}; };
static inline t_parse_action_entry recover(void) static inline t_parse_action_entry recover(void)
{ {
return ((t_parse_action_entry){{.type = ActionTypeRecover}}); return ((t_parse_action_entry){{.type = ActionTypeRecover}});
}; };
static inline t_parse_action_entry accept(void) static inline t_parse_action_entry accept(void)
{ {
return ((t_parse_action_entry){{.type = ActionTypeAccept}}); return ((t_parse_action_entry){{.type = ActionTypeAccept}});
}; };
static inline bool set_contains(t_char_range *ranges, t_u32 len, static inline bool set_contains(t_char_range *ranges, t_u32 len,
int32_t lookahead) int32_t lookahead)
{ {
t_u32 index = 0; t_u32 index = 0;
t_u32 size = len - index; t_u32 size = len - index;
while (size > 1) while (size > 1)
{ {
t_u32 half_size = size / 2; t_u32 half_size = size / 2;
t_u32 mid_index = index + half_size; t_u32 mid_index = index + half_size;
t_char_range *range = &ranges[mid_index]; t_char_range *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end) if (lookahead >= range->start && lookahead <= range->end)
{ {
return true; return (true);
} }
else if (lookahead > range->end) else if (lookahead > range->end)
{ {
@ -175,8 +170,8 @@ static inline bool set_contains(t_char_range *ranges, t_u32 len,
return (lookahead >= range->start && lookahead <= range->end); return (lookahead >= range->start && lookahead <= range->end);
}; };
static inline bool advance_map_inner(t_u32 *map, t_u32 elems, t_lexer *l, static inline bool advance_map_inner(t_u32 *map, t_u32 elems, t_lexer *l,
t_lexer_state *s) t_lexer_state *s)
{ {
t_u32 i; t_u32 i;
@ -187,15 +182,15 @@ static inline bool advance_map_inner(t_u32 *map, t_u32 elems, t_lexer *l,
if (map[i] == (t_u32)s->lookahead) if (map[i] == (t_u32)s->lookahead)
{ {
s->state = map[i + 1]; s->state = map[i + 1];
return true; return (true);
} }
i += 2; i += 2;
} }
return (false); return (false);
}; };
static inline t_lex_modes lex_mode_external(t_u16 lex_state, static inline t_lex_modes lex_mode_external(t_u16 lex_state,
t_u16 ext_lex_state) t_u16 ext_lex_state)
{ {
return ((t_lex_modes){ return ((t_lex_modes){
.lex_state = lex_state, .lex_state = lex_state,
@ -203,21 +198,21 @@ static inline t_lex_modes lex_mode_external(t_u16 lex_state,
}); });
}; };
static inline t_lex_modes lex_mode_normal(t_u16 lex_state) static inline t_lex_modes lex_mode_normal(t_u16 lex_state)
{ {
return ((t_lex_modes){ return ((t_lex_modes){
.lex_state = lex_state, .lex_state = lex_state,
}); });
}; };
static inline t_u16 actions(t_u16 val) static inline t_u16 actions(t_u16 val)
{ {
return (val); return (val);
}; };
static inline t_u16 state(t_u16 val) static inline t_u16 state(t_u16 val)
{ {
return (val); return (val);
}; };
#endif /* PARSETYPES_H */ #endif /* PARSE_TYPES_H */

372
parser/src/array.h
View file

@ -1,11 +1,7 @@
#ifndef TREE_SITTER_ARRAY_H_ #ifndef TREE_SITTER_ARRAY_H_
#define TREE_SITTER_ARRAY_H_ #define TREE_SITTER_ARRAY_H_
#ifdef __cplusplus #include "me/types.h"
extern "C" {
#endif
#include <assert.h> #include <assert.h>
#include <stdbool.h> #include <stdbool.h>
@ -13,31 +9,27 @@ extern "C" {
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#ifdef _MSC_VER #define Array(T) \
#pragma warning(disable : 4101) struct \
#elif defined(__GNUC__) || defined(__clang__) { \
#pragma GCC diagnostic push T *contents; \
#pragma GCC diagnostic ignored "-Wunused-variable" t_u32 size; \
#endif t_u32 capacity; \
}
#define Array(T) \
struct { \
T *contents; \
uint32_t size; \
uint32_t capacity; \
}
/// Initialize an array. /// Initialize an array.
#define array_init(self) \ #define array_init(self) \
((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL) ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
/// Create an empty array. /// Create an empty array.
#define array_new() \ #define array_new() \
{ NULL, 0, 0 } { \
NULL, 0, 0 \
}
/// Get a pointer to the element at a given `index` in the array. /// Get a pointer to the element at a given `index` in the array.
#define array_get(self, _index) \ #define array_get(self, _index) \
(assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index]) (assert((t_u32)(_index) < (self)->size), &(self)->contents[_index])
/// Get a pointer to the first element in the array. /// Get a pointer to the first element in the array.
#define array_front(self) array_get(self, 0) #define array_front(self) array_get(self, 0)
@ -51,67 +43,67 @@ extern "C" {
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is /// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
/// less than the array's current capacity, this function has no effect. /// less than the array's current capacity, this function has no effect.
#define array_reserve(self, new_capacity) \ #define array_reserve(self, new_capacity) \
_array__reserve((Array *)(self), array_elem_size(self), new_capacity) _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
/// Free any memory allocated for this array. Note that this does not free any /// Free any memory allocated for this array. Note that this does not free any
/// memory allocated for the array's contents. /// memory allocated for the array's contents.
#define array_delete(self) _array__delete((Array *)(self)) #define array_delete(self) _array__delete((Array *)(self))
/// Push a new `element` onto the end of the array. /// Push a new `element` onto the end of the array.
#define array_push(self, element) \ #define array_push(self, element) \
(_array__grow((Array *)(self), 1, array_elem_size(self)), \ (_array__grow((Array *)(self), 1, array_elem_size(self)), \
(self)->contents[(self)->size++] = (element)) (self)->contents[(self)->size++] = (element))
/// Increase the array's size by `count` elements. /// Increase the array's size by `count` elements.
/// New elements are zero-initialized. /// New elements are zero-initialized.
#define array_grow_by(self, count) \ #define array_grow_by(self, count) \
do { \ do \
if ((count) == 0) break; \ { \
_array__grow((Array *)(self), count, array_elem_size(self)); \ if ((count) == 0) \
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \ break; \
(self)->size += (count); \ _array__grow((Array *)(self), count, array_elem_size(self)); \
} while (0) memset((self)->contents + (self)->size, 0, \
(count) * array_elem_size(self)); \
(self)->size += (count); \
} while (0)
/// Append all elements from one array to the end of another. /// Append all elements from one array to the end of another.
#define array_push_all(self, other) \ #define array_push_all(self, other) \
array_extend((self), (other)->size, (other)->contents) array_extend((self), (other)->size, (other)->contents)
/// Append `count` elements to the end of the array, reading their values from the /// Append `count` elements to the end of the array, reading their values from
/// `contents` pointer. /// the `contents` pointer.
#define array_extend(self, count, contents) \ #define array_extend(self, count, contents) \
_array__splice( \ _array__splice((Array *)(self), array_elem_size(self), (self)->size, 0, \
(Array *)(self), array_elem_size(self), (self)->size, \ count, contents)
0, count, contents \
)
/// Remove `old_count` elements from the array starting at the given `index`. At /// Remove `old_count` elements from the array starting at the given `index`. At
/// the same index, insert `new_count` new elements, reading their values from the /// the same index, insert `new_count` new elements, reading their values from
/// `new_contents` pointer. /// the `new_contents` pointer.
#define array_splice(self, _index, old_count, new_count, new_contents) \ #define array_splice(self, _index, old_count, new_count, new_contents) \
_array__splice( \ _array__splice((Array *)(self), array_elem_size(self), _index, old_count, \
(Array *)(self), array_elem_size(self), _index, \ new_count, new_contents)
old_count, new_count, new_contents \
)
/// Insert one `element` into the array at the given `index`. /// Insert one `element` into the array at the given `index`.
#define array_insert(self, _index, element) \ #define array_insert(self, _index, element) \
_array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element)) _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, \
&(element))
/// Remove one element from the array at the given `index`. /// Remove one element from the array at the given `index`.
#define array_erase(self, _index) \ #define array_erase(self, _index) \
_array__erase((Array *)(self), array_elem_size(self), _index) _array__erase((Array *)(self), array_elem_size(self), _index)
/// Pop the last element off the array, returning the element by value. /// Pop the last element off the array, returning the element by value.
#define array_pop(self) ((self)->contents[--(self)->size]) #define array_pop(self) ((self)->contents[--(self)->size])
/// Assign the contents of one array to another, reallocating if necessary. /// Assign the contents of one array to another, reallocating if necessary.
#define array_assign(self, other) \ #define array_assign(self, other) \
_array__assign((Array *)(self), (const Array *)(other), array_elem_size(self)) _array__assign((Array *)(self), (const Array *)(other), \
array_elem_size(self))
/// Swap one array with another /// Swap one array with another
#define array_swap(self, other) \ #define array_swap(self, other) _array__swap((Array *)(self), (Array *)(other))
_array__swap((Array *)(self), (Array *)(other))
/// Get the size of the array contents /// Get the size of the array contents
#define array_elem_size(self) (sizeof *(self)->contents) #define array_elem_size(self) (sizeof *(self)->contents)
@ -124,167 +116,187 @@ extern "C" {
/// out-parameter is set to true. Otherwise, `index` is set to an index where /// out-parameter is set to true. Otherwise, `index` is set to an index where
/// `needle` should be inserted in order to preserve the sorting, and `exists` /// `needle` should be inserted in order to preserve the sorting, and `exists`
/// is set to false. /// is set to false.
#define array_search_sorted_with(self, compare, needle, _index, _exists) \ #define array_search_sorted_with(self, compare, needle, _index, _exists) \
_array__search_sorted(self, 0, compare, , needle, _index, _exists) _array__search_sorted(self, 0, compare, , needle, _index, _exists)
/// Search a sorted array for a given `needle` value, using integer comparisons /// Search a sorted array for a given `needle` value, using integer comparisons
/// of a given struct field (specified with a leading dot) to determine the order. /// of a given struct field (specified with a leading dot) to determine the
/// order.
/// ///
/// See also `array_search_sorted_with`. /// See also `array_search_sorted_with`.
#define array_search_sorted_by(self, field, needle, _index, _exists) \ #define array_search_sorted_by(self, field, needle, _index, _exists) \
_array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists) _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
/// Insert a given `value` into a sorted array, using the given `compare` /// Insert a given `value` into a sorted array, using the given `compare`
/// callback to determine the order. /// callback to determine the order.
#define array_insert_sorted_with(self, compare, value) \ #define array_insert_sorted_with(self, compare, value) \
do { \ do \
unsigned _index, _exists; \ { \
array_search_sorted_with(self, compare, &(value), &_index, &_exists); \ unsigned _index, _exists; \
if (!_exists) array_insert(self, _index, value); \ array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
} while (0) if (!_exists) \
array_insert(self, _index, value); \
} while (0)
/// Insert a given `value` into a sorted array, using integer comparisons of /// Insert a given `value` into a sorted array, using integer comparisons of
/// a given struct field (specified with a leading dot) to determine the order. /// a given struct field (specified with a leading dot) to determine the order.
/// ///
/// See also `array_search_sorted_by`. /// See also `array_search_sorted_by`.
#define array_insert_sorted_by(self, field, value) \ #define array_insert_sorted_by(self, field, value) \
do { \ do \
unsigned _index, _exists; \ { \
array_search_sorted_by(self, field, (value) field, &_index, &_exists); \ unsigned _index, _exists; \
if (!_exists) array_insert(self, _index, value); \ array_search_sorted_by(self, field, (value)field, &_index, &_exists); \
} while (0) if (!_exists) \
array_insert(self, _index, value); \
} while (0)
// Private // Private
typedef Array(void) Array; typedef Array(void) Array;
/// This is not what you're looking for, see `array_delete`. /// This is not what you're looking for, see `array_delete`.
static inline void _array__delete(Array *self) { static inline void _array__delete(Array *self)
if (self->contents) { {
free(self->contents); if (self->contents)
self->contents = NULL; {
self->size = 0; free(self->contents);
self->capacity = 0; self->contents = NULL;
} self->size = 0;
self->capacity = 0;
}
} }
/// This is not what you're looking for, see `array_erase`. /// This is not what you're looking for, see `array_erase`.
static inline void _array__erase(Array *self, size_t element_size, static inline void _array__erase(Array *self, size_t element_size, t_u32 index)
uint32_t index) { {
assert(index < self->size); assert(index < self->size);
char *contents = (char *)self->contents; char *contents = (char *)self->contents;
memmove(contents + index * element_size, contents + (index + 1) * element_size, memmove(contents + index * element_size,
(self->size - index - 1) * element_size); contents + (index + 1) * element_size,
self->size--; (self->size - index - 1) * element_size);
self->size--;
} }
/// This is not what you're looking for, see `array_reserve`. /// This is not what you're looking for, see `array_reserve`.
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) { static inline void _array__reserve(Array *self, size_t element_size,
if (new_capacity > self->capacity) { t_u32 new_capacity)
if (self->contents) { {
self->contents = realloc(self->contents, new_capacity * element_size); if (new_capacity > self->capacity)
} else { {
self->contents = malloc(new_capacity * element_size); if (self->contents)
} {
self->capacity = new_capacity; self->contents =
} realloc(self->contents, new_capacity * element_size);
}
else
{
self->contents = malloc(new_capacity * element_size);
}
self->capacity = new_capacity;
}
} }
/// This is not what you're looking for, see `array_assign`. /// This is not what you're looking for, see `array_assign`.
static inline void _array__assign(Array *self, const Array *other, size_t element_size) { static inline void _array__assign(Array *self, const Array *other,
_array__reserve(self, element_size, other->size); size_t element_size)
self->size = other->size; {
memcpy(self->contents, other->contents, self->size * element_size); _array__reserve(self, element_size, other->size);
self->size = other->size;
memcpy(self->contents, other->contents, self->size * element_size);
} }
/// This is not what you're looking for, see `array_swap`. /// This is not what you're looking for, see `array_swap`.
static inline void _array__swap(Array *self, Array *other) { static inline void _array__swap(Array *self, Array *other)
Array swap = *other; {
*other = *self; Array swap = *other;
*self = swap; *other = *self;
*self = swap;
} }
/// This is not what you're looking for, see `array_push` or `array_grow_by`. /// This is not what you're looking for, see `array_push` or `array_grow_by`.
static inline void _array__grow(Array *self, uint32_t count, size_t element_size) { static inline void _array__grow(Array *self, t_u32 count, size_t element_size)
uint32_t new_size = self->size + count; {
if (new_size > self->capacity) { t_u32 new_size = self->size + count;
uint32_t new_capacity = self->capacity * 2; if (new_size > self->capacity)
if (new_capacity < 8) new_capacity = 8; {
if (new_capacity < new_size) new_capacity = new_size; t_u32 new_capacity = self->capacity * 2;
_array__reserve(self, element_size, new_capacity); if (new_capacity < 8)
} new_capacity = 8;
if (new_capacity < new_size)
new_capacity = new_size;
_array__reserve(self, element_size, new_capacity);
}
} }
/// This is not what you're looking for, see `array_splice`. /// This is not what you're looking for, see `array_splice`.
static inline void _array__splice(Array *self, size_t element_size, static inline void _array__splice(Array *self, size_t element_size, t_u32 index,
uint32_t index, uint32_t old_count, t_u32 old_count, t_u32 new_count,
uint32_t new_count, const void *elements) { const void *elements)
uint32_t new_size = self->size + new_count - old_count; {
uint32_t old_end = index + old_count; t_u32 new_size = self->size + new_count - old_count;
uint32_t new_end = index + new_count; t_u32 old_end = index + old_count;
assert(old_end <= self->size); t_u32 new_end = index + new_count;
assert(old_end <= self->size);
_array__reserve(self, element_size, new_size); _array__reserve(self, element_size, new_size);
char *contents = (char *)self->contents; char *contents = (char *)self->contents;
if (self->size > old_end) { if (self->size > old_end)
memmove( {
contents + new_end * element_size, memmove(contents + new_end * element_size,
contents + old_end * element_size, contents + old_end * element_size,
(self->size - old_end) * element_size (self->size - old_end) * element_size);
); }
} if (new_count > 0)
if (new_count > 0) { {
if (elements) { if (elements)
memcpy( {
(contents + index * element_size), memcpy((contents + index * element_size), elements,
elements, new_count * element_size);
new_count * element_size }
); else
} else { {
memset( memset((contents + index * element_size), 0,
(contents + index * element_size), new_count * element_size);
0, }
new_count * element_size }
); self->size += new_count - old_count;
}
}
self->size += new_count - old_count;
} }
/// A binary search routine, based on Rust's `std::slice::binary_search_by`. /// A binary search routine, based on Rust's `std::slice::binary_search_by`.
/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`. /// This is not what you're looking for, see `array_search_sorted_with` or
#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \ /// `array_search_sorted_by`.
do { \ #define _array__search_sorted(self, start, compare, suffix, needle, _index, \
*(_index) = start; \ _exists) \
*(_exists) = false; \ do \
uint32_t size = (self)->size - *(_index); \ { \
if (size == 0) break; \ *(_index) = start; \
int comparison; \ *(_exists) = false; \
while (size > 1) { \ t_u32 size = (self)->size - *(_index); \
uint32_t half_size = size / 2; \ if (size == 0) \
uint32_t mid_index = *(_index) + half_size; \ break; \
comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \ int comparison; \
if (comparison <= 0) *(_index) = mid_index; \ while (size > 1) \
size -= half_size; \ { \
} \ t_u32 half_size = size / 2; \
comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \ t_u32 mid_index = *(_index) + half_size; \
if (comparison == 0) *(_exists) = true; \ comparison = \
else if (comparison < 0) *(_index) += 1; \ compare(&((self)->contents[mid_index] suffix), (needle)); \
} while (0) if (comparison <= 0) \
*(_index) = mid_index; \
size -= half_size; \
} \
comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
if (comparison == 0) \
*(_exists) = true; \
else if (comparison < 0) \
*(_index) += 1; \
} while (0)
/// Helper macro for the `_sorted_by` routines below. This takes the left (existing) /// Helper macro for the `_sorted_by` routines below. This takes the left
/// parameter by reference in order to work with the generic sorting function above. /// (existing) parameter by reference in order to work with the generic sorting
/// function above.
#define _compare_int(a, b) ((int)*(a) - (int)(b)) #define _compare_int(a, b) ((int)*(a) - (int)(b))
#ifdef _MSC_VER #endif // TREE_SITTER_ARRAY_H_
#pragma warning(default : 4101)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic pop
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ARRAY_H_

112
parser/src/language.c
View file

@ -2,34 +2,34 @@
#include "parser/api.h" #include "parser/api.h"
#include <string.h> #include <string.h>
const TSLanguage *ts_language_copy(const TSLanguage *self) { const t_language *ts_language_copy(const t_language *self) {
return self; return self;
} }
void ts_language_delete(const TSLanguage *self) { void ts_language_delete(const t_language *self) {
(void)(self); (void)(self);
} }
uint32_t ts_language_symbol_count(const TSLanguage *self) { t_u32 ts_language_symbol_count(const t_language *self) {
return self->symbol_count + self->alias_count; return self->symbol_count + self->alias_count;
} }
uint32_t ts_language_state_count(const TSLanguage *self) { t_u32 ts_language_state_count(const t_language *self) {
return self->state_count; return self->state_count;
} }
uint32_t ts_language_version(const TSLanguage *self) { t_u32 ts_language_version(const t_language *self) {
return self->version; return self->version;
} }
uint32_t ts_language_field_count(const TSLanguage *self) { t_u32 ts_language_field_count(const t_language *self) {
return self->field_count; return self->field_count;
} }
void ts_language_table_entry( void ts_language_table_entry(
const TSLanguage *self, const t_language *self,
TSStateId state, t_state_id state,
TSSymbol symbol, t_symbol symbol,
TableEntry *result TableEntry *result
) { ) {
if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
@ -38,48 +38,48 @@ void ts_language_table_entry(
result->actions = NULL; result->actions = NULL;
} else { } else {
assert(symbol < self->token_count); assert(symbol < self->token_count);
uint32_t action_index = ts_language_lookup(self, state, symbol); t_u32 action_index = ts_language_lookup(self, state, symbol);
const TSParseActionEntry *entry = &self->parse_actions[action_index]; const t_parse_action_entry *entry = &self->parse_actions[action_index];
result->action_count = entry->entry.count; result->action_count = entry->entry.count;
result->is_reusable = entry->entry.reusable; result->is_reusable = entry->entry.reusable;
result->actions = (const TSParseAction *)(entry + 1); result->actions = (const t_parse_actions *)(entry + 1);
} }
} }
TSSymbolMetadata ts_language_symbol_metadata( t_symbol_metadata ts_language_symbol_metadata(
const TSLanguage *self, const t_language *self,
TSSymbol symbol t_symbol symbol
) { ) {
if (symbol == ts_builtin_sym_error) { if (symbol == ts_builtin_sym_error) {
return (TSSymbolMetadata) {.visible = true, .named = true}; return (t_symbol_metadata) {.visible = true, .named = true};
} else if (symbol == ts_builtin_sym_error_repeat) { } else if (symbol == ts_builtin_sym_error_repeat) {
return (TSSymbolMetadata) {.visible = false, .named = false}; return (t_symbol_metadata) {.visible = false, .named = false};
} else { } else {
return self->symbol_metadata[symbol]; return self->symbol_metadata[symbol];
} }
} }
TSSymbol ts_language_public_symbol( t_symbol ts_language_public_symbol(
const TSLanguage *self, const t_language *self,
TSSymbol symbol t_symbol symbol
) { ) {
if (symbol == ts_builtin_sym_error) return symbol; if (symbol == ts_builtin_sym_error) return symbol;
return self->public_symbol_map[symbol]; return self->public_symbol_map[symbol];
} }
TSStateId ts_language_next_state( t_state_id ts_language_next_state(
const TSLanguage *self, const t_language *self,
TSStateId state, t_state_id state,
TSSymbol symbol t_symbol symbol
) { ) {
if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
return 0; return 0;
} else if (symbol < self->token_count) { } else if (symbol < self->token_count) {
uint32_t count; t_u32 count;
const TSParseAction *actions = ts_language_actions(self, state, symbol, &count); const t_parse_actions *actions = ts_language_actions(self, state, symbol, &count);
if (count > 0) { if (count > 0) {
TSParseAction action = actions[count - 1]; t_parse_actions action = actions[count - 1];
if (action.type == TSParseActionTypeShift) { if (action.type == ActionTypeShift) {
return action.shift.extra ? state : action.shift.state; return action.shift.extra ? state : action.shift.state;
} }
} }
@ -90,8 +90,8 @@ TSStateId ts_language_next_state(
} }
const char *ts_language_symbol_name( const char *ts_language_symbol_name(
const TSLanguage *self, const t_language *self,
TSSymbol symbol t_symbol symbol
) { ) {
if (symbol == ts_builtin_sym_error) { if (symbol == ts_builtin_sym_error) {
return "ERROR"; return "ERROR";
@ -104,16 +104,16 @@ const char *ts_language_symbol_name(
} }
} }
TSSymbol ts_language_symbol_for_name( t_symbol ts_language_symbol_for_name(
const TSLanguage *self, const t_language *self,
const char *string, const char *string,
uint32_t length, t_u32 length,
bool is_named bool is_named
) { ) {
if (!strncmp(string, "ERROR", length)) return ts_builtin_sym_error; if (!strncmp(string, "ERROR", length)) return ts_builtin_sym_error;
uint16_t count = (uint16_t)ts_language_symbol_count(self); t_u16 count = (t_u16)ts_language_symbol_count(self);
for (TSSymbol i = 0; i < count; i++) { for (t_symbol i = 0; i < count; i++) {
TSSymbolMetadata metadata = ts_language_symbol_metadata(self, i); t_symbol_metadata metadata = ts_language_symbol_metadata(self, i);
if ((!metadata.visible && !metadata.supertype) || metadata.named != is_named) continue; if ((!metadata.visible && !metadata.supertype) || metadata.named != is_named) continue;
const char *symbol_name = self->symbol_names[i]; const char *symbol_name = self->symbol_names[i];
if (!strncmp(symbol_name, string, length) && !symbol_name[length]) { if (!strncmp(symbol_name, string, length) && !symbol_name[length]) {
@ -123,25 +123,25 @@ TSSymbol ts_language_symbol_for_name(
return 0; return 0;
} }
TSSymbolType ts_language_symbol_type( t_symbol_type ts_language_symbol_type(
const TSLanguage *self, const t_language *self,
TSSymbol symbol t_symbol symbol
) { ) {
TSSymbolMetadata metadata = ts_language_symbol_metadata(self, symbol); t_symbol_metadata metadata = ts_language_symbol_metadata(self, symbol);
if (metadata.named && metadata.visible) { if (metadata.named && metadata.visible) {
return TSSymbolTypeRegular; return SymbolTypeRegular;
} else if (metadata.visible) { } else if (metadata.visible) {
return TSSymbolTypeAnonymous; return SymbolTypeAnonymous;
} else { } else {
return TSSymbolTypeAuxiliary; return SymbolTypeAuxiliary;
} }
} }
const char *ts_language_field_name_for_id( const char *ts_language_field_name_for_id(
const TSLanguage *self, const t_language *self,
TSFieldId id t_field_id id
) { ) {
uint32_t count = ts_language_field_count(self); t_u32 count = ts_language_field_count(self);
if (count && id <= count) { if (count && id <= count) {
return self->field_names[id]; return self->field_names[id];
} else { } else {
@ -149,13 +149,13 @@ const char *ts_language_field_name_for_id(
} }
} }
TSFieldId ts_language_field_id_for_name( t_field_id ts_language_field_id_for_name(
const TSLanguage *self, const t_language *self,
const char *name, const char *name,
uint32_t name_length t_u32 name_length
) { ) {
uint16_t count = (uint16_t)ts_language_field_count(self); t_u16 count = (t_u16)ts_language_field_count(self);
for (TSSymbol i = 1; i < count + 1; i++) { for (t_symbol i = 1; i < count + 1; i++) {
switch (strncmp(name, self->field_names[i], name_length)) { switch (strncmp(name, self->field_names[i], name_length)) {
case 0: case 0:
if (self->field_names[i][name_length] == 0) return i; if (self->field_names[i][name_length] == 0) return i;
@ -169,7 +169,7 @@ TSFieldId ts_language_field_id_for_name(
return 0; return 0;
} }
TSLookaheadIterator *ts_lookahead_iterator_new(const TSLanguage *self, TSStateId state) { TSLookaheadIterator *ts_lookahead_iterator_new(const t_language *self, t_state_id state) {
if (state >= self->state_count) return NULL; if (state >= self->state_count) return NULL;
LookaheadIterator *iterator = malloc(sizeof(LookaheadIterator)); LookaheadIterator *iterator = malloc(sizeof(LookaheadIterator));
*iterator = ts_language_lookaheads(self, state); *iterator = ts_language_lookaheads(self, state);
@ -180,19 +180,19 @@ void ts_lookahead_iterator_delete(TSLookaheadIterator *self) {
free(self); free(self);
} }
bool ts_lookahead_iterator_reset_state(TSLookaheadIterator * self, TSStateId state) { bool ts_lookahead_iterator_reset_state(TSLookaheadIterator * self, t_state_id state) {
LookaheadIterator *iterator = (LookaheadIterator *)self; LookaheadIterator *iterator = (LookaheadIterator *)self;
if (state >= iterator->language->state_count) return false; if (state >= iterator->language->state_count) return false;
*iterator = ts_language_lookaheads(iterator->language, state); *iterator = ts_language_lookaheads(iterator->language, state);
return true; return true;
} }
const TSLanguage *ts_lookahead_iterator_language(const TSLookaheadIterator *self) { const t_language *ts_lookahead_iterator_language(const TSLookaheadIterator *self) {
const LookaheadIterator *iterator = (const LookaheadIterator *)self; const LookaheadIterator *iterator = (const LookaheadIterator *)self;
return iterator->language; return iterator->language;
} }
bool ts_lookahead_iterator_reset(TSLookaheadIterator *self, const TSLanguage *language, TSStateId state) { bool ts_lookahead_iterator_reset(TSLookaheadIterator *self, const t_language *language, t_state_id state) {
if (state >= language->state_count) return false; if (state >= language->state_count) return false;
LookaheadIterator *iterator = (LookaheadIterator *)self; LookaheadIterator *iterator = (LookaheadIterator *)self;
*iterator = ts_language_lookaheads(language, state); *iterator = ts_language_lookaheads(language, state);
@ -204,7 +204,7 @@ bool ts_lookahead_iterator_next(TSLookaheadIterator *self) {
return ts_lookahead_iterator__next(iterator); return ts_lookahead_iterator__next(iterator);
} }
TSSymbol ts_lookahead_iterator_current_symbol(const TSLookaheadIterator *self) { t_symbol ts_lookahead_iterator_current_symbol(const TSLookaheadIterator *self) {
const LookaheadIterator *iterator = (const LookaheadIterator *)self; const LookaheadIterator *iterator = (const LookaheadIterator *)self;
return iterator->symbol; return iterator->symbol;
} }

493
parser/src/language.h
View file

@ -1,72 +1,75 @@
#ifndef TREE_SITTER_LANGUAGE_H_ #ifndef TREE_SITTER_LANGUAGE_H_
#define TREE_SITTER_LANGUAGE_H_ #define TREE_SITTER_LANGUAGE_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "./subtree.h"
#include "./parser.h" #include "./parser.h"
#include "./subtree.h"
#include "parser/types/types_parse_action_type.h"
#include "parser/types/types_state_id.h"
#include "parser/types/types_symbol.h"
#define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1) #define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1)
#define LANGUAGE_VERSION_WITH_PRIMARY_STATES 14 #define LANGUAGE_VERSION_WITH_PRIMARY_STATES 14
#define LANGUAGE_VERSION_USABLE_VIA_WASM 13 #define LANGUAGE_VERSION_USABLE_VIA_WASM 13
typedef struct { typedef struct
const TSParseAction *actions; {
uint32_t action_count; const t_parse_actions *actions;
bool is_reusable; t_u32 action_count;
bool is_reusable;
} TableEntry; } TableEntry;
typedef struct { typedef struct
const TSLanguage *language; {
const uint16_t *data; const t_language *language;
const uint16_t *group_end; const t_u16 *data;
TSStateId state; const t_u16 *group_end;
uint16_t table_value; t_state_id state;
uint16_t section_index; t_u16 table_value;
uint16_t group_count; t_u16 section_index;
bool is_small_state; t_u16 group_count;
bool is_small_state;
const TSParseAction *actions; const t_parse_actions *actions;
TSSymbol symbol; t_symbol symbol;
TSStateId next_state; t_state_id next_state;
uint16_t action_count; t_u16 action_count;
} LookaheadIterator; } LookaheadIterator;
void ts_language_table_entry(const TSLanguage *, TSStateId, TSSymbol, TableEntry *); void ts_language_table_entry(const t_language *, t_state_id, t_symbol,
TableEntry *);
TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *, TSSymbol); t_symbol_metadata ts_language_symbol_metadata(const t_language *, t_symbol);
TSSymbol ts_language_public_symbol(const TSLanguage *, TSSymbol); t_symbol ts_language_public_symbol(const t_language *, t_symbol);
TSStateId ts_language_next_state(const TSLanguage *self, TSStateId state, TSSymbol symbol); t_state_id ts_language_next_state(const t_language *self, t_state_id state,
t_symbol symbol);
static inline bool ts_language_is_symbol_external(const TSLanguage *self, TSSymbol symbol) { static inline bool ts_language_is_symbol_external(const t_language *self,
return 0 < symbol && symbol < self->external_token_count + 1; t_symbol symbol)
{
return 0 < symbol && symbol < self->external_token_count + 1;
} }
static inline const TSParseAction *ts_language_actions( static inline const t_parse_actions *ts_language_actions(const t_language *self,
const TSLanguage *self, t_state_id state,
TSStateId state, t_symbol symbol,
TSSymbol symbol, t_u32 *count)
uint32_t *count {
) { TableEntry entry;
TableEntry entry; ts_language_table_entry(self, state, symbol, &entry);
ts_language_table_entry(self, state, symbol, &entry); *count = entry.action_count;
*count = entry.action_count; return entry.actions;
return entry.actions;
} }
static inline bool ts_language_has_reduce_action( static inline bool ts_language_has_reduce_action(const t_language *self,
const TSLanguage *self, t_state_id state,
TSStateId state, t_symbol symbol)
TSSymbol symbol {
) { TableEntry entry;
TableEntry entry; ts_language_table_entry(self, state, symbol, &entry);
ts_language_table_entry(self, state, symbol, &entry); return entry.action_count > 0 && entry.actions[0].type == ActionTypeReduce;
return entry.action_count > 0 && entry.actions[0].type == TSParseActionTypeReduce;
} }
// Lookup the table value for a given symbol and state. // Lookup the table value for a given symbol and state.
@ -76,34 +79,37 @@ static inline bool ts_language_has_reduce_action(
// For 'large' parse states, this is a direct lookup. For 'small' parse // For 'large' parse states, this is a direct lookup. For 'small' parse
// states, this requires searching through the symbol groups to find // states, this requires searching through the symbol groups to find
// the given symbol. // the given symbol.
static inline uint16_t ts_language_lookup( static inline t_u16 ts_language_lookup(const t_language *self, t_state_id state,
const TSLanguage *self, t_symbol symbol)
TSStateId state, {
TSSymbol symbol if (state >= self->large_state_count)
) { {
if (state >= self->large_state_count) { t_u32 index =
uint32_t index = self->small_parse_table_map[state - self->large_state_count]; self->small_parse_table_map[state - self->large_state_count];
const uint16_t *data = &self->small_parse_table[index]; const t_u16 *data = &self->small_parse_table[index];
uint16_t group_count = *(data++); t_u16 group_count = *(data++);
for (unsigned i = 0; i < group_count; i++) { for (unsigned i = 0; i < group_count; i++)
uint16_t section_value = *(data++); {
uint16_t symbol_count = *(data++); t_u16 section_value = *(data++);
for (unsigned j = 0; j < symbol_count; j++) { t_u16 symbol_count = *(data++);
if (*(data++) == symbol) return section_value; for (unsigned j = 0; j < symbol_count; j++)
} {
} if (*(data++) == symbol)
return 0; return section_value;
} else { }
return self->parse_table[state * self->symbol_count + symbol]; }
} return 0;
}
else
{
return self->parse_table[state * self->symbol_count + symbol];
}
} }
static inline bool ts_language_has_actions( static inline bool ts_language_has_actions(const t_language *self,
const TSLanguage *self, t_state_id state, t_symbol symbol)
TSStateId state, {
TSSymbol symbol return ts_language_lookup(self, state, symbol) != 0;
) {
return ts_language_lookup(self, state, symbol) != 0;
} }
// Iterate over all of the symbols that are valid in the given state. // Iterate over all of the symbols that are valid in the given state.
@ -112,188 +118,209 @@ static inline bool ts_language_has_actions(
// all possible symbols and checking the parse table for each one. // all possible symbols and checking the parse table for each one.
// For 'small' parse states, this exploits the structure of the // For 'small' parse states, this exploits the structure of the
// table to only visit the valid symbols. // table to only visit the valid symbols.
static inline LookaheadIterator ts_language_lookaheads( static inline LookaheadIterator ts_language_lookaheads(const t_language *self,
const TSLanguage *self, t_state_id state)
TSStateId state {
) { bool is_small_state = state >= self->large_state_count;
bool is_small_state = state >= self->large_state_count; const t_u16 *data;
const uint16_t *data; const t_u16 *group_end = NULL;
const uint16_t *group_end = NULL; t_u16 group_count = 0;
uint16_t group_count = 0; if (is_small_state)
if (is_small_state) { {
uint32_t index = self->small_parse_table_map[state - self->large_state_count]; t_u32 index =
data = &self->small_parse_table[index]; self->small_parse_table_map[state - self->large_state_count];
group_end = data + 1; data = &self->small_parse_table[index];
group_count = *data; group_end = data + 1;
} else { group_count = *data;
data = &self->parse_table[state * self->symbol_count] - 1; }
} else
return (LookaheadIterator) { {
.language = self, data = &self->parse_table[state * self->symbol_count] - 1;
.data = data, }
.group_end = group_end, return (LookaheadIterator){
.group_count = group_count, .language = self,
.is_small_state = is_small_state, .data = data,
.symbol = UINT16_MAX, .group_end = group_end,
.next_state = 0, .group_count = group_count,
}; .is_small_state = is_small_state,
.symbol = UINT16_MAX,
.next_state = 0,
};
} }
static inline bool ts_lookahead_iterator__next(LookaheadIterator *self) { static inline bool ts_lookahead_iterator__next(LookaheadIterator *self)
// For small parse states, valid symbols are listed explicitly, {
// grouped by their value. There's no need to look up the actions // For small parse states, valid symbols are listed explicitly,
// again until moving to the next group. // grouped by their value. There's no need to look up the actions
if (self->is_small_state) { // again until moving to the next group.
self->data++; if (self->is_small_state)
if (self->data == self->group_end) { {
if (self->group_count == 0) return false; self->data++;
self->group_count--; if (self->data == self->group_end)
self->table_value = *(self->data++); {
unsigned symbol_count = *(self->data++); if (self->group_count == 0)
self->group_end = self->data + symbol_count; return false;
self->symbol = *self->data; self->group_count--;
} else { self->table_value = *(self->data++);
self->symbol = *self->data; unsigned symbol_count = *(self->data++);
return true; self->group_end = self->data + symbol_count;
} self->symbol = *self->data;
} }
else
{
self->symbol = *self->data;
return true;
}
}
// For large parse states, iterate through every symbol until one // For large parse states, iterate through every symbol until one
// is found that has valid actions. // is found that has valid actions.
else { else
do { {
self->data++; do
self->symbol++; {
if (self->symbol >= self->language->symbol_count) return false; self->data++;
self->table_value = *self->data; self->symbol++;
} while (!self->table_value); if (self->symbol >= self->language->symbol_count)
} return false;
self->table_value = *self->data;
} while (!self->table_value);
}
// Depending on if the symbols is terminal or non-terminal, the table value either // Depending on if the symbols is terminal or non-terminal, the table
// represents a list of actions or a successor state. // value either represents a list of actions or a successor state.
if (self->symbol < self->language->token_count) { if (self->symbol < self->language->token_count)
const TSParseActionEntry *entry = &self->language->parse_actions[self->table_value]; {
self->action_count = entry->entry.count; const t_parse_action_entry *entry =
self->actions = (const TSParseAction *)(entry + 1); &self->language->parse_actions[self->table_value];
self->next_state = 0; self->action_count = entry->entry.count;
} else { self->actions = (const t_parse_actions *)(entry + 1);
self->action_count = 0; self->next_state = 0;
self->next_state = self->table_value; }
} else
return true; {
self->action_count = 0;
self->next_state = self->table_value;
}
return true;
} }
// Whether the state is a "primary state". If this returns false, it indicates that there exists // Whether the state is a "primary state". If this returns false, it
// another state that behaves identically to this one with respect to query analysis. // indicates that there exists another state that behaves identically to
static inline bool ts_language_state_is_primary( // this one with respect to query analysis.
const TSLanguage *self, static inline bool ts_language_state_is_primary(const t_language *self,
TSStateId state t_state_id state)
) { {
if (self->version >= LANGUAGE_VERSION_WITH_PRIMARY_STATES) { if (self->version >= LANGUAGE_VERSION_WITH_PRIMARY_STATES)
return state == self->primary_state_ids[state]; {
} else { return state == self->primary_state_ids[state];
return true; }
} else
{
return true;
}
} }
static inline const bool *ts_language_enabled_external_tokens( static inline const bool *ts_language_enabled_external_tokens(
const TSLanguage *self, const t_language *self, unsigned external_scanner_state)
unsigned external_scanner_state {
) { if (external_scanner_state == 0)
if (external_scanner_state == 0) { {
return NULL; return NULL;
} else { }
return self->external_scanner.states + self->external_token_count * external_scanner_state; else
} {
return self->external_scanner.states +
self->external_token_count * external_scanner_state;
}
} }
static inline const TSSymbol *ts_language_alias_sequence( static inline const t_symbol *ts_language_alias_sequence(const t_language *self,
const TSLanguage *self, t_u32 production_id)
uint32_t production_id {
) { return production_id
return production_id ? ? &self->alias_sequences[production_id *
&self->alias_sequences[production_id * self->max_alias_sequence_length] : self->max_alias_sequence_length]
NULL; : NULL;
} }
static inline TSSymbol ts_language_alias_at( static inline t_symbol ts_language_alias_at(const t_language *self,
const TSLanguage *self, t_u32 production_id,
uint32_t production_id, t_u32 child_index)
uint32_t child_index {
) { return production_id
return production_id ? ? self->alias_sequences[production_id *
self->alias_sequences[production_id * self->max_alias_sequence_length + child_index] : self->max_alias_sequence_length +
0; child_index]
: 0;
} }
static inline void ts_language_field_map( static inline void ts_language_field_map(const t_language *self,
const TSLanguage *self, t_u32 production_id,
uint32_t production_id, const t_field_map_entry **start,
const TSFieldMapEntry **start, const t_field_map_entry **end)
const TSFieldMapEntry **end {
) { if (self->field_count == 0)
if (self->field_count == 0) { {
*start = NULL; *start = NULL;
*end = NULL; *end = NULL;
return; return;
} }
TSFieldMapSlice slice = self->field_map_slices[production_id]; t_field_map_slice slice = self->field_map_slices[production_id];
*start = &self->field_map_entries[slice.index]; *start = &self->field_map_entries[slice.index];
*end = &self->field_map_entries[slice.index] + slice.length; *end = &self->field_map_entries[slice.index] + slice.length;
} }
static inline void ts_language_aliases_for_symbol( static inline void ts_language_aliases_for_symbol(const t_language *self,
const TSLanguage *self, t_symbol original_symbol,
TSSymbol original_symbol, const t_symbol **start,
const TSSymbol **start, const t_symbol **end)
const TSSymbol **end {
) { *start = &self->public_symbol_map[original_symbol];
*start = &self->public_symbol_map[original_symbol]; *end = *start + 1;
*end = *start + 1;
unsigned idx = 0; unsigned idx = 0;
for (;;) { for (;;)
TSSymbol symbol = self->alias_map[idx++]; {
if (symbol == 0 || symbol > original_symbol) break; t_symbol symbol = self->alias_map[idx++];
uint16_t count = self->alias_map[idx++]; if (symbol == 0 || symbol > original_symbol)
if (symbol == original_symbol) { break;
*start = &self->alias_map[idx]; t_u16 count = self->alias_map[idx++];
*end = &self->alias_map[idx + count]; if (symbol == original_symbol)
break; {
} *start = &self->alias_map[idx];
idx += count; *end = &self->alias_map[idx + count];
} break;
}
idx += count;
}
} }
static inline void ts_language_write_symbol_as_dot_string( static inline void ts_language_write_symbol_as_dot_string(
const TSLanguage *self, const t_language *self, FILE *f, t_symbol symbol)
FILE *f, {
TSSymbol symbol const char *name = ts_language_symbol_name(self, symbol);
) { for (const char *chr = name; *chr; chr++)
const char *name = ts_language_symbol_name(self, symbol); {
for (const char *chr = name; *chr; chr++) { switch (*chr)
switch (*chr) { {
case '"': case '"':
case '\\': case '\\':
fputc('\\', f); fputc('\\', f);
fputc(*chr, f); fputc(*chr, f);
break; break;
case '\n': case '\n':
fputs("\\n", f); fputs("\\n", f);
break; break;
case '\t': case '\t':
fputs("\\t", f); fputs("\\t", f);
break; break;
default: default:
fputc(*chr, f); fputc(*chr, f);
break; break;
} }
} }
} }
#ifdef __cplusplus #endif // TREE_SITTER_LANGUAGE_H_
}
#endif
#endif // TREE_SITTER_LANGUAGE_H_

2
parser/src/length.h
View file

@ -7,7 +7,7 @@
#include "parser/api.h" #include "parser/api.h"
typedef struct { typedef struct {
uint32_t bytes; t_u32 bytes;
t_point extent; t_point extent;
} Length; } Length;

104
parser/src/lexer.c
View file

@ -16,33 +16,33 @@
self->debug_buffer); \ self->debug_buffer); \
} }
static const int32_t BYTE_ORDER_MARK = 0xFEFF; static const t_i32 BYTE_ORDER_MARK = 0xFEFF;
static const t_parser_range DEFAULT_RANGE = {.start_point = static const t_parser_range DEFAULT_RANGE = {.start_point =
{ {
.row = 0, .row = 0,
.column = 0, .column = 0,
}, },
.end_point = .end_point =
{ {
.row = UINT32_MAX, .row = UINT32_MAX,
.column = UINT32_MAX, .column = UINT32_MAX,
}, },
.start_byte = 0, .start_byte = 0,
.end_byte = UINT32_MAX}; .end_byte = UINT32_MAX};
// Check if the lexer has reached EOF. This state is stored // Check if the lexer has reached EOF. This state is stored
// by setting the lexer's `current_included_range_index` such that // by setting the lexer's `current_included_range_index` such that
// it has consumed all of its available ranges. // it has consumed all of its available ranges.
static bool ts_lexer__eof(const TSLexer *_self) static bool ts_lexer__eof(const t_lexer *_self)
{ {
Lexer *self = (Lexer *)_self; t_liblexer *self = (t_liblexer *)_self;
return self->current_included_range_index == self->included_range_count; return self->current_included_range_index == self->included_range_count;
} }
// Clear the currently stored chunk of source code, because the lexer's // Clear the currently stored chunk of source code, because the lexer's
// position has changed. // position has changed.
static void ts_lexer__clear_chunk(Lexer *self) static void ts_lexer__clear_chunk(t_liblexer *self)
{ {
self->chunk = NULL; self->chunk = NULL;
self->chunk_size = 0; self->chunk_size = 0;
@ -51,7 +51,7 @@ static void ts_lexer__clear_chunk(Lexer *self)
// Call the lexer's input callback to obtain a new chunk of source code // Call the lexer's input callback to obtain a new chunk of source code
// for the current position. // for the current position.
static void ts_lexer__get_chunk(Lexer *self) static void ts_lexer__get_chunk(t_liblexer *self)
{ {
self->chunk_start = self->current_position.bytes; self->chunk_start = self->current_position.bytes;
self->chunk = self->chunk =
@ -64,13 +64,13 @@ static void ts_lexer__get_chunk(Lexer *self)
} }
} }
typedef uint32_t (*UnicodeDecodeFunction)(const uint8_t *chunk, uint32_t size, typedef t_i32 (*UnicodeDecodeFunction)(const t_i8 *chunk, t_i32 size,
int32_t *lookahead); t_i32 *lookahead);
uint32_t my_decode(const uint8_t *chunk, uint32_t size, int32_t *lookahead) t_i32 my_decode(const t_i8 *chunk, t_i32 size, t_i32 *lookahead)
{ {
(void)(size); (void)(size);
*((uint32_t *)lookahead) = *chunk; *((t_i32 *)lookahead) = *chunk;
return (1); return (1);
} }
@ -79,11 +79,11 @@ uint32_t my_decode(const uint8_t *chunk, uint32_t size, int32_t *lookahead)
// Decode the next unicode character in the current chunk of source code. // Decode the next unicode character in the current chunk of source code.
// This assumes that the lexer has already retrieved a chunk of source // This assumes that the lexer has already retrieved a chunk of source
// code that spans the current position. // code that spans the current position.
static void ts_lexer__get_lookahead(Lexer *self) static void ts_lexer__get_lookahead(t_liblexer *self)
{ {
uint32_t position_in_chunk = t_i32 position_in_chunk =
self->current_position.bytes - self->chunk_start; self->current_position.bytes - self->chunk_start;
uint32_t size = self->chunk_size - position_in_chunk; t_i32 size = self->chunk_size - position_in_chunk;
if (size == 0) if (size == 0)
{ {
@ -92,7 +92,7 @@ static void ts_lexer__get_lookahead(Lexer *self)
return; return;
} }
const uint8_t *chunk = (const uint8_t *)self->chunk + position_in_chunk; const t_i8 *chunk = (const t_i8 *)self->chunk + position_in_chunk;
UnicodeDecodeFunction decode = my_decode; UnicodeDecodeFunction decode = my_decode;
self->lookahead_size = decode(chunk, size, &self->data.lookahead); self->lookahead_size = decode(chunk, size, &self->data.lookahead);
@ -102,7 +102,7 @@ static void ts_lexer__get_lookahead(Lexer *self)
if (self->data.lookahead == TS_DECODE_ERROR && size < 4) if (self->data.lookahead == TS_DECODE_ERROR && size < 4)
{ {
ts_lexer__get_chunk(self); ts_lexer__get_chunk(self);
chunk = (const uint8_t *)self->chunk; chunk = (const t_i8 *)self->chunk;
size = self->chunk_size; size = self->chunk_size;
self->lookahead_size = decode(chunk, size, &self->data.lookahead); self->lookahead_size = decode(chunk, size, &self->data.lookahead);
} }
@ -113,7 +113,7 @@ static void ts_lexer__get_lookahead(Lexer *self)
} }
} }
static void ts_lexer_goto(Lexer *self, Length position) static void ts_lexer_goto(t_liblexer *self, Length position)
{ {
self->current_position = position; self->current_position = position;
@ -172,7 +172,7 @@ static void ts_lexer_goto(Lexer *self, Length position)
} }
// Intended to be called only from functions that control logging. // Intended to be called only from functions that control logging.
static void ts_lexer__do_advance(Lexer *self, bool skip) static void ts_lexer__do_advance(t_liblexer *self, bool skip)
{ {
if (self->lookahead_size) if (self->lookahead_size)
{ {
@ -235,9 +235,9 @@ static void ts_lexer__do_advance(Lexer *self, bool skip)
// Advance to the next character in the source code, retrieving a new // Advance to the next character in the source code, retrieving a new
// chunk of source code if needed. // chunk of source code if needed.
static void ts_lexer__advance(TSLexer *_self, bool skip) static void ts_lexer__advance(t_lexer *_self, bool skip)
{ {
Lexer *self = (Lexer *)_self; t_liblexer *self = (t_liblexer *)_self;
if (!self->chunk) if (!self->chunk)
return; return;
ts_lexer__do_advance(self, skip); ts_lexer__do_advance(self, skip);
@ -245,9 +245,9 @@ static void ts_lexer__advance(TSLexer *_self, bool skip)
// Mark that a token match has completed. This can be called multiple // Mark that a token match has completed. This can be called multiple
// times if a longer match is found later. // times if a longer match is found later.
static void ts_lexer__mark_end(TSLexer *_self) static void ts_lexer__mark_end(t_lexer *_self)
{ {
Lexer *self = (Lexer *)_self; t_liblexer *self = (t_liblexer *)_self;
if (!ts_lexer__eof(&self->data)) if (!ts_lexer__eof(&self->data))
{ {
// If the lexer is right at the beginning of included range, // If the lexer is right at the beginning of included range,
@ -258,7 +258,8 @@ static void ts_lexer__mark_end(TSLexer *_self)
if (self->current_included_range_index > 0 && if (self->current_included_range_index > 0 &&
self->current_position.bytes == current_included_range->start_byte) self->current_position.bytes == current_included_range->start_byte)
{ {
t_parser_range *previous_included_range = current_included_range - 1; t_parser_range *previous_included_range =
current_included_range - 1;
self->token_end_position = (Length){ self->token_end_position = (Length){
previous_included_range->end_byte, previous_included_range->end_byte,
previous_included_range->end_point, previous_included_range->end_point,
@ -269,11 +270,11 @@ static void ts_lexer__mark_end(TSLexer *_self)
self->token_end_position = self->current_position; self->token_end_position = self->current_position;
} }
static uint32_t ts_lexer__get_column(TSLexer *_self) static t_i32 ts_lexer__get_column(t_lexer *_self)
{ {
Lexer *self = (Lexer *)_self; t_liblexer *self = (t_liblexer *)_self;
uint32_t goal_byte = self->current_position.bytes; t_u32 goal_byte = self->current_position.bytes;
self->did_get_column = true; self->did_get_column = true;
self->current_position.bytes -= self->current_position.extent.column; self->current_position.bytes -= self->current_position.extent.column;
@ -284,7 +285,7 @@ static uint32_t ts_lexer__get_column(TSLexer *_self)
ts_lexer__get_chunk(self); ts_lexer__get_chunk(self);
} }
uint32_t result = 0; t_i32 result = 0;
if (!ts_lexer__eof(_self)) if (!ts_lexer__eof(_self))
{ {
ts_lexer__get_lookahead(self); ts_lexer__get_lookahead(self);
@ -303,9 +304,9 @@ static uint32_t ts_lexer__get_column(TSLexer *_self)
// Is the lexer at a boundary between two disjoint included ranges of // Is the lexer at a boundary between two disjoint included ranges of
// source code? This is exposed as an API because some languages' external // source code? This is exposed as an API because some languages' external
// scanners need to perform custom actions at these boundaries. // scanners need to perform custom actions at these boundaries.
static bool ts_lexer__is_at_included_range_start(const TSLexer *_self) static bool ts_lexer__is_at_included_range_start(const t_lexer *_self)
{ {
const Lexer *self = (const Lexer *)_self; const t_liblexer *self = (const t_liblexer *)_self;
if (self->current_included_range_index < self->included_range_count) if (self->current_included_range_index < self->included_range_count)
{ {
t_parser_range *current_range = t_parser_range *current_range =
@ -318,9 +319,9 @@ static bool ts_lexer__is_at_included_range_start(const TSLexer *_self)
} }
} }
void ts_lexer_init(Lexer *self) void ts_lexer_init(t_liblexer *self)
{ {
*self = (Lexer){ *self = (t_liblexer){
.data = .data =
{ {
// The lexer's methods are stored as struct fields so that // The lexer's methods are stored as struct fields so that
@ -349,12 +350,12 @@ void ts_lexer_init(Lexer *self)
ts_lexer_set_included_ranges(self, NULL, 0); ts_lexer_set_included_ranges(self, NULL, 0);
} }
void ts_lexer_delete(Lexer *self) void ts_lexer_delete(t_liblexer *self)
{ {
free(self->included_ranges); free(self->included_ranges);
} }
void ts_lexer_set_input(Lexer *self, TSInput input) void ts_lexer_set_input(t_liblexer *self, TSInput input)
{ {
self->input = input; self->input = input;
ts_lexer__clear_chunk(self); ts_lexer__clear_chunk(self);
@ -363,7 +364,7 @@ void ts_lexer_set_input(Lexer *self, TSInput input)
// Move the lexer to the given position. This doesn't do any work // Move the lexer to the given position. This doesn't do any work
// if the parser is already at the given position. // if the parser is already at the given position.
void ts_lexer_reset(Lexer *self, Length position) void ts_lexer_reset(t_liblexer *self, Length position)
{ {
if (position.bytes != self->current_position.bytes) if (position.bytes != self->current_position.bytes)
{ {
@ -371,7 +372,7 @@ void ts_lexer_reset(Lexer *self, Length position)
} }
} }
void ts_lexer_start(Lexer *self) void ts_lexer_start(t_liblexer *self)
{ {
self->token_start_position = self->current_position; self->token_start_position = self->current_position;
self->token_end_position = LENGTH_UNDEFINED; self->token_end_position = LENGTH_UNDEFINED;
@ -389,7 +390,7 @@ void ts_lexer_start(Lexer *self)
} }
} }
void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte) void ts_lexer_finish(t_liblexer *self, t_i32 *lookahead_end_byte)
{ {
if (length_is_undefined(self->token_end_position)) if (length_is_undefined(self->token_end_position))
{ {
@ -404,7 +405,7 @@ void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte)
self->token_start_position = self->token_end_position; self->token_start_position = self->token_end_position;
} }
uint32_t current_lookahead_end_byte = self->current_position.bytes + 1; t_i32 current_lookahead_end_byte = self->current_position.bytes + 1;
// In order to determine that a byte sequence is invalid UTF8 or UTF16, // In order to determine that a byte sequence is invalid UTF8 or UTF16,
// the character decoding algorithm may have looked at the following byte. // the character decoding algorithm may have looked at the following byte.
@ -421,7 +422,7 @@ void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte)
} }
} }
void ts_lexer_advance_to_end(Lexer *self) void ts_lexer_advance_to_end(t_liblexer *self)
{ {
while (self->chunk) while (self->chunk)
{ {
@ -429,13 +430,13 @@ void ts_lexer_advance_to_end(Lexer *self)
} }
} }
void ts_lexer_mark_end(Lexer *self) void ts_lexer_mark_end(t_liblexer *self)
{ {
ts_lexer__mark_end(&self->data); ts_lexer__mark_end(&self->data);
} }
bool ts_lexer_set_included_ranges(Lexer *self, const t_parser_range *ranges, bool ts_lexer_set_included_ranges(t_liblexer *self,
uint32_t count) const t_parser_range *ranges, t_u32 count)
{ {
ranges = &DEFAULT_RANGE; ranges = &DEFAULT_RANGE;
count = 1; count = 1;
@ -447,7 +448,8 @@ bool ts_lexer_set_included_ranges(Lexer *self, const t_parser_range *ranges,
return true; return true;
} }
t_parser_range *ts_lexer_included_ranges(const Lexer *self, uint32_t *count) t_parser_range *ts_lexer_included_ranges(const t_liblexer *self,
t_u32 *count)
{ {
*count = self->included_range_count; *count = self->included_range_count;
return self->included_ranges; return self->included_ranges;

83
parser/src/lexer.h
View file

@ -1,49 +1,60 @@
/* ************************************************************************** */
/* */
/* ::: :::::::: */
/* lexer.h :+: :+: :+: */
/* +:+ +:+ +:+ */
/* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */
/* +#+#+#+#+#+ +#+ */
/* Created: 2024/04/29 16:17:29 by maiboyer #+# #+# */
/* Updated: 2024/04/29 16:55:37 by maiboyer ### ########.fr */
/* */
/* ************************************************************************** */
#ifndef TREE_SITTER_LEXER_H_ #ifndef TREE_SITTER_LEXER_H_
#define TREE_SITTER_LEXER_H_ #define TREE_SITTER_LEXER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "./length.h" #include "./length.h"
#include "./parser.h"
#include "./subtree.h" #include "./subtree.h"
#include "parser/api.h" #include "parser/api.h"
#include "./parser.h"
typedef struct { #include "me/types.h"
TSLexer data; #include "parser/types/types_lexer.h"
Length current_position;
Length token_start_position;
Length token_end_position;
t_parser_range *included_ranges; #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
const char *chunk;
TSInput input;
TSLogger logger;
uint32_t included_range_count; typedef struct s_liblexer
uint32_t current_included_range_index; {
uint32_t chunk_start; t_lexer data;
uint32_t chunk_size; Length current_position;
uint32_t lookahead_size; Length token_start_position;
bool did_get_column; Length token_end_position;
char debug_buffer[TREE_SITTER_SERIALIZATION_BUFFER_SIZE]; t_parser_range *included_ranges;
} Lexer; const char *chunk;
TSInput input;
TSLogger logger;
void ts_lexer_init(Lexer *); t_u32 included_range_count;
void ts_lexer_delete(Lexer *); t_u32 current_included_range_index;
void ts_lexer_set_input(Lexer *, TSInput); t_u32 chunk_start;
void ts_lexer_reset(Lexer *, Length); t_u32 chunk_size;
void ts_lexer_start(Lexer *); t_u32 lookahead_size;
void ts_lexer_finish(Lexer *, uint32_t *); bool did_get_column;
void ts_lexer_advance_to_end(Lexer *);
void ts_lexer_mark_end(Lexer *);
bool ts_lexer_set_included_ranges(Lexer *self, const t_parser_range *ranges, uint32_t count);
t_parser_range *ts_lexer_included_ranges(const Lexer *self, uint32_t *count);
#ifdef __cplusplus char debug_buffer[TREE_SITTER_SERIALIZATION_BUFFER_SIZE];
} } t_liblexer;
#endif
#endif // TREE_SITTER_LEXER_H_ void ts_lexer_init(t_liblexer *);
void ts_lexer_delete(t_liblexer *);
void ts_lexer_set_input(t_liblexer *, TSInput);
void ts_lexer_reset(t_liblexer *, Length);
void ts_lexer_start(t_liblexer *);
void ts_lexer_finish(t_liblexer *, t_i32 *);
void ts_lexer_advance_to_end(t_liblexer *);
void ts_lexer_mark_end(t_liblexer *);
bool ts_lexer_set_included_ranges(t_liblexer *self,
const t_parser_range *ranges, t_u32 count);
t_parser_range *ts_lexer_included_ranges(const t_liblexer *self, t_u32 *count);
#endif // TREE_SITTER_LEXER_H_

1351
parser/src/node.c
View file

File diff suppressed because it is too large Load diff

354
parser/src/parser.c
View file

@ -18,9 +18,10 @@
#include <time.h> #include <time.h>
#include "me/vec/vec_parser_range.h" #include "me/vec/vec_parser_range.h"
#include "parser/types/types_language.h"
typedef uint64_t TSDuration; typedef t_u64 t_duration;
typedef uint64_t TSClock; typedef t_u64 t_clock;
#define LOG(...) \ #define LOG(...) \
if (self->lexer.logger.log || self->dot_graph_file) \ if (self->lexer.logger.log || self->dot_graph_file) \
@ -102,70 +103,70 @@ static const unsigned MAX_SUMMARY_DEPTH = 16;
static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE; static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE;
// static const unsigned OP_COUNT_PER_TIMEOUT_CHECK = 100; // static const unsigned OP_COUNT_PER_TIMEOUT_CHECK = 100;
typedef struct typedef struct s_token_cache
{ {
Subtree token; Subtree token;
Subtree last_external_token; Subtree last_external_token;
uint32_t byte_index; t_u32 byte_index;
} TokenCache; } t_token_cache;
struct TSParser typedef struct s_parser
{ {
Lexer lexer; t_liblexer lexer;
Stack *stack; Stack *stack;
SubtreePool tree_pool; SubtreePool tree_pool;
const TSLanguage *language; const t_language *language;
void *wasm_store; void *wasm_store;
ReduceActionSet reduce_actions; ReduceActionSet reduce_actions;
Subtree finished_tree; Subtree finished_tree;
SubtreeArray trailing_extras; SubtreeArray trailing_extras;
SubtreeArray trailing_extras2; SubtreeArray trailing_extras2;
SubtreeArray scratch_trees; SubtreeArray scratch_trees;
TokenCache token_cache; t_token_cache token_cache;
ReusableNode reusable_node; ReusableNode reusable_node;
void *external_scanner_payload; void *external_scanner_payload;
FILE *dot_graph_file; FILE *dot_graph_file;
TSClock end_clock; t_clock end_clock;
TSDuration timeout_duration; t_duration timeot_duration;
unsigned accept_count; t_u32 accept_count;
unsigned operation_count; t_u32 operation_count;
const volatile size_t *cancellation_flag; const volatile size_t *cancellation_flag;
Subtree old_tree; Subtree old_tree;
t_vec_parser_range included_range_differences; t_vec_parser_range included_range_differences;
unsigned included_range_difference_index; t_u32 included_range_difference_index;
bool has_scanner_error; bool has_scanner_error;
}; } t_parser;
typedef struct typedef struct s_error_status
{ {
unsigned cost; t_u32 cost;
unsigned node_count; t_u32 node_count;
int dynamic_precedence; t_i32 dynamic_precedence;
bool is_in_error; bool is_in_error;
} ErrorStatus; } t_error_status;
typedef enum typedef enum e_error_comparison
{ {
ErrorComparisonTakeLeft, ErrorComparisonTakeLeft,
ErrorComparisonPreferLeft, ErrorComparisonPreferLeft,
ErrorComparisonNone, ErrorComparisonNone,
ErrorComparisonPreferRight, ErrorComparisonPreferRight,
ErrorComparisonTakeRight, ErrorComparisonTakeRight,
} ErrorComparison; } t_error_comparison;
typedef struct typedef struct s_string_input
{ {
const char *string; const char *string;
uint32_t length; t_u32 length;
} TSStringInput; } t_string_input;
// StringInput // StringInput
static const char *ts_string_input_read(void *_self, uint32_t byte, static const char *ts_string_inpt_read(void *_self, t_u32 byte,
t_point point, uint32_t *length) t_point point, t_u32 *length)
{ {
(void)point; (void)point;
TSStringInput *self = (TSStringInput *)_self; t_string_input *self = (t_string_input *)_self;
if (byte >= self->length) if (byte >= self->length)
{ {
*length = 0; *length = 0;
@ -180,7 +181,7 @@ static const char *ts_string_input_read(void *_self, uint32_t byte,
// Parser - Private // Parser - Private
static void ts_parser__log(TSParser *self) static void ts_parser__log(t_parser *self)
{ {
if (self->lexer.logger.log) if (self->lexer.logger.log)
{ {
@ -201,7 +202,7 @@ static void ts_parser__log(TSParser *self)
} }
} }
static bool ts_parser__breakdown_top_of_stack(TSParser *self, static bool ts_parser__breakdown_top_of_stack(t_parser *self,
StackVersion version) StackVersion version)
{ {
bool did_break_down = false; bool did_break_down = false;
@ -215,13 +216,13 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self,
did_break_down = true; did_break_down = true;
pending = false; pending = false;
for (uint32_t i = 0; i < pop.size; i++) for (t_u32 i = 0; i < pop.size; i++)
{ {
StackSlice slice = pop.contents[i]; StackSlice slice = pop.contents[i];
TSStateId state = ts_stack_state(self->stack, slice.version); t_state_id state = ts_stack_state(self->stack, slice.version);
Subtree parent = *array_front(&slice.subtrees); Subtree parent = *array_front(&slice.subtrees);
for (uint32_t j = 0, n = ts_subtree_child_count(parent); j < n; j++) for (t_u32 j = 0, n = ts_subtree_child_count(parent); j < n; j++)
{ {
Subtree child = ts_subtree_children(parent)[j]; Subtree child = ts_subtree_children(parent)[j];
pending = ts_subtree_child_count(child) > 0; pending = ts_subtree_child_count(child) > 0;
@ -241,7 +242,7 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self,
state); state);
} }
for (uint32_t j = 1; j < slice.subtrees.size; j++) for (t_u32 j = 1; j < slice.subtrees.size; j++)
{ {
Subtree tree = slice.subtrees.contents[j]; Subtree tree = slice.subtrees.contents[j];
ts_stack_push(self->stack, slice.version, tree, false, state); ts_stack_push(self->stack, slice.version, tree, false, state);
@ -258,8 +259,8 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self,
return did_break_down; return did_break_down;
} }
static void ts_parser__breakdown_lookahead(TSParser *self, Subtree *lookahead, static void ts_parser__breakdown_lookahead(t_parser *self, Subtree *lookahead,
TSStateId state, t_state_id state,
ReusableNode *reusable_node) ReusableNode *reusable_node)
{ {
bool did_descend = false; bool did_descend = false;
@ -281,8 +282,9 @@ static void ts_parser__breakdown_lookahead(TSParser *self, Subtree *lookahead,
} }
} }
static ErrorComparison ts_parser__compare_versions(TSParser *self, static t_error_comparison ts_parser__compare_versions(t_parser *self,
ErrorStatus a, ErrorStatus b) t_error_status a,
t_error_status b)
{ {
(void)self; (void)self;
if (!a.is_in_error && b.is_in_error) if (!a.is_in_error && b.is_in_error)
@ -340,14 +342,14 @@ static ErrorComparison ts_parser__compare_versions(TSParser *self,
return ErrorComparisonNone; return ErrorComparisonNone;
} }
static ErrorStatus ts_parser__version_status(TSParser *self, static t_error_status ts_parser__version_status(t_parser *self,
StackVersion version) StackVersion version)
{ {
unsigned cost = ts_stack_error_cost(self->stack, version); unsigned cost = ts_stack_error_cost(self->stack, version);
bool is_paused = ts_stack_is_paused(self->stack, version); bool is_paused = ts_stack_is_paused(self->stack, version);
if (is_paused) if (is_paused)
cost += ERROR_COST_PER_SKIPPED_TREE; cost += ERROR_COST_PER_SKIPPED_TREE;
return (ErrorStatus){ return (t_error_status){
.cost = cost, .cost = cost,
.node_count = ts_stack_node_count_since_error(self->stack, version), .node_count = ts_stack_node_count_since_error(self->stack, version),
.dynamic_precedence = ts_stack_dynamic_precedence(self->stack, version), .dynamic_precedence = ts_stack_dynamic_precedence(self->stack, version),
@ -355,7 +357,7 @@ static ErrorStatus ts_parser__version_status(TSParser *self,
is_paused || ts_stack_state(self->stack, version) == ERROR_STATE}; is_paused || ts_stack_state(self->stack, version) == ERROR_STATE};
} }
static bool ts_parser__better_version_exists(TSParser *self, static bool ts_parser__better_version_exists(t_parser *self,
StackVersion version, StackVersion version,
bool is_in_error, unsigned cost) bool is_in_error, unsigned cost)
{ {
@ -365,8 +367,8 @@ static bool ts_parser__better_version_exists(TSParser *self,
return true; return true;
} }
Length position = ts_stack_position(self->stack, version); Length position = ts_stack_position(self->stack, version);
ErrorStatus status = { t_error_status status = {
.cost = cost, .cost = cost,
.is_in_error = is_in_error, .is_in_error = is_in_error,
.dynamic_precedence = ts_stack_dynamic_precedence(self->stack, version), .dynamic_precedence = ts_stack_dynamic_precedence(self->stack, version),
@ -379,7 +381,7 @@ static bool ts_parser__better_version_exists(TSParser *self,
if (i == version || !ts_stack_is_active(self->stack, i) || if (i == version || !ts_stack_is_active(self->stack, i) ||
ts_stack_position(self->stack, i).bytes < position.bytes) ts_stack_position(self->stack, i).bytes < position.bytes)
continue; continue;
ErrorStatus status_i = ts_parser__version_status(self, i); t_error_status status_i = ts_parser__version_status(self, i);
switch (ts_parser__compare_versions(self, status, status_i)) switch (ts_parser__compare_versions(self, status, status_i))
{ {
case ErrorComparisonTakeRight: case ErrorComparisonTakeRight:
@ -396,19 +398,19 @@ static bool ts_parser__better_version_exists(TSParser *self,
return false; return false;
} }
static bool ts_parser__call_main_lex_fn(TSParser *self, TSLexMode lex_mode) static bool ts_parser__call_main_lex_fn(t_parser *self, t_lex_modes lex_mode)
{ {
(void)(lex_mode); (void)(lex_mode);
return self->language->lex_fn(&self->lexer.data, lex_mode.lex_state); return self->language->lex_fn(&self->lexer.data, lex_mode.lex_state);
} }
static bool ts_parser__call_keyword_lex_fn(TSParser *self, TSLexMode lex_mode) static bool ts_parser__call_keyword_lex_fn(t_parser *self, t_lex_modes lex_mode)
{ {
(void)(lex_mode); (void)(lex_mode);
return self->language->keyword_lex_fn(&self->lexer.data, 0); return self->language->keyword_lex_fn(&self->lexer.data, 0);
} }
static void ts_parser__external_scanner_create(TSParser *self) static void ts_parser__external_scanner_create(t_parser *self)
{ {
if (self->language && self->language->external_scanner.states) if (self->language && self->language->external_scanner.states)
{ {
@ -417,7 +419,7 @@ static void ts_parser__external_scanner_create(TSParser *self)
} }
} }
static void ts_parser__external_scanner_destroy(TSParser *self) static void ts_parser__external_scanner_destroy(t_parser *self)
{ {
if (self->language && self->external_scanner_payload && if (self->language && self->external_scanner_payload &&
self->language->external_scanner.destroy) self->language->external_scanner.destroy)
@ -428,17 +430,17 @@ static void ts_parser__external_scanner_destroy(TSParser *self)
self->external_scanner_payload = NULL; self->external_scanner_payload = NULL;
} }
static unsigned ts_parser__external_scanner_serialize(TSParser *self) static unsigned ts_parser__external_scanner_serialize(t_parser *self)
{ {
return self->language->external_scanner.serialize( return self->language->external_scanner.serialize(
self->external_scanner_payload, self->lexer.debug_buffer); self->external_scanner_payload, self->lexer.debug_buffer);
} }
static void ts_parser__external_scanner_deserialize(TSParser *self, static void ts_parser__external_scanner_deserialize(t_parser *self,
Subtree external_token) Subtree external_token)
{ {
const char *data = NULL; const char *data = NULL;
uint32_t length = 0; t_u32 length = 0;
if (external_token.ptr) if (external_token.ptr)
{ {
data = ts_external_scanner_state_data( data = ts_external_scanner_state_data(
@ -450,8 +452,8 @@ static void ts_parser__external_scanner_deserialize(TSParser *self,
data, length); data, length);
} }
static bool ts_parser__external_scanner_scan(TSParser *self, static bool ts_parser__external_scanner_scan(t_parser *self,
TSStateId external_lex_state) t_state_id external_lex_state)
{ {
const bool *valid_external_tokens = const bool *valid_external_tokens =
ts_language_enabled_external_tokens(self->language, external_lex_state); ts_language_enabled_external_tokens(self->language, external_lex_state);
@ -460,26 +462,26 @@ static bool ts_parser__external_scanner_scan(TSParser *self,
valid_external_tokens); valid_external_tokens);
} }
static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state, static bool ts_parser__can_reuse_first_leaf(t_parser *self, t_state_id state,
Subtree tree, Subtree tree,
TableEntry *table_entry) TableEntry *table_entry)
{ {
TSLexMode current_lex_mode = self->language->lex_modes[state]; t_lex_modes current_lex_mode = self->language->lex_modes[state];
TSSymbol leaf_symbol = ts_subtree_leaf_symbol(tree); t_symbol leaf_symbol = ts_subtree_leaf_symbol(tree);
TSStateId leaf_state = ts_subtree_leaf_parse_state(tree); t_state_id leaf_state = ts_subtree_leaf_parse_state(tree);
TSLexMode leaf_lex_mode = self->language->lex_modes[leaf_state]; t_lex_modes leaf_lex_mode = self->language->lex_modes[leaf_state];
// At the end of a non-terminal extra node, the lexer normally returns // At the end of a non-terminal extra node, the lexer normally returns
// NULL, which indicates that the parser should look for a reduce action // NULL, which indicates that the parser should look for a reduce action
// at symbol `0`. Avoid reusing tokens in this situation to ensure that // at symbol `0`. Avoid reusing tokens in this situation to ensure that
// the same thing happens when incrementally reparsing. // the same thing happens when incrementally reparsing.
if (current_lex_mode.lex_state == (uint16_t)(-1)) if (current_lex_mode.lex_state == (t_u16)(-1))
return false; return false;
// If the token was created in a state with the same set of lookaheads, it // If the token was created in a state with the same set of lookaheads, it
// is reusable. // is reusable.
if (table_entry->action_count > 0 && if (table_entry->action_count > 0 &&
memcmp(&leaf_lex_mode, &current_lex_mode, sizeof(TSLexMode)) == 0 && memcmp(&leaf_lex_mode, &current_lex_mode, sizeof(t_lex_modes)) == 0 &&
(leaf_symbol != self->language->keyword_capture_token || (leaf_symbol != self->language->keyword_capture_token ||
(!ts_subtree_is_keyword(tree) && (!ts_subtree_is_keyword(tree) &&
ts_subtree_parse_state(tree) == state))) ts_subtree_parse_state(tree) == state)))
@ -494,11 +496,11 @@ static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state,
return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable; return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable;
} }
static Subtree ts_parser__lex(TSParser *self, StackVersion version, static Subtree ts_parser__lex(t_parser *self, StackVersion version,
TSStateId parse_state) t_state_id parse_state)
{ {
TSLexMode lex_mode = self->language->lex_modes[parse_state]; t_lex_modes lex_mode = self->language->lex_modes[parse_state];
if (lex_mode.lex_state == (uint16_t)-1) if (lex_mode.lex_state == (t_u16)-1)
{ {
LOG("no_lookahead_after_non_terminal_extra"); LOG("no_lookahead_after_non_terminal_extra");
return NULL_SUBTREE; return NULL_SUBTREE;
@ -512,11 +514,11 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version,
bool error_mode = parse_state == ERROR_STATE; bool error_mode = parse_state == ERROR_STATE;
bool skipped_error = false; bool skipped_error = false;
bool called_get_column = false; bool called_get_column = false;
int32_t first_error_character = 0; t_i32 first_error_character = 0;
Length error_start_position = length_zero(); Length error_start_position = length_zero();
Length error_end_position = length_zero(); Length error_end_position = length_zero();
uint32_t lookahead_end_byte = 0; t_i32 lookahead_end_byte = 0;
uint32_t external_scanner_state_len = 0; t_i32 external_scanner_state_len = 0;
bool external_scanner_state_changed = false; bool external_scanner_state_changed = false;
ts_lexer_reset(&self->lexer, start_position); ts_lexer_reset(&self->lexer, start_position);
@ -626,7 +628,7 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version,
{ {
Length padding = length_sub(error_start_position, start_position); Length padding = length_sub(error_start_position, start_position);
Length size = length_sub(error_end_position, error_start_position); Length size = length_sub(error_end_position, error_start_position);
uint32_t lookahead_bytes = t_u32 lookahead_bytes =
lookahead_end_byte - error_end_position.bytes; lookahead_end_byte - error_end_position.bytes;
result = ts_subtree_new_error(&self->tree_pool, first_error_character, result = ts_subtree_new_error(&self->tree_pool, first_error_character,
padding, size, lookahead_bytes, padding, size, lookahead_bytes,
@ -635,12 +637,12 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version,
else else
{ {
bool is_keyword = false; bool is_keyword = false;
TSSymbol symbol = self->lexer.data.result_symbol; t_symbol symbol = self->lexer.data.result_symbol;
Length padding = Length padding =
length_sub(self->lexer.token_start_position, start_position); length_sub(self->lexer.token_start_position, start_position);
Length size = length_sub(self->lexer.token_end_position, Length size = length_sub(self->lexer.token_end_position,
self->lexer.token_start_position); self->lexer.token_start_position);
uint32_t lookahead_bytes = t_u32 lookahead_bytes =
lookahead_end_byte - self->lexer.token_end_position.bytes; lookahead_end_byte - self->lexer.token_end_position.bytes;
if (found_external_token) if (found_external_token)
@ -649,7 +651,7 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version,
} }
else if (symbol == self->language->keyword_capture_token && symbol != 0) else if (symbol == self->language->keyword_capture_token && symbol != 0)
{ {
uint32_t end_byte = self->lexer.token_end_position.bytes; t_u32 end_byte = self->lexer.token_end_position.bytes;
ts_lexer_reset(&self->lexer, self->lexer.token_start_position); ts_lexer_reset(&self->lexer, self->lexer.token_start_position);
ts_lexer_start(&self->lexer); ts_lexer_start(&self->lexer);
@ -671,11 +673,11 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version,
if (found_external_token) if (found_external_token)
{ {
MutableSubtree mut_result = ts_subtree_to_mut_unsafe(result); MutableSubtree mt_result = ts_subtree_to_mt_unsafe(result);
ts_external_scanner_state_init( ts_external_scanner_state_init(
&mut_result.ptr->external_scanner_state, &mt_result.ptr->external_scanner_state,
self->lexer.debug_buffer, external_scanner_state_len); self->lexer.debug_buffer, external_scanner_state_len);
mut_result.ptr->has_external_scanner_state_change = mt_result.ptr->has_external_scanner_state_change =
external_scanner_state_changed; external_scanner_state_changed;
} }
} }
@ -685,12 +687,12 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version,
return result; return result;
} }
static Subtree ts_parser__get_cached_token(TSParser *self, TSStateId state, static Subtree ts_parser__get_cached_token(t_parser *self, t_state_id state,
size_t position, size_t position,
Subtree last_external_token, Subtree last_external_token,
TableEntry *table_entry) TableEntry *table_entry)
{ {
TokenCache *cache = &self->token_cache; t_token_cache *cache = &self->token_cache;
if (cache->token.ptr && cache->byte_index == position && if (cache->token.ptr && cache->byte_index == position &&
ts_subtree_external_scanner_state_eq(cache->last_external_token, ts_subtree_external_scanner_state_eq(cache->last_external_token,
last_external_token)) last_external_token))
@ -707,11 +709,11 @@ static Subtree ts_parser__get_cached_token(TSParser *self, TSStateId state,
return NULL_SUBTREE; return NULL_SUBTREE;
} }
static void ts_parser__set_cached_token(TSParser *self, uint32_t byte_index, static void ts_parser__set_cached_token(t_parser *self, t_u32 byte_index,
Subtree last_external_token, Subtree last_external_token,
Subtree token) Subtree token)
{ {
TokenCache *cache = &self->token_cache; t_token_cache *cache = &self->token_cache;
if (token.ptr) if (token.ptr)
ts_subtree_retain(token); ts_subtree_retain(token);
if (last_external_token.ptr) if (last_external_token.ptr)
@ -725,16 +727,16 @@ static void ts_parser__set_cached_token(TSParser *self, uint32_t byte_index,
cache->last_external_token = last_external_token; cache->last_external_token = last_external_token;
} }
static Subtree ts_parser__reuse_node(TSParser *self, StackVersion version, static Subtree ts_parser__reuse_node(t_parser *self, StackVersion version,
TSStateId *state, uint32_t position, t_state_id *state, t_u32 position,
Subtree last_external_token, Subtree last_external_token,
TableEntry *table_entry) TableEntry *table_entry)
{ {
Subtree result; Subtree result;
while ((result = reusable_node_tree(&self->reusable_node)).ptr) while ((result = reusable_node_tree(&self->reusable_node)).ptr)
{ {
uint32_t byte_offset = reusable_node_byte_offset(&self->reusable_node); t_u32 byte_offset = reusable_node_byte_offset(&self->reusable_node);
uint32_t end_byte_offset = byte_offset + ts_subtree_total_bytes(result); t_u32 end_byte_offset = byte_offset + ts_subtree_total_bytes(result);
// Do not reuse an EOF node if the included ranges array has changes // Do not reuse an EOF node if the included ranges array has changes
// later on in the file. // later on in the file.
@ -797,7 +799,7 @@ static Subtree ts_parser__reuse_node(TSParser *self, StackVersion version,
continue; continue;
} }
TSSymbol leaf_symbol = ts_subtree_leaf_symbol(result); t_symbol leaf_symbol = ts_subtree_leaf_symbol(result);
ts_language_table_entry(self->language, *state, leaf_symbol, ts_language_table_entry(self->language, *state, leaf_symbol,
table_entry); table_entry);
if (!ts_parser__can_reuse_first_leaf(self, *state, result, table_entry)) if (!ts_parser__can_reuse_first_leaf(self, *state, result, table_entry))
@ -821,7 +823,7 @@ static Subtree ts_parser__reuse_node(TSParser *self, StackVersion version,
// The decision is based on the trees' error costs (if any), their dynamic // The decision is based on the trees' error costs (if any), their dynamic
// precedence, and finally, as a default, by a recursive comparison of the // precedence, and finally, as a default, by a recursive comparison of the
// trees' symbols. // trees' symbols.
static bool ts_parser__select_tree(TSParser *self, Subtree left, Subtree right) static bool ts_parser__select_parse_tree(t_parser *self, Subtree left, Subtree right)
{ {
if (!left.ptr) if (!left.ptr)
return true; return true;
@ -886,7 +888,7 @@ static bool ts_parser__select_tree(TSParser *self, Subtree left, Subtree right)
// Determine if a given tree's children should be replaced by an alternative // Determine if a given tree's children should be replaced by an alternative
// array of children. // array of children.
static bool ts_parser__select_children(TSParser *self, Subtree left, static bool ts_parser__select_children(t_parser *self, Subtree left,
const SubtreeArray *children) const SubtreeArray *children)
{ {
array_assign(&self->scratch_trees, children); array_assign(&self->scratch_trees, children);
@ -898,12 +900,12 @@ static bool ts_parser__select_children(TSParser *self, Subtree left,
MutableSubtree scratch_tree = ts_subtree_new_node( MutableSubtree scratch_tree = ts_subtree_new_node(
ts_subtree_symbol(left), &self->scratch_trees, 0, self->language); ts_subtree_symbol(left), &self->scratch_trees, 0, self->language);
return ts_parser__select_tree(self, left, return ts_parser__select_parse_tree(self, left,
ts_subtree_from_mut(scratch_tree)); ts_subtree_from_mut(scratch_tree));
} }
static void ts_parser__shift(TSParser *self, StackVersion version, static void ts_parser__shift(t_parser *self, StackVersion version,
TSStateId state, Subtree lookahead, bool extra) t_state_id state, Subtree lookahead, bool extra)
{ {
bool is_leaf = ts_subtree_child_count(lookahead) == 0; bool is_leaf = ts_subtree_child_count(lookahead) == 0;
Subtree subtree_to_push = lookahead; Subtree subtree_to_push = lookahead;
@ -924,13 +926,13 @@ static void ts_parser__shift(TSParser *self, StackVersion version,
} }
} }
static StackVersion ts_parser__reduce(TSParser *self, StackVersion version, static StackVersion ts_parser__reduce(t_parser *self, StackVersion version,
TSSymbol symbol, uint32_t count, t_symbol symbol, t_u32 count,
int dynamic_precedence, int dynamic_precedence,
uint16_t production_id, bool is_fragile, t_u16 production_id, bool is_fragile,
bool end_of_non_terminal_extra) bool end_of_non_terminal_extra)
{ {
uint32_t initial_version_count = ts_stack_version_count(self->stack); t_u32 initial_version_count = ts_stack_version_count(self->stack);
// Pop the given number of nodes from the given version of the parse stack. // Pop the given number of nodes from the given version of the parse stack.
// If stack versions have previously merged, then there may be more than one // If stack versions have previously merged, then there may be more than one
@ -938,8 +940,8 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version,
// contain the popped children, and push it onto the stack in place of the // contain the popped children, and push it onto the stack in place of the
// children. // children.
StackSliceArray pop = ts_stack_pop_count(self->stack, version, count); StackSliceArray pop = ts_stack_pop_count(self->stack, version, count);
uint32_t removed_version_count = 0; t_u32 removed_version_count = 0;
for (uint32_t i = 0; i < pop.size; i++) for (t_u32 i = 0; i < pop.size; i++)
{ {
StackSlice slice = pop.contents[i]; StackSlice slice = pop.contents[i];
StackVersion slice_version = slice.version - removed_version_count; StackVersion slice_version = slice.version - removed_version_count;
@ -1007,8 +1009,8 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version,
} }
} }
TSStateId state = ts_stack_state(self->stack, slice_version); t_state_id state = ts_stack_state(self->stack, slice_version);
TSStateId next_state = t_state_id next_state =
ts_language_next_state(self->language, state, symbol); ts_language_next_state(self->language, state, symbol);
if (end_of_non_terminal_extra && next_state == state) if (end_of_non_terminal_extra && next_state == state)
{ {
@ -1030,7 +1032,7 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version,
// were previously on top of the stack. // were previously on top of the stack.
ts_stack_push(self->stack, slice_version, ts_subtree_from_mut(parent), ts_stack_push(self->stack, slice_version, ts_subtree_from_mut(parent),
false, next_state); false, next_state);
for (uint32_t j = 0; j < self->trailing_extras.size; j++) for (t_u32 j = 0; j < self->trailing_extras.size; j++)
{ {
ts_stack_push(self->stack, slice_version, ts_stack_push(self->stack, slice_version,
self->trailing_extras.contents[j], false, next_state); self->trailing_extras.contents[j], false, next_state);
@ -1054,27 +1056,27 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version,
: STACK_VERSION_NONE; : STACK_VERSION_NONE;
} }
static void ts_parser__accept(TSParser *self, StackVersion version, static void ts_parser__accept(t_parser *self, StackVersion version,
Subtree lookahead) Subtree lookahead)
{ {
assert(ts_subtree_is_eof(lookahead)); assert(ts_subtree_is_eof(lookahead));
ts_stack_push(self->stack, version, lookahead, false, 1); ts_stack_push(self->stack, version, lookahead, false, 1);
StackSliceArray pop = ts_stack_pop_all(self->stack, version); StackSliceArray pop = ts_stack_pop_all(self->stack, version);
for (uint32_t i = 0; i < pop.size; i++) for (t_u32 i = 0; i < pop.size; i++)
{ {
SubtreeArray trees = pop.contents[i].subtrees; SubtreeArray trees = pop.contents[i].subtrees;
Subtree root = NULL_SUBTREE; Subtree root = NULL_SUBTREE;
for (uint32_t j = trees.size - 1; j + 1 > 0; j--) for (t_u32 j = trees.size - 1; j + 1 > 0; j--)
{ {
Subtree tree = trees.contents[j]; Subtree tree = trees.contents[j];
if (!ts_subtree_extra(tree)) if (!ts_subtree_extra(tree))
{ {
assert(!tree.data.is_inline); assert(!tree.data.is_inline);
uint32_t child_count = ts_subtree_child_count(tree); t_u32 child_count = ts_subtree_child_count(tree);
const Subtree *children = ts_subtree_children(tree); const Subtree *children = ts_subtree_children(tree);
for (uint32_t k = 0; k < child_count; k++) for (t_u32 k = 0; k < child_count; k++)
{ {
ts_subtree_retain(children[k]); ts_subtree_retain(children[k]);
} }
@ -1092,7 +1094,7 @@ static void ts_parser__accept(TSParser *self, StackVersion version,
if (self->finished_tree.ptr) if (self->finished_tree.ptr)
{ {
if (ts_parser__select_tree(self, self->finished_tree, root)) if (ts_parser__select_parse_tree(self, self->finished_tree, root))
{ {
ts_subtree_release(&self->tree_pool, self->finished_tree); ts_subtree_release(&self->tree_pool, self->finished_tree);
self->finished_tree = root; self->finished_tree = root;
@ -1113,15 +1115,15 @@ static void ts_parser__accept(TSParser *self, StackVersion version,
} }
static bool ts_parser__do_all_potential_reductions( static bool ts_parser__do_all_potential_reductions(
TSParser *self, StackVersion starting_version, TSSymbol lookahead_symbol) t_parser *self, StackVersion starting_version, t_symbol lookahead_symbol)
{ {
uint32_t initial_version_count = ts_stack_version_count(self->stack); t_u32 initial_version_count = ts_stack_version_count(self->stack);
bool can_shift_lookahead_symbol = false; bool can_shift_lookahead_symbol = false;
StackVersion version = starting_version; StackVersion version = starting_version;
for (unsigned i = 0; true; i++) for (unsigned i = 0; true; i++)
{ {
uint32_t version_count = ts_stack_version_count(self->stack); t_u32 version_count = ts_stack_version_count(self->stack);
if (version >= version_count) if (version >= version_count)
break; break;
@ -1137,11 +1139,11 @@ static bool ts_parser__do_all_potential_reductions(
if (merged) if (merged)
continue; continue;
TSStateId state = ts_stack_state(self->stack, version); t_state_id state = ts_stack_state(self->stack, version);
bool has_shift_action = false; bool has_shift_action = false;
array_clear(&self->reduce_actions); array_clear(&self->reduce_actions);
TSSymbol first_symbol, end_symbol; t_symbol first_symbol, end_symbol;
if (lookahead_symbol != 0) if (lookahead_symbol != 0)
{ {
first_symbol = lookahead_symbol; first_symbol = lookahead_symbol;
@ -1153,21 +1155,21 @@ static bool ts_parser__do_all_potential_reductions(
end_symbol = self->language->token_count; end_symbol = self->language->token_count;
} }
for (TSSymbol symbol = first_symbol; symbol < end_symbol; symbol++) for (t_symbol symbol = first_symbol; symbol < end_symbol; symbol++)
{ {
TableEntry entry; TableEntry entry;
ts_language_table_entry(self->language, state, symbol, &entry); ts_language_table_entry(self->language, state, symbol, &entry);
for (uint32_t j = 0; j < entry.action_count; j++) for (t_u32 j = 0; j < entry.action_count; j++)
{ {
TSParseAction action = entry.actions[j]; t_parse_actions action = entry.actions[j];
switch (action.type) switch (action.type)
{ {
case TSParseActionTypeShift: case ActionTypeShift:
case TSParseActionTypeRecover: case ActionTypeRecover:
if (!action.shift.extra && !action.shift.repetition) if (!action.shift.extra && !action.shift.repetition)
has_shift_action = true; has_shift_action = true;
break; break;
case TSParseActionTypeReduce: case ActionTypeReduce:
if (action.reduce.child_count > 0) if (action.reduce.child_count > 0)
ts_reduce_action_set_add( ts_reduce_action_set_add(
&self->reduce_actions, &self->reduce_actions,
@ -1186,7 +1188,7 @@ static bool ts_parser__do_all_potential_reductions(
} }
StackVersion reduction_version = STACK_VERSION_NONE; StackVersion reduction_version = STACK_VERSION_NONE;
for (uint32_t j = 0; j < self->reduce_actions.size; j++) for (t_u32 j = 0; j < self->reduce_actions.size; j++)
{ {
ReduceAction action = self->reduce_actions.contents[j]; ReduceAction action = self->reduce_actions.contents[j];
@ -1223,8 +1225,8 @@ static bool ts_parser__do_all_potential_reductions(
return can_shift_lookahead_symbol; return can_shift_lookahead_symbol;
} }
static bool ts_parser__recover_to_state(TSParser *self, StackVersion version, static bool ts_parser__recover_to_state(t_parser *self, StackVersion version,
unsigned depth, TSStateId goal_state) unsigned depth, t_state_id goal_state)
{ {
StackSliceArray pop = ts_stack_pop_count(self->stack, version, depth); StackSliceArray pop = ts_stack_pop_count(self->stack, version, depth);
StackVersion previous_version = STACK_VERSION_NONE; StackVersion previous_version = STACK_VERSION_NONE;
@ -1254,7 +1256,7 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version,
{ {
assert(error_trees.size == 1); assert(error_trees.size == 1);
Subtree error_tree = error_trees.contents[0]; Subtree error_tree = error_trees.contents[0];
uint32_t error_child_count = ts_subtree_child_count(error_tree); t_u32 error_child_count = ts_subtree_child_count(error_tree);
if (error_child_count > 0) if (error_child_count > 0)
{ {
array_splice(&slice.subtrees, 0, 0, error_child_count, array_splice(&slice.subtrees, 0, 0, error_child_count,
@ -1293,7 +1295,7 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version,
return previous_version != STACK_VERSION_NONE; return previous_version != STACK_VERSION_NONE;
} }
static void ts_parser__recover(TSParser *self, StackVersion version, static void ts_parser__recover(t_parser *self, StackVersion version,
Subtree lookahead) Subtree lookahead)
{ {
bool did_recover = false; bool did_recover = false;
@ -1439,9 +1441,9 @@ static void ts_parser__recover(TSParser *self, StackVersion version,
// If the current lookahead token is an extra token, mark it as extra. This // If the current lookahead token is an extra token, mark it as extra. This
// means it won't be counted in error cost calculations. // means it won't be counted in error cost calculations.
unsigned n; unsigned n;
const TSParseAction *actions = ts_language_actions( const t_parse_actions *actions = ts_language_actions(
self->language, 1, ts_subtree_symbol(lookahead), &n); self->language, 1, ts_subtree_symbol(lookahead), &n);
if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && if (n > 0 && actions[n - 1].type == ActionTypeShift &&
actions[n - 1].shift.extra) actions[n - 1].shift.extra)
{ {
MutableSubtree mutable_lookahead = MutableSubtree mutable_lookahead =
@ -1503,16 +1505,16 @@ static void ts_parser__recover(TSParser *self, StackVersion version,
} }
} }
static void ts_parser__handle_error(TSParser *self, StackVersion version, static void ts_parser__handle_error(t_parser *self, StackVersion version,
Subtree lookahead) Subtree lookahead)
{ {
uint32_t previous_version_count = ts_stack_version_count(self->stack); t_u32 previous_version_count = ts_stack_version_count(self->stack);
// Perform any reductions that can happen in this state, regardless of the // Perform any reductions that can happen in this state, regardless of the
// lookahead. After skipping one or more invalid tokens, the parser might // lookahead. After skipping one or more invalid tokens, the parser might
// find a token that would have allowed a reduction to take place. // find a token that would have allowed a reduction to take place.
ts_parser__do_all_potential_reductions(self, version, 0); ts_parser__do_all_potential_reductions(self, version, 0);
uint32_t version_count = ts_stack_version_count(self->stack); t_u32 version_count = ts_stack_version_count(self->stack);
Length position = ts_stack_position(self->stack, version); Length position = ts_stack_position(self->stack, version);
// Push a discontinuity onto the stack. Merge all of the stack versions that // Push a discontinuity onto the stack. Merge all of the stack versions that
@ -1522,12 +1524,12 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version,
{ {
if (!did_insert_missing_token) if (!did_insert_missing_token)
{ {
TSStateId state = ts_stack_state(self->stack, v); t_state_id state = ts_stack_state(self->stack, v);
for (TSSymbol missing_symbol = 1; for (t_symbol missing_symbol = 1;
missing_symbol < (uint16_t)self->language->token_count; missing_symbol < (t_u16)self->language->token_count;
missing_symbol++) missing_symbol++)
{ {
TSStateId state_after_missing_symbol = ts_language_next_state( t_state_id state_after_missing_symbol = ts_language_next_state(
self->language, state, missing_symbol); self->language, state, missing_symbol);
if (state_after_missing_symbol == 0 || if (state_after_missing_symbol == 0 ||
state_after_missing_symbol == state) state_after_missing_symbol == state)
@ -1547,7 +1549,7 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version,
ts_lexer_mark_end(&self->lexer); ts_lexer_mark_end(&self->lexer);
Length padding = Length padding =
length_sub(self->lexer.token_end_position, position); length_sub(self->lexer.token_end_position, position);
uint32_t lookahead_bytes = t_u32 lookahead_bytes =
ts_subtree_total_bytes(lookahead) + ts_subtree_total_bytes(lookahead) +
ts_subtree_lookahead_bytes(lookahead); ts_subtree_lookahead_bytes(lookahead);
@ -1604,11 +1606,11 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version,
LOG_STACK(); LOG_STACK();
} }
static bool ts_parser__advance(TSParser *self, StackVersion version, static bool ts_parser__advance(t_parser *self, StackVersion version,
bool allow_node_reuse) bool allow_node_reuse)
{ {
TSStateId state = ts_stack_state(self->stack, version); t_state_id state = ts_stack_state(self->stack, version);
uint32_t position = ts_stack_position(self->stack, version).bytes; t_u32 position = ts_stack_position(self->stack, version).bytes;
Subtree last_external_token = Subtree last_external_token =
ts_stack_last_external_token(self->stack, version); ts_stack_last_external_token(self->stack, version);
@ -1671,16 +1673,16 @@ static bool ts_parser__advance(TSParser *self, StackVersion version,
// version, whereas SHIFT actions update the existing stack version // version, whereas SHIFT actions update the existing stack version
// and terminate this loop. // and terminate this loop.
StackVersion last_reduction_version = STACK_VERSION_NONE; StackVersion last_reduction_version = STACK_VERSION_NONE;
for (uint32_t i = 0; i < table_entry.action_count; i++) for (t_u32 i = 0; i < table_entry.action_count; i++)
{ {
TSParseAction action = table_entry.actions[i]; t_parse_actions action = table_entry.actions[i];
switch (action.type) switch (action.type)
{ {
case TSParseActionTypeShift: { case ActionTypeShift: {
if (action.shift.repetition) if (action.shift.repetition)
break; break;
TSStateId next_state; t_state_id next_state;
if (action.shift.extra) if (action.shift.extra)
{ {
next_state = state; next_state = state;
@ -1707,7 +1709,7 @@ static bool ts_parser__advance(TSParser *self, StackVersion version,
return true; return true;
} }
case TSParseActionTypeReduce: { case ActionTypeReduce: {
bool is_fragile = table_entry.action_count > 1; bool is_fragile = table_entry.action_count > 1;
bool end_of_non_terminal_extra = lookahead.ptr == NULL; bool end_of_non_terminal_extra = lookahead.ptr == NULL;
LOG("reduce sym:%s, child_count:%u", LOG("reduce sym:%s, child_count:%u",
@ -1724,13 +1726,13 @@ static bool ts_parser__advance(TSParser *self, StackVersion version,
break; break;
} }
case TSParseActionTypeAccept: { case ActionTypeAccept: {
LOG("accept"); LOG("accept");
ts_parser__accept(self, version, lookahead); ts_parser__accept(self, version, lookahead);
return true; return true;
} }
case TSParseActionTypeRecover: { case ActionTypeRecover: {
if (ts_subtree_child_count(lookahead) > 0) if (ts_subtree_child_count(lookahead) > 0)
{ {
ts_parser__breakdown_lookahead( ts_parser__breakdown_lookahead(
@ -1843,7 +1845,7 @@ static bool ts_parser__advance(TSParser *self, StackVersion version,
} }
} }
static unsigned ts_parser__condense_stack(TSParser *self) static unsigned ts_parser__condense_stack(t_parser *self)
{ {
bool made_changes = false; bool made_changes = false;
unsigned min_error_cost = UINT_MAX; unsigned min_error_cost = UINT_MAX;
@ -1859,7 +1861,7 @@ static unsigned ts_parser__condense_stack(TSParser *self)
// Keep track of the minimum error cost of any stack version so // Keep track of the minimum error cost of any stack version so
// that it can be returned. // that it can be returned.
ErrorStatus status_i = ts_parser__version_status(self, i); t_error_status status_i = ts_parser__version_status(self, i);
if (!status_i.is_in_error && status_i.cost < min_error_cost) if (!status_i.is_in_error && status_i.cost < min_error_cost)
{ {
min_error_cost = status_i.cost; min_error_cost = status_i.cost;
@ -1870,7 +1872,7 @@ static unsigned ts_parser__condense_stack(TSParser *self)
// are ordered from most promising to least promising. // are ordered from most promising to least promising.
for (StackVersion j = 0; j < i; j++) for (StackVersion j = 0; j < i; j++)
{ {
ErrorStatus status_j = ts_parser__version_status(self, j); t_error_status status_j = ts_parser__version_status(self, j);
switch (ts_parser__compare_versions(self, status_j, status_i)) switch (ts_parser__compare_versions(self, status_j, status_i))
{ {
@ -1965,7 +1967,7 @@ static unsigned ts_parser__condense_stack(TSParser *self)
return min_error_cost; return min_error_cost;
} }
static bool ts_parser_has_outstanding_parse(TSParser *self) static bool ts_parser_has_outstanding_parse(t_parser *self)
{ {
return (self->external_scanner_payload || return (self->external_scanner_payload ||
ts_stack_state(self->stack, 0) != 1 || ts_stack_state(self->stack, 0) != 1 ||
@ -1974,9 +1976,9 @@ static bool ts_parser_has_outstanding_parse(TSParser *self)
// Parser - Public // Parser - Public
TSParser *ts_parser_new(void) t_parser *ts_parser_new(void)
{ {
TSParser *self = calloc(1, sizeof(TSParser)); t_parser *self = calloc(1, sizeof(t_parser));
ts_lexer_init(&self->lexer); ts_lexer_init(&self->lexer);
array_init(&self->reduce_actions); array_init(&self->reduce_actions);
array_reserve(&self->reduce_actions, 4); array_reserve(&self->reduce_actions, 4);
@ -1986,7 +1988,7 @@ TSParser *ts_parser_new(void)
self->reusable_node = reusable_node_new(); self->reusable_node = reusable_node_new();
self->dot_graph_file = NULL; self->dot_graph_file = NULL;
self->cancellation_flag = NULL; self->cancellation_flag = NULL;
self->timeout_duration = 0; self->timeot_duration = 0;
self->language = NULL; self->language = NULL;
self->has_scanner_error = false; self->has_scanner_error = false;
self->external_scanner_payload = NULL; self->external_scanner_payload = NULL;
@ -1999,7 +2001,7 @@ TSParser *ts_parser_new(void)
return self; return self;
} }
void ts_parser_delete(TSParser *self) void ts_parser_delete(t_parser *self)
{ {
if (!self) if (!self)
return; return;
@ -2029,12 +2031,12 @@ void ts_parser_delete(TSParser *self)
free(self); free(self);
} }
const TSLanguage *ts_parser_language(const TSParser *self) const t_language *ts_parser_language(const t_parser *self)
{ {
return self->language; return self->language;
} }
bool ts_parser_set_language(TSParser *self, const TSLanguage *language) bool ts_parser_set_language(t_parser *self, const t_language *language)
{ {
ts_parser_reset(self); ts_parser_reset(self);
ts_language_delete(self->language); ts_language_delete(self->language);
@ -2051,17 +2053,17 @@ bool ts_parser_set_language(TSParser *self, const TSLanguage *language)
return true; return true;
} }
TSLogger ts_parser_logger(const TSParser *self) TSLogger ts_parser_logger(const t_parser *self)
{ {
return self->lexer.logger; return self->lexer.logger;
} }
void ts_parser_set_logger(TSParser *self, TSLogger logger) void ts_parser_set_logger(t_parser *self, TSLogger logger)
{ {
self->lexer.logger = logger; self->lexer.logger = logger;
} }
void ts_parser_print_dot_graphs(TSParser *self, int fd) void ts_parser_print_dot_graphs(t_parser *self, int fd)
{ {
if (self->dot_graph_file) if (self->dot_graph_file)
{ {
@ -2082,41 +2084,41 @@ void ts_parser_print_dot_graphs(TSParser *self, int fd)
} }
} }
const size_t *ts_parser_cancellation_flag(const TSParser *self) const size_t *ts_parser_cancellation_flag(const t_parser *self)
{ {
return (const size_t *)self->cancellation_flag; return (const size_t *)self->cancellation_flag;
} }
void ts_parser_set_cancellation_flag(TSParser *self, const size_t *flag) void ts_parser_set_cancellation_flag(t_parser *self, const size_t *flag)
{ {
self->cancellation_flag = (const volatile size_t *)flag; self->cancellation_flag = (const volatile size_t *)flag;
} }
uint64_t ts_parser_timeout_micros(const TSParser *self) t_u64 ts_parser_timeot_micros(const t_parser *self)
{ {
(void)(self); (void)(self);
return 0; return 0;
} }
void ts_parser_set_timeout_micros(TSParser *self, uint64_t timeout_micros) void ts_parser_set_timeot_micros(t_parser *self, t_u64 timeot_micros)
{ {
(void)(timeout_micros); (void)(timeot_micros);
self->timeout_duration = 0; self->timeot_duration = 0;
} }
bool ts_parser_set_included_ranges(TSParser *self, const t_parser_range *ranges, bool ts_parser_set_included_ranges(t_parser *self, const t_parser_range *ranges,
uint32_t count) t_u32 count)
{ {
return ts_lexer_set_included_ranges(&self->lexer, ranges, count); return ts_lexer_set_included_ranges(&self->lexer, ranges, count);
} }
const t_parser_range *ts_parser_included_ranges(const TSParser *self, const t_parser_range *ts_parser_included_ranges(const t_parser *self,
uint32_t *count) t_u32 *count)
{ {
return ts_lexer_included_ranges(&self->lexer, count); return ts_lexer_included_ranges(&self->lexer, count);
} }
void ts_parser_reset(TSParser *self) void ts_parser_reset(t_parser *self)
{ {
ts_parser__external_scanner_destroy(self); ts_parser__external_scanner_destroy(self);
@ -2139,9 +2141,9 @@ void ts_parser_reset(TSParser *self)
self->has_scanner_error = false; self->has_scanner_error = false;
} }
TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) t_parse_tree *ts_parser_parse(t_parser *self, const t_parse_tree *old_tree, TSInput input)
{ {
TSTree *result = NULL; t_parse_tree *result = NULL;
old_tree = NULL; old_tree = NULL;
(void)(old_tree); (void)(old_tree);
if (!self->language || !input.read) if (!self->language || !input.read)
@ -2167,7 +2169,7 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input)
self->operation_count = 0; self->operation_count = 0;
uint32_t position = 0, last_position = 0, version_count = 0; t_u32 position = 0, last_position = 0, version_count = 0;
do do
{ {
for (StackVersion version = 0; for (StackVersion version = 0;
@ -2252,22 +2254,22 @@ exit:
return result; return result;
} }
TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree, t_parse_tree *ts_parser_parse_string(t_parser *self, const t_parse_tree *old_tree,
const char *string, uint32_t length) const char *string, t_u32 length)
{ {
return ts_parser_parse_string_encoding(self, old_tree, string, length, return ts_parser_parse_string_encoding(self, old_tree, string, length,
TSInputEncodingUTF8); TSInputEncodingUTF8);
} }
TSTree *ts_parser_parse_string_encoding(TSParser *self, const TSTree *old_tree, t_parse_tree *ts_parser_parse_string_encoding(t_parser *self, const t_parse_tree *old_tree,
const char *string, uint32_t length, const char *string, t_u32 length,
TSInputEncoding encoding) TSInputEncoding encoding)
{ {
TSStringInput input = {string, length}; t_string_input input = {string, length};
return ts_parser_parse(self, old_tree, return ts_parser_parse(self, old_tree,
(TSInput){ (TSInput){
&input, &input,
ts_string_input_read, ts_string_inpt_read,
encoding, encoding,
}); });
} }

265
parser/src/parser.h
View file

@ -1,265 +0,0 @@
#ifndef TREE_SITTER_PARSER_H_
#define TREE_SITTER_PARSER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#define ts_builtin_sym_error ((TSSymbol)-1)
#define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
#ifndef TREE_SITTER_API_H_
typedef uint16_t TSStateId;
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
#endif
typedef struct {
TSFieldId field_id;
uint8_t child_index;
bool inherited;
} TSFieldMapEntry;
typedef struct {
uint16_t index;
uint16_t length;
} TSFieldMapSlice;
typedef struct {
bool visible;
bool named;
bool supertype;
} TSSymbolMetadata;
typedef struct TSLexer TSLexer;
struct TSLexer {
int32_t lookahead;
TSSymbol result_symbol;
void (*advance)(TSLexer *, bool);
void (*mark_end)(TSLexer *);
uint32_t (*get_column)(TSLexer *);
bool (*is_at_included_range_start)(const TSLexer *);
bool (*eof)(const TSLexer *);
};
typedef enum {
TSParseActionTypeShift,
TSParseActionTypeReduce,
TSParseActionTypeAccept,
TSParseActionTypeRecover,
} TSParseActionType;
typedef union {
struct {
uint8_t type;
TSStateId state;
bool extra;
bool repetition;
} shift;
struct {
uint8_t type;
uint8_t child_count;
TSSymbol symbol;
int16_t dynamic_precedence;
uint16_t production_id;
} reduce;
uint8_t type;
} TSParseAction;
typedef struct {
uint16_t lex_state;
uint16_t external_lex_state;
} TSLexMode;
typedef union {
TSParseAction action;
struct {
uint8_t count;
bool reusable;
} entry;
} TSParseActionEntry;
typedef struct {
int32_t start;
int32_t end;
} TSCharacterRange;
struct TSLanguage {
uint32_t version;
uint32_t symbol_count;
uint32_t alias_count;
uint32_t token_count;
uint32_t external_token_count;
uint32_t state_count;
uint32_t large_state_count;
uint32_t production_id_count;
uint32_t field_count;
uint16_t max_alias_sequence_length;
const uint16_t *parse_table;
const uint16_t *small_parse_table;
const uint32_t *small_parse_table_map;
const TSParseActionEntry *parse_actions;
const char * const *symbol_names;
const char * const *field_names;
const TSFieldMapSlice *field_map_slices;
const TSFieldMapEntry *field_map_entries;
const TSSymbolMetadata *symbol_metadata;
const TSSymbol *public_symbol_map;
const uint16_t *alias_map;
const TSSymbol *alias_sequences;
const TSLexMode *lex_modes;
bool (*lex_fn)(TSLexer *, TSStateId);
bool (*keyword_lex_fn)(TSLexer *, TSStateId);
TSSymbol keyword_capture_token;
struct {
const bool *states;
const TSSymbol *symbol_map;
void *(*create)(void);
void (*destroy)(void *);
bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
unsigned (*serialize)(void *, char *);
void (*deserialize)(void *, const char *, unsigned);
} external_scanner;
const TSStateId *primary_state_ids;
};
static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
uint32_t index = 0;
uint32_t size = len - index;
while (size > 1) {
uint32_t half_size = size / 2;
uint32_t mid_index = index + half_size;
TSCharacterRange *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end) {
return true;
} else if (lookahead > range->end) {
index = mid_index;
}
size -= half_size;
}
TSCharacterRange *range = &ranges[index];
return (lookahead >= range->start && lookahead <= range->end);
}
/*
* Lexer Macros
*/
#ifdef _MSC_VER
#define UNUSED __pragma(warning(suppress : 4101))
#else
#define UNUSED __attribute__((unused))
#endif
#define START_LEXER() \
bool result = false; \
bool skip = false; \
UNUSED \
bool eof = false; \
int32_t lookahead; \
goto start; \
next_state: \
lexer->advance(lexer, skip); \
start: \
skip = false; \
lookahead = lexer->lookahead;
#define ADVANCE(state_value) \
{ \
state = state_value; \
goto next_state; \
}
#define ADVANCE_MAP(...) \
{ \
static const uint16_t map[] = { __VA_ARGS__ }; \
for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \
if (map[i] == lookahead) { \
state = map[i + 1]; \
goto next_state; \
} \
} \
}
#define SKIP(state_value) \
{ \
skip = true; \
state = state_value; \
goto next_state; \
}
#define ACCEPT_TOKEN(symbol_value) \
result = true; \
lexer->result_symbol = symbol_value; \
lexer->mark_end(lexer);
#define END_STATE() return result;
/*
* Parse Table Macros
*/
#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
#define STATE(id) id
#define ACTIONS(id) id
#define SHIFT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = (state_value) \
} \
}}
#define SHIFT_REPEAT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = (state_value), \
.repetition = true \
} \
}}
#define SHIFT_EXTRA() \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.extra = true \
} \
}}
#define REDUCE(symbol_name, children, precedence, prod_id) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_name, \
.child_count = children, \
.dynamic_precedence = precedence, \
.production_id = prod_id \
}, \
}}
#define RECOVER() \
{{ \
.type = TSParseActionTypeRecover \
}}
#define ACCEPT_INPUT() \
{{ \
.type = TSParseActionTypeAccept \
}}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_PARSER_H_

6
parser/src/reduce_action.h
View file

@ -9,8 +9,8 @@ extern "C" {
#include "parser/api.h" #include "parser/api.h"
typedef struct { typedef struct {
uint32_t count; t_u32 count;
TSSymbol symbol; t_symbol symbol;
int dynamic_precedence; int dynamic_precedence;
unsigned short production_id; unsigned short production_id;
} ReduceAction; } ReduceAction;
@ -19,7 +19,7 @@ typedef Array(ReduceAction) ReduceActionSet;
static inline void ts_reduce_action_set_add(ReduceActionSet *self, static inline void ts_reduce_action_set_add(ReduceActionSet *self,
ReduceAction new_action) { ReduceAction new_action) {
for (uint32_t i = 0; i < self->size; i++) { for (t_u32 i = 0; i < self->size; i++) {
ReduceAction action = self->contents[i]; ReduceAction action = self->contents[i];
if (action.symbol == new_action.symbol && action.count == new_action.count) if (action.symbol == new_action.symbol && action.count == new_action.count)
return; return;

10
parser/src/reusable_node.h
View file

@ -2,8 +2,8 @@
typedef struct { typedef struct {
Subtree tree; Subtree tree;
uint32_t child_index; t_u32 child_index;
uint32_t byte_offset; t_u32 byte_offset;
} StackEntry; } StackEntry;
typedef struct { typedef struct {
@ -26,7 +26,7 @@ static inline Subtree reusable_node_tree(ReusableNode *self) {
: NULL_SUBTREE; : NULL_SUBTREE;
} }
static inline uint32_t reusable_node_byte_offset(ReusableNode *self) { static inline t_u32 reusable_node_byte_offset(ReusableNode *self) {
return self->stack.size > 0 return self->stack.size > 0
? self->stack.contents[self->stack.size - 1].byte_offset ? self->stack.contents[self->stack.size - 1].byte_offset
: UINT32_MAX; : UINT32_MAX;
@ -38,13 +38,13 @@ static inline void reusable_node_delete(ReusableNode *self) {
static inline void reusable_node_advance(ReusableNode *self) { static inline void reusable_node_advance(ReusableNode *self) {
StackEntry last_entry = *array_back(&self->stack); StackEntry last_entry = *array_back(&self->stack);
uint32_t byte_offset = last_entry.byte_offset + ts_subtree_total_bytes(last_entry.tree); t_u32 byte_offset = last_entry.byte_offset + ts_subtree_total_bytes(last_entry.tree);
if (ts_subtree_has_external_tokens(last_entry.tree)) { if (ts_subtree_has_external_tokens(last_entry.tree)) {
self->last_external_token = ts_subtree_last_external_token(last_entry.tree); self->last_external_token = ts_subtree_last_external_token(last_entry.tree);
} }
Subtree tree; Subtree tree;
uint32_t next_index; t_u32 next_index;
do { do {
StackEntry popped_entry = array_pop(&self->stack); StackEntry popped_entry = array_pop(&self->stack);
next_index = popped_entry.child_index + 1; next_index = popped_entry.child_index + 1;

69
parser/src/scanner.c
View file

@ -1,11 +1,14 @@
#include "array.h" #include "array.h"
#include "parser.h" #include "parser.h"
#include "parser/types/types_lexer.h"
#include <assert.h> #include <assert.h>
#include <ctype.h> #include <ctype.h>
#include <string.h> #include <string.h>
#include <wctype.h> #include <wctype.h>
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
enum TokenType enum TokenType
{ {
HEREDOC_START, HEREDOC_START,
@ -64,18 +67,18 @@ static inline t_heredoc heredoc_new(void)
typedef struct s_scanner typedef struct s_scanner
{ {
uint8_t last_glob_paren_depth; t_u8 last_glob_paren_depth;
bool ext_was_in_double_quote; bool ext_was_in_double_quote;
bool ext_saw_outside_quote; bool ext_saw_outside_quote;
Array(t_heredoc) heredocs; Array(t_heredoc) heredocs;
} t_scanner; } t_scanner;
static inline void advance(TSLexer *lexer) static inline void advance(t_lexer *lexer)
{ {
lexer->advance(lexer, false); lexer->advance(lexer, false);
} }
static inline void skip(TSLexer *lexer) static inline void skip(t_lexer *lexer)
{ {
lexer->advance(lexer, true); lexer->advance(lexer, true);
} }
@ -104,7 +107,7 @@ static inline void reset_heredoc(t_heredoc *heredoc)
static inline void reset(t_scanner *scanner) static inline void reset(t_scanner *scanner)
{ {
uint32_t i; t_u32 i;
i = 0; i = 0;
while (i < scanner->heredocs.size) while (i < scanner->heredocs.size)
@ -116,8 +119,8 @@ static inline void reset(t_scanner *scanner)
static unsigned serialize(t_scanner *scanner, char *buffer) static unsigned serialize(t_scanner *scanner, char *buffer)
{ {
uint32_t size; t_u32 size;
uint32_t i; t_u32 i;
t_heredoc *heredoc; t_heredoc *heredoc;
size = 0; size = 0;
@ -135,8 +138,8 @@ static unsigned serialize(t_scanner *scanner, char *buffer)
buffer[size++] = (char)heredoc->is_raw; buffer[size++] = (char)heredoc->is_raw;
buffer[size++] = (char)heredoc->started; buffer[size++] = (char)heredoc->started;
buffer[size++] = (char)heredoc->allows_indent; buffer[size++] = (char)heredoc->allows_indent;
memcpy(&buffer[size], &heredoc->delimiter.size, sizeof(uint32_t)); memcpy(&buffer[size], &heredoc->delimiter.size, sizeof(t_u32));
size += sizeof(uint32_t); size += sizeof(t_u32);
if (heredoc->delimiter.size > 0) if (heredoc->delimiter.size > 0)
{ {
memcpy(&buffer[size], heredoc->delimiter.contents, memcpy(&buffer[size], heredoc->delimiter.contents,
@ -150,10 +153,10 @@ static unsigned serialize(t_scanner *scanner, char *buffer)
static void deserialize(t_scanner *scanner, const char *buffer, unsigned length) static void deserialize(t_scanner *scanner, const char *buffer, unsigned length)
{ {
uint32_t size; t_u32 size;
uint32_t heredoc_count; t_u32 heredoc_count;
t_heredoc *heredoc; t_heredoc *heredoc;
uint32_t i; t_u32 i;
size = 0; size = 0;
if (length == 0) if (length == 0)
@ -178,8 +181,8 @@ static void deserialize(t_scanner *scanner, const char *buffer, unsigned length)
heredoc->is_raw = buffer[size++]; heredoc->is_raw = buffer[size++];
heredoc->started = buffer[size++]; heredoc->started = buffer[size++];
heredoc->allows_indent = buffer[size++]; heredoc->allows_indent = buffer[size++];
memcpy(&heredoc->delimiter.size, &buffer[size], sizeof(uint32_t)); memcpy(&heredoc->delimiter.size, &buffer[size], sizeof(t_u32));
size += sizeof(uint32_t); size += sizeof(t_u32);
array_reserve(&heredoc->delimiter, heredoc->delimiter.size); array_reserve(&heredoc->delimiter, heredoc->delimiter.size);
if (heredoc->delimiter.size > 0) if (heredoc->delimiter.size > 0)
{ {
@ -200,10 +203,10 @@ static void deserialize(t_scanner *scanner, const char *buffer, unsigned length)
* POSIX-mandated substitution, and assumes the default value for * POSIX-mandated substitution, and assumes the default value for
* IFS. * IFS.
*/ */
static bool advance_word(TSLexer *lexer, t_string *unquoted_word) static bool advance_word(t_lexer *lexer, t_string *unquoted_word)
{ {
bool empty; bool empty;
int32_t quote; t_i32 quote;
quote = 0; quote = 0;
empty = true; empty = true;
@ -230,7 +233,7 @@ static bool advance_word(TSLexer *lexer, t_string *unquoted_word)
return (!empty); return (!empty);
} }
static inline bool scan_bare_dollar(TSLexer *lexer) static inline bool scan_bare_dollar(t_lexer *lexer)
{ {
while (isspace(lexer->lookahead) && lexer->lookahead != '\n' && while (isspace(lexer->lookahead) && lexer->lookahead != '\n' &&
!lexer->eof(lexer)) !lexer->eof(lexer))
@ -248,7 +251,7 @@ static inline bool scan_bare_dollar(TSLexer *lexer)
return (false); return (false);
} }
static bool scan_heredoc_start(t_heredoc *heredoc, TSLexer *lexer) static bool scan_heredoc_start(t_heredoc *heredoc, t_lexer *lexer)
{ {
bool found_delimiter; bool found_delimiter;
@ -266,18 +269,18 @@ static bool scan_heredoc_start(t_heredoc *heredoc, TSLexer *lexer)
return found_delimiter; return found_delimiter;
} }
static bool scan_heredoc_end_identifier(t_heredoc *heredoc, TSLexer *lexer) static bool scan_heredoc_end_identifier(t_heredoc *heredoc, t_lexer *lexer)
{ {
reset_string(&heredoc->current_leading_word); reset_string(&heredoc->current_leading_word);
// Scan the first 'n' characters on this line, to see if they match the // Scan the first 'n' characters on this line, to see if they match the
// heredoc delimiter // heredoc delimiter
int32_t size; t_i32 size;
size = 0; size = 0;
if (heredoc->delimiter.size > 0) if (heredoc->delimiter.size > 0)
{ {
while (lexer->lookahead != '\0' && lexer->lookahead != '\n' && while (lexer->lookahead != '\0' && lexer->lookahead != '\n' &&
(int32_t)*array_get(&heredoc->delimiter, size) == (t_i32)*array_get(&heredoc->delimiter, size) ==
lexer->lookahead && lexer->lookahead &&
heredoc->current_leading_word.size < heredoc->delimiter.size) heredoc->current_leading_word.size < heredoc->delimiter.size)
{ {
@ -293,7 +296,7 @@ static bool scan_heredoc_end_identifier(t_heredoc *heredoc, TSLexer *lexer)
heredoc->delimiter.contents) == 0; heredoc->delimiter.contents) == 0;
} }
static bool scan_heredoc_content(t_scanner *scanner, TSLexer *lexer, static bool scan_heredoc_content(t_scanner *scanner, t_lexer *lexer,
enum TokenType middle_type, enum TokenType middle_type,
enum TokenType end_type) enum TokenType end_type)
{ {
@ -422,7 +425,7 @@ static bool scan_heredoc_content(t_scanner *scanner, TSLexer *lexer,
} }
} }
} }
static bool regex_scan(t_scanner *scanner, TSLexer *lexer, static bool regex_scan(t_scanner *scanner, t_lexer *lexer,
const bool *valid_symbols) const bool *valid_symbols)
{ {
(void)(scanner); (void)(scanner);
@ -451,9 +454,9 @@ static bool regex_scan(t_scanner *scanner, TSLexer *lexer,
bool found_non_alnumdollarunderdash; bool found_non_alnumdollarunderdash;
bool last_was_escape; bool last_was_escape;
bool in_single_quote; bool in_single_quote;
uint32_t paren_depth; t_u32 paren_depth;
uint32_t bracket_depth; t_u32 bracket_depth;
uint32_t brace_depth; t_u32 brace_depth;
} State; } State;
if (lexer->lookahead == '$' && valid_symbols[REGEX_NO_SLASH]) if (lexer->lookahead == '$' && valid_symbols[REGEX_NO_SLASH])
@ -647,7 +650,7 @@ static bool regex_scan(t_scanner *scanner, TSLexer *lexer,
return (false); return (false);
} }
static bool extglob_pattern_scan(t_scanner *scanner, TSLexer *lexer, static bool extglob_pattern_scan(t_scanner *scanner, t_lexer *lexer,
const bool *valid_symbols) const bool *valid_symbols)
{ {
if (valid_symbols[EXTGLOB_PATTERN] && !in_error_recovery(valid_symbols)) if (valid_symbols[EXTGLOB_PATTERN] && !in_error_recovery(valid_symbols))
@ -792,9 +795,9 @@ static bool extglob_pattern_scan(t_scanner *scanner, TSLexer *lexer,
{ {
bool done; bool done;
bool saw_non_alphadot; bool saw_non_alphadot;
uint32_t paren_depth; t_u32 paren_depth;
uint32_t bracket_depth; t_u32 bracket_depth;
uint32_t brace_depth; t_u32 brace_depth;
} State; } State;
State state = {false, was_non_alpha, scanner->last_glob_paren_depth, State state = {false, was_non_alpha, scanner->last_glob_paren_depth,
@ -923,7 +926,7 @@ static bool extglob_pattern_scan(t_scanner *scanner, TSLexer *lexer,
return (false); return (false);
} }
static bool expansion_word_scan(t_scanner *scanner, TSLexer *lexer, static bool expansion_word_scan(t_scanner *scanner, t_lexer *lexer,
const bool *valid_symbols) const bool *valid_symbols)
{ {
(void)(scanner); (void)(scanner);
@ -1027,7 +1030,7 @@ static bool expansion_word_scan(t_scanner *scanner, TSLexer *lexer,
return (false); return (false);
} }
static bool brace_start_scan(t_scanner *scanner, TSLexer *lexer, static bool brace_start_scan(t_scanner *scanner, t_lexer *lexer,
const bool *valid_symbols) const bool *valid_symbols)
{ {
(void)(scanner); (void)(scanner);
@ -1079,7 +1082,7 @@ static bool brace_start_scan(t_scanner *scanner, TSLexer *lexer,
} }
return (false); return (false);
} }
static bool scan(t_scanner *scanner, TSLexer *lexer, const bool *valid_symbols) static bool scan(t_scanner *scanner, t_lexer *lexer, const bool *valid_symbols)
{ {
if (valid_symbols[CONCAT] && !in_error_recovery(valid_symbols)) if (valid_symbols[CONCAT] && !in_error_recovery(valid_symbols))
{ {
@ -1485,7 +1488,7 @@ void *tree_sitter_bash_external_scanner_create()
return (scanner); return (scanner);
} }
bool tree_sitter_bash_external_scanner_scan(void *payload, TSLexer *lexer, bool tree_sitter_bash_external_scanner_scan(void *payload, t_lexer *lexer,
const bool *valid_symbols) const bool *valid_symbols)
{ {
t_scanner *scanner = (t_scanner *)payload; t_scanner *scanner = (t_scanner *)payload;

54
parser/src/stack.c
View file

@ -27,11 +27,11 @@ typedef struct {
} StackLink; } StackLink;
struct StackNode { struct StackNode {
TSStateId state; t_state_id state;
Length position; Length position;
StackLink links[MAX_LINK_COUNT]; StackLink links[MAX_LINK_COUNT];
short unsigned int link_count; short unsigned int link_count;
uint32_t ref_count; t_u32 ref_count;
unsigned error_cost; unsigned error_cost;
unsigned node_count; unsigned node_count;
int dynamic_precedence; int dynamic_precedence;
@ -40,7 +40,7 @@ struct StackNode {
typedef struct { typedef struct {
StackNode *node; StackNode *node;
SubtreeArray subtrees; SubtreeArray subtrees;
uint32_t subtree_count; t_u32 subtree_count;
bool is_pending; bool is_pending;
} StackIterator; } StackIterator;
@ -123,8 +123,8 @@ recur:
/// Get the number of nodes in the subtree, for the purpose of measuring /// Get the number of nodes in the subtree, for the purpose of measuring
/// how much progress has been made by a given version of the stack. /// how much progress has been made by a given version of the stack.
static uint32_t stack__subtree_node_count(Subtree subtree) { static t_u32 stack__subtree_node_count(Subtree subtree) {
uint32_t count = ts_subtree_visible_descendant_count(subtree); t_u32 count = ts_subtree_visible_descendant_count(subtree);
if (ts_subtree_visible(subtree)) count++; if (ts_subtree_visible(subtree)) count++;
// Count intermediate error nodes even though they are not visible, // Count intermediate error nodes even though they are not visible,
@ -139,7 +139,7 @@ static StackNode *stack_node_new(
StackNode *previous_node, StackNode *previous_node,
Subtree subtree, Subtree subtree,
bool is_pending, bool is_pending,
TSStateId state, t_state_id state,
StackNodeArray *pool StackNodeArray *pool
) { ) {
StackNode *node = pool->size > 0 StackNode *node = pool->size > 0
@ -234,7 +234,7 @@ static void stack_node_add_link(
for (int j = 0; j < link.node->link_count; j++) { for (int j = 0; j < link.node->link_count; j++) {
stack_node_add_link(existing_link->node, link.node->links[j], subtree_pool); stack_node_add_link(existing_link->node, link.node->links[j], subtree_pool);
} }
int32_t dynamic_precedence = link.node->dynamic_precedence; t_i32 dynamic_precedence = link.node->dynamic_precedence;
if (link.subtree.ptr) { if (link.subtree.ptr) {
dynamic_precedence += ts_subtree_dynamic_precedence(link.subtree); dynamic_precedence += ts_subtree_dynamic_precedence(link.subtree);
} }
@ -307,7 +307,7 @@ static void ts_stack__add_slice(
StackNode *node, StackNode *node,
SubtreeArray *subtrees SubtreeArray *subtrees
) { ) {
for (uint32_t i = self->slices.size - 1; i + 1 > 0; i--) { for (t_u32 i = self->slices.size - 1; i + 1 > 0; i--) {
StackVersion version = self->slices.contents[i].version; StackVersion version = self->slices.contents[i].version;
if (self->heads.contents[version].node == node) { if (self->heads.contents[version].node == node) {
StackSlice slice = {*subtrees, version}; StackSlice slice = {*subtrees, version};
@ -342,13 +342,13 @@ static StackSliceArray stack__iter(
bool include_subtrees = false; bool include_subtrees = false;
if (goal_subtree_count >= 0) { if (goal_subtree_count >= 0) {
include_subtrees = true; include_subtrees = true;
array_reserve(&new_iterator.subtrees, (uint32_t)ts_subtree_alloc_size(goal_subtree_count) / sizeof(Subtree)); array_reserve(&new_iterator.subtrees, (t_u32)ts_subtree_alloc_size(goal_subtree_count) / sizeof(Subtree));
} }
array_push(&self->iterators, new_iterator); array_push(&self->iterators, new_iterator);
while (self->iterators.size > 0) { while (self->iterators.size > 0) {
for (uint32_t i = 0, size = self->iterators.size; i < size; i++) { for (t_u32 i = 0, size = self->iterators.size; i < size; i++) {
StackIterator *iterator = &self->iterators.contents[i]; StackIterator *iterator = &self->iterators.contents[i];
StackNode *node = iterator->node; StackNode *node = iterator->node;
@ -379,7 +379,7 @@ static StackSliceArray stack__iter(
continue; continue;
} }
for (uint32_t j = 1; j <= node->link_count; j++) { for (t_u32 j = 1; j <= node->link_count; j++) {
StackIterator *next_iterator; StackIterator *next_iterator;
StackLink link; StackLink link;
if (j == node->link_count) { if (j == node->link_count) {
@ -443,12 +443,12 @@ void ts_stack_delete(Stack *self) {
if (self->iterators.contents) if (self->iterators.contents)
array_delete(&self->iterators); array_delete(&self->iterators);
stack_node_release(self->base_node, &self->node_pool, self->subtree_pool); stack_node_release(self->base_node, &self->node_pool, self->subtree_pool);
for (uint32_t i = 0; i < self->heads.size; i++) { for (t_u32 i = 0; i < self->heads.size; i++) {
stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool); stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool);
} }
array_clear(&self->heads); array_clear(&self->heads);
if (self->node_pool.contents) { if (self->node_pool.contents) {
for (uint32_t i = 0; i < self->node_pool.size; i++) for (t_u32 i = 0; i < self->node_pool.size; i++)
free(self->node_pool.contents[i]); free(self->node_pool.contents[i]);
array_delete(&self->node_pool); array_delete(&self->node_pool);
} }
@ -456,11 +456,11 @@ void ts_stack_delete(Stack *self) {
free(self); free(self);
} }
uint32_t ts_stack_version_count(const Stack *self) { t_u32 ts_stack_version_count(const Stack *self) {
return self->heads.size; return self->heads.size;
} }
TSStateId ts_stack_state(const Stack *self, StackVersion version) { t_state_id ts_stack_state(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->node->state; return array_get(&self->heads, version)->node->state;
} }
@ -503,7 +503,7 @@ void ts_stack_push(
StackVersion version, StackVersion version,
Subtree subtree, Subtree subtree,
bool pending, bool pending,
TSStateId state t_state_id state
) { ) {
StackHead *head = array_get(&self->heads, version); StackHead *head = array_get(&self->heads, version);
StackNode *new_node = stack_node_new(head->node, subtree, pending, state, &self->node_pool); StackNode *new_node = stack_node_new(head->node, subtree, pending, state, &self->node_pool);
@ -520,7 +520,7 @@ forceinline StackAction pop_count_callback(void *payload, const StackIterator *i
} }
} }
StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t count) { StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, t_u32 count) {
return stack__iter(self, version, pop_count_callback, &count, (int)count); return stack__iter(self, version, pop_count_callback, &count, (int)count);
} }
@ -593,7 +593,7 @@ typedef struct {
forceinline StackAction summarize_stack_callback(void *payload, const StackIterator *iterator) { forceinline StackAction summarize_stack_callback(void *payload, const StackIterator *iterator) {
SummarizeStackSession *session = payload; SummarizeStackSession *session = payload;
TSStateId state = iterator->node->state; t_state_id state = iterator->node->state;
unsigned depth = iterator->subtree_count; unsigned depth = iterator->subtree_count;
if (depth > session->max_depth) return StackActionStop; if (depth > session->max_depth) return StackActionStop;
for (unsigned i = session->summary->size - 1; i + 1 > 0; i--) { for (unsigned i = session->summary->size - 1; i + 1 > 0; i--) {
@ -664,7 +664,7 @@ void ts_stack_remove_version(Stack *self, StackVersion version) {
void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) { void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) {
if (v1 == v2) return; if (v1 == v2) return;
assert(v2 < v1); assert(v2 < v1);
assert((uint32_t)v1 < self->heads.size); assert((t_u32)v1 < self->heads.size);
StackHead *source_head = &self->heads.contents[v1]; StackHead *source_head = &self->heads.contents[v1];
StackHead *target_head = &self->heads.contents[v2]; StackHead *target_head = &self->heads.contents[v2];
if (target_head->summary && !source_head->summary) { if (target_head->summary && !source_head->summary) {
@ -696,7 +696,7 @@ bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2) {
if (!ts_stack_can_merge(self, version1, version2)) return false; if (!ts_stack_can_merge(self, version1, version2)) return false;
StackHead *head1 = &self->heads.contents[version1]; StackHead *head1 = &self->heads.contents[version1];
StackHead *head2 = &self->heads.contents[version2]; StackHead *head2 = &self->heads.contents[version2];
for (uint32_t i = 0; i < head2->node->link_count; i++) { for (t_u32 i = 0; i < head2->node->link_count; i++) {
stack_node_add_link(head1->node, head2->node->links[i], self->subtree_pool); stack_node_add_link(head1->node, head2->node->links[i], self->subtree_pool);
} }
if (head1->node->state == ERROR_STATE) { if (head1->node->state == ERROR_STATE) {
@ -752,7 +752,7 @@ Subtree ts_stack_resume(Stack *self, StackVersion version) {
void ts_stack_clear(Stack *self) { void ts_stack_clear(Stack *self) {
stack_node_retain(self->base_node); stack_node_retain(self->base_node);
for (uint32_t i = 0; i < self->heads.size; i++) { for (t_u32 i = 0; i < self->heads.size; i++) {
stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool); stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool);
} }
array_clear(&self->heads); array_clear(&self->heads);
@ -764,7 +764,7 @@ void ts_stack_clear(Stack *self) {
})); }));
} }
bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) { bool ts_stack_print_dot_graph(Stack *self, const t_language *language, FILE *f) {
array_reserve(&self->iterators, 32); array_reserve(&self->iterators, 32);
if (!f) f = stderr; if (!f) f = stderr;
@ -775,7 +775,7 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
Array(StackNode *) visited_nodes = array_new(); Array(StackNode *) visited_nodes = array_new();
array_clear(&self->iterators); array_clear(&self->iterators);
for (uint32_t i = 0; i < self->heads.size; i++) { for (t_u32 i = 0; i < self->heads.size; i++) {
StackHead *head = &self->heads.contents[i]; StackHead *head = &self->heads.contents[i];
if (head->status == StackStatusHalted) continue; if (head->status == StackStatusHalted) continue;
@ -794,14 +794,14 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
if (head->summary) { if (head->summary) {
fprintf(f, "\nsummary:"); fprintf(f, "\nsummary:");
for (uint32_t j = 0; j < head->summary->size; j++) fprintf(f, " %u", head->summary->contents[j].state); for (t_u32 j = 0; j < head->summary->size; j++) fprintf(f, " %u", head->summary->contents[j].state);
} }
if (head->last_external_token.ptr) { if (head->last_external_token.ptr) {
const ExternalScannerState *state = &head->last_external_token.ptr->external_scanner_state; const ExternalScannerState *state = &head->last_external_token.ptr->external_scanner_state;
const char *data = ts_external_scanner_state_data(state); const char *data = ts_external_scanner_state_data(state);
fprintf(f, "\nexternal_scanner_state:"); fprintf(f, "\nexternal_scanner_state:");
for (uint32_t j = 0; j < state->length; j++) fprintf(f, " %2X", data[j]); for (t_u32 j = 0; j < state->length; j++) fprintf(f, " %2X", data[j]);
} }
fprintf(f, "\"]\n"); fprintf(f, "\"]\n");
@ -814,11 +814,11 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
while (!all_iterators_done) { while (!all_iterators_done) {
all_iterators_done = true; all_iterators_done = true;
for (uint32_t i = 0; i < self->iterators.size; i++) { for (t_u32 i = 0; i < self->iterators.size; i++) {
StackIterator iterator = self->iterators.contents[i]; StackIterator iterator = self->iterators.contents[i];
StackNode *node = iterator.node; StackNode *node = iterator.node;
for (uint32_t j = 0; j < visited_nodes.size; j++) { for (t_u32 j = 0; j < visited_nodes.size; j++) {
if (visited_nodes.contents[j] == node) { if (visited_nodes.contents[j] == node) {
node = NULL; node = NULL;
break; break;

14
parser/src/stack.h
View file

@ -24,7 +24,7 @@ typedef Array(StackSlice) StackSliceArray;
typedef struct { typedef struct {
Length position; Length position;
unsigned depth; unsigned depth;
TSStateId state; t_state_id state;
} StackSummaryEntry; } StackSummaryEntry;
typedef Array(StackSummaryEntry) StackSummary; typedef Array(StackSummaryEntry) StackSummary;
@ -35,11 +35,11 @@ Stack *ts_stack_new(SubtreePool *);
void ts_stack_delete(Stack *); void ts_stack_delete(Stack *);
// Get the stack's current number of versions. // Get the stack's current number of versions.
uint32_t ts_stack_version_count(const Stack *); t_u32 ts_stack_version_count(const Stack *);
// Get the state at the top of the given version of the stack. If the stack is // Get the state at the top of the given version of the stack. If the stack is
// empty, this returns the initial state, 0. // empty, this returns the initial state, 0.
TSStateId ts_stack_state(const Stack *, StackVersion); t_state_id ts_stack_state(const Stack *, StackVersion);
// Get the last external token associated with a given version of the stack. // Get the last external token associated with a given version of the stack.
Subtree ts_stack_last_external_token(const Stack *, StackVersion); Subtree ts_stack_last_external_token(const Stack *, StackVersion);
@ -55,14 +55,14 @@ Length ts_stack_position(const Stack *, StackVersion);
// This transfers ownership of the tree to the Stack. Callers that // This transfers ownership of the tree to the Stack. Callers that
// need to retain ownership of the tree for their own purposes should // need to retain ownership of the tree for their own purposes should
// first retain the tree. // first retain the tree.
void ts_stack_push(Stack *, StackVersion, Subtree , bool, TSStateId); void ts_stack_push(Stack *, StackVersion, Subtree , bool, t_state_id);
// Pop the given number of entries from the given version of the stack. This // Pop the given number of entries from the given version of the stack. This
// operation can increase the number of stack versions by revealing multiple // operation can increase the number of stack versions by revealing multiple
// versions which had previously been merged. It returns an array that // versions which had previously been merged. It returns an array that
// specifies the index of each revealed version and the trees that were // specifies the index of each revealed version and the trees that were
// removed from that version. // removed from that version.
StackSliceArray ts_stack_pop_count(Stack *, StackVersion, uint32_t count); StackSliceArray ts_stack_pop_count(Stack *, StackVersion, t_u32 count);
// Remove an error at the top of the given version of the stack. // Remove an error at the top of the given version of the stack.
SubtreeArray ts_stack_pop_error(Stack *, StackVersion); SubtreeArray ts_stack_pop_error(Stack *, StackVersion);
@ -122,9 +122,9 @@ void ts_stack_remove_version(Stack *, StackVersion);
void ts_stack_clear(Stack *); void ts_stack_clear(Stack *);
bool ts_stack_print_dot_graph(Stack *, const TSLanguage *, FILE *); bool ts_stack_print_dot_graph(Stack *, const t_language *, FILE *);
typedef void (*StackIterateCallback)(void *, TSStateId, uint32_t); typedef void (*StackIterateCallback)(void *, t_state_id, t_u32);
#ifdef __cplusplus #ifdef __cplusplus
} }

176
parser/src/subtree.c
View file

@ -90,7 +90,7 @@ void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest)
{ {
dest->contents = calloc(self.capacity, sizeof(Subtree)); dest->contents = calloc(self.capacity, sizeof(Subtree));
memcpy(dest->contents, self.contents, self.size * sizeof(Subtree)); memcpy(dest->contents, self.contents, self.size * sizeof(Subtree));
for (uint32_t i = 0; i < self.size; i++) for (t_u32 i = 0; i < self.size; i++)
{ {
ts_subtree_retain(dest->contents[i]); ts_subtree_retain(dest->contents[i]);
} }
@ -99,7 +99,7 @@ void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest)
void ts_subtree_array_clear(SubtreePool *pool, SubtreeArray *self) void ts_subtree_array_clear(SubtreePool *pool, SubtreeArray *self)
{ {
for (uint32_t i = 0; i < self->size; i++) for (t_u32 i = 0; i < self->size; i++)
{ {
ts_subtree_release(pool, self->contents[i]); ts_subtree_release(pool, self->contents[i]);
} }
@ -134,7 +134,7 @@ void ts_subtree_array_remove_trailing_extras(SubtreeArray *self,
void ts_subtree_array_reverse(SubtreeArray *self) void ts_subtree_array_reverse(SubtreeArray *self)
{ {
for (uint32_t i = 0, limit = self->size / 2; i < limit; i++) for (t_u32 i = 0, limit = self->size / 2; i < limit; i++)
{ {
size_t reverse_index = self->size - 1 - i; size_t reverse_index = self->size - 1 - i;
Subtree swap = self->contents[i]; Subtree swap = self->contents[i];
@ -145,7 +145,7 @@ void ts_subtree_array_reverse(SubtreeArray *self)
// SubtreePool // SubtreePool
SubtreePool ts_subtree_pool_new(uint32_t capacity) SubtreePool ts_subtree_pool_new(t_u32 capacity)
{ {
SubtreePool self = {array_new(), array_new()}; SubtreePool self = {array_new(), array_new()};
array_reserve(&self.free_trees, capacity); array_reserve(&self.free_trees, capacity);
@ -194,7 +194,7 @@ static void ts_subtree_pool_free(SubtreePool *self, SubtreeHeapData *tree)
// Subtree // Subtree
static inline bool ts_subtree_can_inline(Length padding, Length size, static inline bool ts_subtree_can_inline(Length padding, Length size,
uint32_t lookahead_bytes) t_u32 lookahead_bytes)
{ {
return padding.bytes < TS_MAX_INLINE_TREE_LENGTH && return padding.bytes < TS_MAX_INLINE_TREE_LENGTH &&
padding.extent.row < 16 && padding.extent.row < 16 &&
@ -204,13 +204,13 @@ static inline bool ts_subtree_can_inline(Length padding, Length size,
lookahead_bytes < 16; lookahead_bytes < 16;
} }
Subtree ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding, Subtree ts_subtree_new_leaf(SubtreePool *pool, t_symbol symbol, Length padding,
Length size, uint32_t lookahead_bytes, Length size, t_u32 lookahead_bytes,
TSStateId parse_state, bool has_external_tokens, t_state_id parse_state, bool has_external_tokens,
bool depends_on_column, bool is_keyword, bool depends_on_column, bool is_keyword,
const TSLanguage *language) const t_language *language)
{ {
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); t_symbol_metadata metadata = ts_language_symbol_metadata(language, symbol);
bool extra = symbol == ts_builtin_sym_end; bool extra = symbol == ts_builtin_sym_end;
bool is_inline = (symbol <= UINT8_MAX && !has_external_tokens && bool is_inline = (symbol <= UINT8_MAX && !has_external_tokens &&
@ -263,10 +263,10 @@ Subtree ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding,
} }
} }
void ts_subtree_set_symbol(MutableSubtree *self, TSSymbol symbol, void ts_subtree_set_symbol(MutableSubtree *self, t_symbol symbol,
const TSLanguage *language) const t_language *language)
{ {
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); t_symbol_metadata metadata = ts_language_symbol_metadata(language, symbol);
if (self->data.is_inline) if (self->data.is_inline)
{ {
assert(symbol < UINT8_MAX); assert(symbol < UINT8_MAX);
@ -282,10 +282,10 @@ void ts_subtree_set_symbol(MutableSubtree *self, TSSymbol symbol,
} }
} }
Subtree ts_subtree_new_error(SubtreePool *pool, int32_t lookahead_char, Subtree ts_subtree_new_error(SubtreePool *pool, t_i32 lookahead_char,
Length padding, Length size, Length padding, Length size,
uint32_t bytes_scanned, TSStateId parse_state, t_u32 bytes_scanned, t_state_id parse_state,
const TSLanguage *language) const t_language *language)
{ {
Subtree result = ts_subtree_new_leaf(pool, ts_builtin_sym_error, padding, Subtree result = ts_subtree_new_leaf(pool, ts_builtin_sym_error, padding,
size, bytes_scanned, parse_state, size, bytes_scanned, parse_state,
@ -308,7 +308,7 @@ MutableSubtree ts_subtree_clone(Subtree self)
(SubtreeHeapData *)&new_children[self.ptr->child_count]; (SubtreeHeapData *)&new_children[self.ptr->child_count];
if (self.ptr->child_count > 0) if (self.ptr->child_count > 0)
{ {
for (uint32_t i = 0; i < self.ptr->child_count; i++) for (t_u32 i = 0; i < self.ptr->child_count; i++)
{ {
ts_subtree_retain(new_children[i]); ts_subtree_retain(new_children[i]);
} }
@ -332,33 +332,33 @@ MutableSubtree ts_subtree_make_mut(SubtreePool *pool, Subtree self)
if (self.data.is_inline) if (self.data.is_inline)
return (MutableSubtree){self.data}; return (MutableSubtree){self.data};
if (self.ptr->ref_count == 1) if (self.ptr->ref_count == 1)
return ts_subtree_to_mut_unsafe(self); return ts_subtree_to_mt_unsafe(self);
MutableSubtree result = ts_subtree_clone(self); MutableSubtree result = ts_subtree_clone(self);
ts_subtree_release(pool, self); ts_subtree_release(pool, self);
return result; return result;
} }
static void ts_subtree__compress(MutableSubtree self, unsigned count, static void ts_subtree__compress(MutableSubtree self, unsigned count,
const TSLanguage *language, const t_language *language,
MutableSubtreeArray *stack) MutableSubtreeArray *stack)
{ {
unsigned initial_stack_size = stack->size; unsigned initial_stack_size = stack->size;
MutableSubtree tree = self; MutableSubtree tree = self;
TSSymbol symbol = tree.ptr->symbol; t_symbol symbol = tree.ptr->symbol;
for (unsigned i = 0; i < count; i++) for (unsigned i = 0; i < count; i++)
{ {
if (tree.ptr->ref_count > 1 || tree.ptr->child_count < 2) if (tree.ptr->ref_count > 1 || tree.ptr->child_count < 2)
break; break;
MutableSubtree child = MutableSubtree child =
ts_subtree_to_mut_unsafe(ts_subtree_children(tree)[0]); ts_subtree_to_mt_unsafe(ts_subtree_children(tree)[0]);
if (child.data.is_inline || child.ptr->child_count < 2 || if (child.data.is_inline || child.ptr->child_count < 2 ||
child.ptr->ref_count > 1 || child.ptr->symbol != symbol) child.ptr->ref_count > 1 || child.ptr->symbol != symbol)
break; break;
MutableSubtree grandchild = MutableSubtree grandchild =
ts_subtree_to_mut_unsafe(ts_subtree_children(child)[0]); ts_subtree_to_mt_unsafe(ts_subtree_children(child)[0]);
if (grandchild.data.is_inline || grandchild.ptr->child_count < 2 || if (grandchild.data.is_inline || grandchild.ptr->child_count < 2 ||
grandchild.ptr->ref_count > 1 || grandchild.ptr->symbol != symbol) grandchild.ptr->ref_count > 1 || grandchild.ptr->symbol != symbol)
break; break;
@ -376,8 +376,8 @@ static void ts_subtree__compress(MutableSubtree self, unsigned count,
{ {
tree = array_pop(stack); tree = array_pop(stack);
MutableSubtree child = MutableSubtree child =
ts_subtree_to_mut_unsafe(ts_subtree_children(tree)[0]); ts_subtree_to_mt_unsafe(ts_subtree_children(tree)[0]);
MutableSubtree grandchild = ts_subtree_to_mut_unsafe( MutableSubtree grandchild = ts_subtree_to_mt_unsafe(
ts_subtree_children(child)[child.ptr->child_count - 1]); ts_subtree_children(child)[child.ptr->child_count - 1]);
ts_subtree_summarize_children(grandchild, language); ts_subtree_summarize_children(grandchild, language);
ts_subtree_summarize_children(child, language); ts_subtree_summarize_children(child, language);
@ -386,13 +386,13 @@ static void ts_subtree__compress(MutableSubtree self, unsigned count,
} }
void ts_subtree_balance(Subtree self, SubtreePool *pool, void ts_subtree_balance(Subtree self, SubtreePool *pool,
const TSLanguage *language) const t_language *language)
{ {
array_clear(&pool->tree_stack); array_clear(&pool->tree_stack);
if (ts_subtree_child_count(self) > 0 && self.ptr->ref_count == 1) if (ts_subtree_child_count(self) > 0 && self.ptr->ref_count == 1)
{ {
array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(self)); array_push(&pool->tree_stack, ts_subtree_to_mt_unsafe(self));
} }
while (pool->tree_stack.size > 0) while (pool->tree_stack.size > 0)
@ -417,12 +417,12 @@ void ts_subtree_balance(Subtree self, SubtreePool *pool,
} }
} }
for (uint32_t i = 0; i < tree.ptr->child_count; i++) for (t_u32 i = 0; i < tree.ptr->child_count; i++)
{ {
Subtree child = ts_subtree_children(tree)[i]; Subtree child = ts_subtree_children(tree)[i];
if (ts_subtree_child_count(child) > 0 && child.ptr->ref_count == 1) if (ts_subtree_child_count(child) > 0 && child.ptr->ref_count == 1)
{ {
array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(child)); array_push(&pool->tree_stack, ts_subtree_to_mt_unsafe(child));
} }
} }
} }
@ -430,7 +430,7 @@ void ts_subtree_balance(Subtree self, SubtreePool *pool,
// Assign all of the node's properties that depend on its children. // Assign all of the node's properties that depend on its children.
void ts_subtree_summarize_children(MutableSubtree self, void ts_subtree_summarize_children(MutableSubtree self,
const TSLanguage *language) const t_language *language)
{ {
assert(!self.data.is_inline); assert(!self.data.is_inline);
@ -444,13 +444,13 @@ void ts_subtree_summarize_children(MutableSubtree self,
self.ptr->has_external_scanner_state_change = false; self.ptr->has_external_scanner_state_change = false;
self.ptr->dynamic_precedence = 0; self.ptr->dynamic_precedence = 0;
uint32_t structural_index = 0; t_u32 structural_index = 0;
const TSSymbol *alias_sequence = const t_symbol *alias_sequence =
ts_language_alias_sequence(language, self.ptr->production_id); ts_language_alias_sequence(language, self.ptr->production_id);
uint32_t lookahead_end_byte = 0; t_u32 lookahead_end_byte = 0;
const Subtree *children = ts_subtree_children(self); const Subtree *children = ts_subtree_children(self);
for (uint32_t i = 0; i < self.ptr->child_count; i++) for (t_u32 i = 0; i < self.ptr->child_count; i++)
{ {
Subtree child = children[i]; Subtree child = children[i];
@ -476,7 +476,7 @@ void ts_subtree_summarize_children(MutableSubtree self,
length_add(self.ptr->size, ts_subtree_total_size(child)); length_add(self.ptr->size, ts_subtree_total_size(child));
} }
uint32_t child_lookahead_end_byte = self.ptr->padding.bytes + t_u32 child_lookahead_end_byte = self.ptr->padding.bytes +
self.ptr->size.bytes + self.ptr->size.bytes +
ts_subtree_lookahead_bytes(child); ts_subtree_lookahead_bytes(child);
if (child_lookahead_end_byte > lookahead_end_byte) if (child_lookahead_end_byte > lookahead_end_byte)
@ -489,7 +489,7 @@ void ts_subtree_summarize_children(MutableSubtree self,
self.ptr->error_cost += ts_subtree_error_cost(child); self.ptr->error_cost += ts_subtree_error_cost(child);
} }
uint32_t grandchild_count = ts_subtree_child_count(child); t_u32 grandchild_count = ts_subtree_child_count(child);
if (self.ptr->symbol == ts_builtin_sym_error || if (self.ptr->symbol == ts_builtin_sym_error ||
self.ptr->symbol == ts_builtin_sym_error_repeat) self.ptr->symbol == ts_builtin_sym_error_repeat)
{ {
@ -598,11 +598,11 @@ void ts_subtree_summarize_children(MutableSubtree self,
// Create a new parent node with the given children. // Create a new parent node with the given children.
// //
// This takes ownership of the children array. // This takes ownership of the children array.
MutableSubtree ts_subtree_new_node(TSSymbol symbol, SubtreeArray *children, MutableSubtree ts_subtree_new_node(t_symbol symbol, SubtreeArray *children,
unsigned production_id, unsigned production_id,
const TSLanguage *language) const t_language *language)
{ {
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); t_symbol_metadata metadata = ts_language_symbol_metadata(language, symbol);
bool fragile = bool fragile =
symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat; symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat;
@ -611,7 +611,7 @@ MutableSubtree ts_subtree_new_node(TSSymbol symbol, SubtreeArray *children,
if (children->capacity * sizeof(Subtree) < new_byte_size) if (children->capacity * sizeof(Subtree) < new_byte_size)
{ {
children->contents = realloc(children->contents, new_byte_size); children->contents = realloc(children->contents, new_byte_size);
children->capacity = (uint32_t)(new_byte_size / sizeof(Subtree)); children->capacity = (t_u32)(new_byte_size / sizeof(Subtree));
} }
SubtreeHeapData *data = SubtreeHeapData *data =
(SubtreeHeapData *)&children->contents[children->size]; (SubtreeHeapData *)&children->contents[children->size];
@ -641,7 +641,7 @@ MutableSubtree ts_subtree_new_node(TSSymbol symbol, SubtreeArray *children,
// This node is treated as 'extra'. Its children are prevented from having // This node is treated as 'extra'. Its children are prevented from having
// having any effect on the parse state. // having any effect on the parse state.
Subtree ts_subtree_new_error_node(SubtreeArray *children, bool extra, Subtree ts_subtree_new_error_node(SubtreeArray *children, bool extra,
const TSLanguage *language) const t_language *language)
{ {
MutableSubtree result = MutableSubtree result =
ts_subtree_new_node(ts_builtin_sym_error, children, 0, language); ts_subtree_new_node(ts_builtin_sym_error, children, 0, language);
@ -653,9 +653,9 @@ Subtree ts_subtree_new_error_node(SubtreeArray *children, bool extra,
// //
// This node is treated as 'extra'. Its children are prevented from having // This node is treated as 'extra'. Its children are prevented from having
// having any effect on the parse state. // having any effect on the parse state.
Subtree ts_subtree_new_missing_leaf(SubtreePool *pool, TSSymbol symbol, Subtree ts_subtree_new_missing_leaf(SubtreePool *pool, t_symbol symbol,
Length padding, uint32_t lookahead_bytes, Length padding, t_u32 lookahead_bytes,
const TSLanguage *language) const t_language *language)
{ {
Subtree result = Subtree result =
ts_subtree_new_leaf(pool, symbol, padding, length_zero(), ts_subtree_new_leaf(pool, symbol, padding, length_zero(),
@ -676,7 +676,7 @@ void ts_subtree_retain(Subtree self)
if (self.data.is_inline) if (self.data.is_inline)
return; return;
assert(self.ptr->ref_count > 0); assert(self.ptr->ref_count > 0);
*(uint32_t *)&self.ptr->ref_count += 1; *(t_u32 *)&self.ptr->ref_count += 1;
assert(self.ptr->ref_count != 0); assert(self.ptr->ref_count != 0);
} }
@ -687,9 +687,9 @@ void ts_subtree_release(SubtreePool *pool, Subtree self)
array_clear(&pool->tree_stack); array_clear(&pool->tree_stack);
assert(self.ptr->ref_count > 0); assert(self.ptr->ref_count > 0);
if (--(*(uint32_t *)&self.ptr->ref_count) == 0) if (--(*(t_u32 *)&self.ptr->ref_count) == 0)
{ {
array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(self)); array_push(&pool->tree_stack, ts_subtree_to_mt_unsafe(self));
} }
while (pool->tree_stack.size > 0) while (pool->tree_stack.size > 0)
@ -698,16 +698,16 @@ void ts_subtree_release(SubtreePool *pool, Subtree self)
if (tree.ptr->child_count > 0) if (tree.ptr->child_count > 0)
{ {
Subtree *children = ts_subtree_children(tree); Subtree *children = ts_subtree_children(tree);
for (uint32_t i = 0; i < tree.ptr->child_count; i++) for (t_u32 i = 0; i < tree.ptr->child_count; i++)
{ {
Subtree child = children[i]; Subtree child = children[i];
if (child.data.is_inline) if (child.data.is_inline)
continue; continue;
assert(child.ptr->ref_count > 0); assert(child.ptr->ref_count > 0);
if (--*(uint32_t *)&child.ptr->ref_count == 0) if (--*(t_u32 *)&child.ptr->ref_count == 0)
{ {
array_push(&pool->tree_stack, array_push(&pool->tree_stack,
ts_subtree_to_mut_unsafe(child)); ts_subtree_to_mt_unsafe(child));
} }
} }
free(children); free(children);
@ -726,8 +726,8 @@ void ts_subtree_release(SubtreePool *pool, Subtree self)
int ts_subtree_compare(Subtree left, Subtree right, SubtreePool *pool) int ts_subtree_compare(Subtree left, Subtree right, SubtreePool *pool)
{ {
array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(left)); array_push(&pool->tree_stack, ts_subtree_to_mt_unsafe(left));
array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(right)); array_push(&pool->tree_stack, ts_subtree_to_mt_unsafe(right));
while (pool->tree_stack.size > 0) while (pool->tree_stack.size > 0)
{ {
@ -749,13 +749,13 @@ int ts_subtree_compare(Subtree left, Subtree right, SubtreePool *pool)
return result; return result;
} }
for (uint32_t i = ts_subtree_child_count(left); i > 0; i--) for (t_u32 i = ts_subtree_child_count(left); i > 0; i--)
{ {
Subtree left_child = ts_subtree_children(left)[i - 1]; Subtree left_child = ts_subtree_children(left)[i - 1];
Subtree right_child = ts_subtree_children(right)[i - 1]; Subtree right_child = ts_subtree_children(right)[i - 1];
array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(left_child)); array_push(&pool->tree_stack, ts_subtree_to_mt_unsafe(left_child));
array_push(&pool->tree_stack, array_push(&pool->tree_stack,
ts_subtree_to_mut_unsafe(right_child)); ts_subtree_to_mt_unsafe(right_child));
} }
} }
@ -774,7 +774,7 @@ static inline void ts_subtree_set_has_changes(MutableSubtree *self)
} }
} }
Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit, Subtree ts_subtree_edit(Subtree self, const t_input_edit *inpt_edit,
SubtreePool *pool) SubtreePool *pool)
{ {
typedef struct typedef struct
@ -790,11 +790,11 @@ Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit,
.tree = &self, .tree = &self,
.edit = .edit =
(Edit){ (Edit){
.start = {input_edit->start_byte, input_edit->start_point}, .start = {inpt_edit->start_byte, inpt_edit->start_point},
.old_end = {input_edit->old_end_byte, .old_end = {inpt_edit->old_end_byte,
input_edit->old_end_point}, inpt_edit->old_end_point},
.new_end = {input_edit->new_end_byte, .new_end = {inpt_edit->new_end_byte,
input_edit->new_end_point}, inpt_edit->new_end_point},
}, },
})); }));
@ -810,8 +810,8 @@ Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit,
Length size = ts_subtree_size(*entry.tree); Length size = ts_subtree_size(*entry.tree);
Length padding = ts_subtree_padding(*entry.tree); Length padding = ts_subtree_padding(*entry.tree);
Length total_size = length_add(padding, size); Length total_size = length_add(padding, size);
uint32_t lookahead_bytes = ts_subtree_lookahead_bytes(*entry.tree); t_u32 lookahead_bytes = ts_subtree_lookahead_bytes(*entry.tree);
uint32_t end_byte = total_size.bytes + lookahead_bytes; t_u32 end_byte = total_size.bytes + lookahead_bytes;
if (edit.start.bytes > end_byte || if (edit.start.bytes > end_byte ||
(is_noop && edit.start.bytes == end_byte)) (is_noop && edit.start.bytes == end_byte))
continue; continue;
@ -896,7 +896,7 @@ Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit,
*entry.tree = ts_subtree_from_mut(result); *entry.tree = ts_subtree_from_mut(result);
Length child_left, child_right = length_zero(); Length child_left, child_right = length_zero();
for (uint32_t i = 0, n = ts_subtree_child_count(*entry.tree); i < n; for (t_u32 i = 0, n = ts_subtree_child_count(*entry.tree); i < n;
i++) i++)
{ {
Subtree *child = &ts_subtree_children(*entry.tree)[i]; Subtree *child = &ts_subtree_children(*entry.tree)[i];
@ -964,7 +964,7 @@ Subtree ts_subtree_last_external_token(Subtree tree)
return NULL_SUBTREE; return NULL_SUBTREE;
while (tree.ptr->child_count > 0) while (tree.ptr->child_count > 0)
{ {
for (uint32_t i = tree.ptr->child_count - 1; i + 1 > 0; i--) for (t_u32 i = tree.ptr->child_count - 1; i + 1 > 0; i--)
{ {
Subtree child = ts_subtree_children(tree)[i]; Subtree child = ts_subtree_children(tree)[i];
if (ts_subtree_has_external_tokens(child)) if (ts_subtree_has_external_tokens(child))
@ -977,7 +977,7 @@ Subtree ts_subtree_last_external_token(Subtree tree)
return tree; return tree;
} }
static size_t ts_subtree__write_char_to_string(char *str, size_t n, int32_t chr) static size_t ts_subtree__write_char_to_string(char *str, size_t n, t_i32 chr)
{ {
if (chr == -1) if (chr == -1)
return snprintf(str, n, "INVALID"); return snprintf(str, n, "INVALID");
@ -998,8 +998,8 @@ static size_t ts_subtree__write_char_to_string(char *str, size_t n, int32_t chr)
static const char *const ROOT_FIELD = "__ROOT__"; static const char *const ROOT_FIELD = "__ROOT__";
static size_t ts_subtree__write_to_string( static size_t ts_subtree__write_to_string(
Subtree self, char *string, size_t limit, const TSLanguage *language, Subtree self, char *string, size_t limit, const t_language *language,
bool include_all, TSSymbol alias_symbol, bool alias_is_named, bool include_all, t_symbol alias_symbol, bool alias_is_named,
const char *field_name) const char *field_name)
{ {
if (!self.ptr) if (!self.ptr)
@ -1033,7 +1033,7 @@ static size_t ts_subtree__write_to_string(
} }
else else
{ {
TSSymbol symbol = t_symbol symbol =
alias_symbol ? alias_symbol : ts_subtree_symbol(self); alias_symbol ? alias_symbol : ts_subtree_symbol(self);
const char *symbol_name = ts_language_symbol_name(language, symbol); const char *symbol_name = ts_language_symbol_name(language, symbol);
if (ts_subtree_missing(self)) if (ts_subtree_missing(self))
@ -1056,7 +1056,7 @@ static size_t ts_subtree__write_to_string(
} }
else if (is_root) else if (is_root)
{ {
TSSymbol symbol = alias_symbol ? alias_symbol : ts_subtree_symbol(self); t_symbol symbol = alias_symbol ? alias_symbol : ts_subtree_symbol(self);
const char *symbol_name = ts_language_symbol_name(language, symbol); const char *symbol_name = ts_language_symbol_name(language, symbol);
if (ts_subtree_child_count(self) > 0) if (ts_subtree_child_count(self) > 0)
{ {
@ -1074,14 +1074,14 @@ static size_t ts_subtree__write_to_string(
if (ts_subtree_child_count(self)) if (ts_subtree_child_count(self))
{ {
const TSSymbol *alias_sequence = const t_symbol *alias_sequence =
ts_language_alias_sequence(language, self.ptr->production_id); ts_language_alias_sequence(language, self.ptr->production_id);
const TSFieldMapEntry *field_map, *field_map_end; const t_field_map_entry *field_map, *field_map_end;
ts_language_field_map(language, self.ptr->production_id, &field_map, ts_language_field_map(language, self.ptr->production_id, &field_map,
&field_map_end); &field_map_end);
uint32_t structural_child_index = 0; t_u32 structural_child_index = 0;
for (uint32_t i = 0; i < self.ptr->child_count; i++) for (t_u32 i = 0; i < self.ptr->child_count; i++)
{ {
Subtree child = ts_subtree_children(self)[i]; Subtree child = ts_subtree_children(self)[i];
if (ts_subtree_extra(child)) if (ts_subtree_extra(child))
@ -1092,7 +1092,7 @@ static size_t ts_subtree__write_to_string(
} }
else else
{ {
TSSymbol subtree_alias_symbol = t_symbol subtree_alias_symbol =
alias_sequence ? alias_sequence[structural_child_index] : 0; alias_sequence ? alias_sequence[structural_child_index] : 0;
bool subtree_alias_is_named = bool subtree_alias_is_named =
subtree_alias_symbol ? ts_language_symbol_metadata( subtree_alias_symbol ? ts_language_symbol_metadata(
@ -1101,7 +1101,7 @@ static size_t ts_subtree__write_to_string(
: false; : false;
const char *child_field_name = is_visible ? NULL : field_name; const char *child_field_name = is_visible ? NULL : field_name;
for (const TSFieldMapEntry *map = field_map; for (const t_field_map_entry *map = field_map;
map < field_map_end; map++) map < field_map_end; map++)
{ {
if (!map->inherited && if (!map->inherited &&
@ -1127,8 +1127,8 @@ static size_t ts_subtree__write_to_string(
return cursor - string; return cursor - string;
} }
char *ts_subtree_string(Subtree self, TSSymbol alias_symbol, char *ts_subtree_string(Subtree self, t_symbol alias_symbol,
bool alias_is_named, const TSLanguage *language, bool alias_is_named, const t_language *language,
bool include_all) bool include_all)
{ {
char scratch_string[1]; char scratch_string[1];
@ -1142,13 +1142,13 @@ char *ts_subtree_string(Subtree self, TSSymbol alias_symbol,
return result; return result;
} }
void ts_subtree__print_dot_graph(const Subtree *self, uint32_t start_offset, void ts_subtree__print_dot_graph(const Subtree *self, t_u32 start_offset,
const TSLanguage *language, const t_language *language,
TSSymbol alias_symbol, FILE *f) t_symbol alias_symbol, FILE *f)
{ {
TSSymbol subtree_symbol = ts_subtree_symbol(*self); t_symbol subtree_symbol = ts_subtree_symbol(*self);
TSSymbol symbol = alias_symbol ? alias_symbol : subtree_symbol; t_symbol symbol = alias_symbol ? alias_symbol : subtree_symbol;
uint32_t end_offset = start_offset + ts_subtree_total_bytes(*self); t_u32 end_offset = start_offset + ts_subtree_total_bytes(*self);
fprintf(f, "tree_%p [label=\"", (void *)self); fprintf(f, "tree_%p [label=\"", (void *)self);
ts_language_write_symbol_as_dot_string(language, f, symbol); ts_language_write_symbol_as_dot_string(language, f, symbol);
fprintf(f, "\""); fprintf(f, "\"");
@ -1182,13 +1182,13 @@ void ts_subtree__print_dot_graph(const Subtree *self, uint32_t start_offset,
fprintf(f, "\"]\n"); fprintf(f, "\"]\n");
uint32_t child_start_offset = start_offset; t_u32 child_start_offset = start_offset;
uint32_t child_info_offset = t_u32 child_info_offset =
language->max_alias_sequence_length * ts_subtree_production_id(*self); language->max_alias_sequence_length * ts_subtree_production_id(*self);
for (uint32_t i = 0, n = ts_subtree_child_count(*self); i < n; i++) for (t_u32 i = 0, n = ts_subtree_child_count(*self); i < n; i++)
{ {
const Subtree *child = &ts_subtree_children(*self)[i]; const Subtree *child = &ts_subtree_children(*self)[i];
TSSymbol subtree_alias_symbol = 0; t_symbol subtree_alias_symbol = 0;
if (!ts_subtree_extra(*child) && child_info_offset) if (!ts_subtree_extra(*child) && child_info_offset)
{ {
subtree_alias_symbol = language->alias_sequences[child_info_offset]; subtree_alias_symbol = language->alias_sequences[child_info_offset];
@ -1202,7 +1202,7 @@ void ts_subtree__print_dot_graph(const Subtree *self, uint32_t start_offset,
} }
} }
void ts_subtree_print_dot_graph(Subtree self, const TSLanguage *language, void ts_subtree_print_dot_graph(Subtree self, const t_language *language,
FILE *f) FILE *f)
{ {
fprintf(f, "digraph tree {\n"); fprintf(f, "digraph tree {\n");

545
parser/src/subtree.h
View file

@ -1,22 +1,21 @@
#ifndef TREE_SITTER_SUBTREE_H_ #ifndef TREE_SITTER_SUBTREE_H_
#define TREE_SITTER_SUBTREE_H_ #define TREE_SITTER_SUBTREE_H_
#ifdef __cplusplus #include "me/types.h"
extern "C" { #include "parser/types/types_symbol.h"
#endif
#include <limits.h>
#include <stdbool.h>
#include <stdio.h>
#include "./length.h"
#include "./array.h" #include "./array.h"
#include "./error_costs.h" #include "./error_costs.h"
#include "./host.h" #include "./host.h"
#include "parser/api.h" #include "./length.h"
#include "./parser.h" #include "./parser.h"
#include "parser/api.h"
#include <limits.h>
#include <stdbool.h>
#include <stdio.h>
#define TS_TREE_STATE_NONE USHRT_MAX #define TS_TREE_STATE_NONE USHRT_MAX
#define NULL_SUBTREE ((Subtree) {.ptr = NULL}) #define NULL_SUBTREE ((Subtree){.ptr = NULL})
// The serialized state of an external scanner. // The serialized state of an external scanner.
// //
@ -28,12 +27,13 @@ extern "C" {
// //
// Small byte arrays are stored inline, and long ones are allocated // Small byte arrays are stored inline, and long ones are allocated
// separately on the heap. // separately on the heap.
typedef struct { typedef struct
union { {
char *long_data; union {
char short_data[24]; char *long_data;
}; char short_data[24];
uint32_t length; };
t_u32 length;
} ExternalScannerState; } ExternalScannerState;
// A compact representation of a subtree. // A compact representation of a subtree.
@ -47,136 +47,108 @@ typedef struct {
// Because of alignment, for any valid pointer this will be 0, giving // Because of alignment, for any valid pointer this will be 0, giving
// us the opportunity to make use of this bit to signify whether to use // us the opportunity to make use of this bit to signify whether to use
// the pointer or the inline struct. // the pointer or the inline struct.
typedef struct SubtreeInlineData SubtreeInlineData; typedef struct s_subtree_inline_data t_subtree_inline_data;
#define SUBTREE_BITS \ struct s_subtree_inline_data
bool visible : 1; \ {
bool named : 1; \ bool is_inline : 1;
bool extra : 1; \ bool visible : 1;
bool has_changes : 1; \ bool named : 1;
bool is_missing : 1; \ bool extra : 1;
bool is_keyword : 1; bool has_changes : 1;
bool is_missing : 1;
#define SUBTREE_SIZE \ bool is_keyword : 1;
uint8_t padding_columns; \ t_u8 symbol;
uint8_t padding_rows : 4; \ t_u16 parse_state;
uint8_t lookahead_bytes : 4; \ t_u8 padding_columns;
uint8_t padding_bytes; \ t_u8 padding_rows : 4;
uint8_t size_bytes; t_u8 lookahead_bytes : 4;
t_u8 padding_bytes;
#if TS_BIG_ENDIAN t_u8 size_bytes;
#if TS_PTR_SIZE == 32
struct SubtreeInlineData {
uint16_t parse_state;
uint8_t symbol;
SUBTREE_BITS
bool unused : 1;
bool is_inline : 1;
SUBTREE_SIZE
}; };
#else
struct SubtreeInlineData {
SUBTREE_SIZE
uint16_t parse_state;
uint8_t symbol;
SUBTREE_BITS
bool unused : 1;
bool is_inline : 1;
};
#endif
#else
struct SubtreeInlineData {
bool is_inline : 1;
SUBTREE_BITS
uint8_t symbol;
uint16_t parse_state;
SUBTREE_SIZE
};
#endif
#undef SUBTREE_BITS
#undef SUBTREE_SIZE
// A heap-allocated representation of a subtree. // A heap-allocated representation of a subtree.
// //
// This representation is used for parent nodes, external tokens, // This representation is used for parent nodes, external tokens,
// errors, and other leaf nodes whose data is too large to fit into // errors, and other leaf nodes whose data is too large to fit into
// the inline representation. // the inline representation.
typedef struct { typedef struct
volatile uint32_t ref_count; {
Length padding; volatile t_u32 ref_count;
Length size; Length padding;
uint32_t lookahead_bytes; Length size;
uint32_t error_cost; t_u32 lookahead_bytes;
uint32_t child_count; t_u32 error_cost;
TSSymbol symbol; t_u32 child_count;
TSStateId parse_state; t_symbol symbol;
t_state_id parse_state;
bool visible : 1; bool visible : 1;
bool named : 1; bool named : 1;
bool extra : 1; bool extra : 1;
bool fragile_left : 1; bool fragile_left : 1;
bool fragile_right : 1; bool fragile_right : 1;
bool has_changes : 1; bool has_changes : 1;
bool has_external_tokens : 1; bool has_external_tokens : 1;
bool has_external_scanner_state_change : 1; bool has_external_scanner_state_change : 1;
bool depends_on_column: 1; bool depends_on_column : 1;
bool is_missing : 1; bool is_missing : 1;
bool is_keyword : 1; bool is_keyword : 1;
union { union {
// Non-terminal subtrees (`child_count > 0`) // Non-terminal subtrees (`child_count > 0`)
struct { struct
uint32_t visible_child_count; {
uint32_t named_child_count; t_u32 visible_child_count;
uint32_t visible_descendant_count; t_u32 named_child_count;
int32_t dynamic_precedence; t_u32 visible_descendant_count;
uint16_t repeat_depth; t_i32 dynamic_precedence;
uint16_t production_id; t_u16 repeat_depth;
struct { t_u16 production_id;
TSSymbol symbol; struct
TSStateId parse_state; {
} first_leaf; t_symbol symbol;
}; t_state_id parse_state;
} first_leaf;
};
// External terminal subtrees (`child_count == 0 && has_external_tokens`) // External terminal subtrees (`child_count == 0 &&
ExternalScannerState external_scanner_state; // has_external_tokens`)
ExternalScannerState external_scanner_state;
// Error terminal subtrees (`child_count == 0 && symbol == ts_builtin_sym_error`) // Error terminal subtrees (`child_count == 0 && symbol ==
int32_t lookahead_char; // ts_builtin_sym_error`)
}; t_i32 lookahead_char;
};
} SubtreeHeapData; } SubtreeHeapData;
// The fundamental building block of a syntax tree. // The fundamental building block of a syntax tree.
typedef union { typedef union {
SubtreeInlineData data; t_subtree_inline_data data;
const SubtreeHeapData *ptr; const SubtreeHeapData *ptr;
} Subtree; } Subtree;
// Like Subtree, but mutable. // Like Subtree, but mutable.
typedef union { typedef union {
SubtreeInlineData data; t_subtree_inline_data data;
SubtreeHeapData *ptr; SubtreeHeapData *ptr;
} MutableSubtree; } MutableSubtree;
typedef Array(Subtree) SubtreeArray; typedef Array(Subtree) SubtreeArray;
typedef Array(MutableSubtree) MutableSubtreeArray; typedef Array(MutableSubtree) MutableSubtreeArray;
typedef struct { typedef struct
MutableSubtreeArray free_trees; {
MutableSubtreeArray tree_stack; MutableSubtreeArray free_trees;
MutableSubtreeArray tree_stack;
} SubtreePool; } SubtreePool;
void ts_external_scanner_state_init(ExternalScannerState *, const char *, unsigned); void ts_external_scanner_state_init(ExternalScannerState *, const char *,
unsigned);
const char *ts_external_scanner_state_data(const ExternalScannerState *); const char *ts_external_scanner_state_data(const ExternalScannerState *);
bool ts_external_scanner_state_eq(const ExternalScannerState *self, const char *, unsigned); bool ts_external_scanner_state_eq(const ExternalScannerState *self,
void ts_external_scanner_state_delete(ExternalScannerState *self); const char *, unsigned);
void ts_external_scanner_state_delete(ExternalScannerState *self);
void ts_subtree_array_copy(SubtreeArray, SubtreeArray *); void ts_subtree_array_copy(SubtreeArray, SubtreeArray *);
void ts_subtree_array_clear(SubtreePool *, SubtreeArray *); void ts_subtree_array_clear(SubtreePool *, SubtreeArray *);
@ -184,199 +156,280 @@ void ts_subtree_array_delete(SubtreePool *, SubtreeArray *);
void ts_subtree_array_remove_trailing_extras(SubtreeArray *, SubtreeArray *); void ts_subtree_array_remove_trailing_extras(SubtreeArray *, SubtreeArray *);
void ts_subtree_array_reverse(SubtreeArray *); void ts_subtree_array_reverse(SubtreeArray *);
SubtreePool ts_subtree_pool_new(uint32_t capacity); SubtreePool ts_subtree_pool_new(t_u32 capacity);
void ts_subtree_pool_delete(SubtreePool *); void ts_subtree_pool_delete(SubtreePool *);
Subtree ts_subtree_new_leaf( Subtree ts_subtree_new_leaf(SubtreePool *, t_symbol, Length, Length, t_u32,
SubtreePool *, TSSymbol, Length, Length, uint32_t, t_state_id, bool, bool, bool, const t_language *);
TSStateId, bool, bool, bool, const TSLanguage * Subtree ts_subtree_new_error(SubtreePool *, t_i32, Length, Length, t_u32,
); t_state_id, const t_language *);
Subtree ts_subtree_new_error( MutableSubtree ts_subtree_new_node(t_symbol, SubtreeArray *, unsigned,
SubtreePool *, int32_t, Length, Length, uint32_t, TSStateId, const TSLanguage * const t_language *);
); Subtree ts_subtree_new_error_node(SubtreeArray *, bool, const t_language *);
MutableSubtree ts_subtree_new_node(TSSymbol, SubtreeArray *, unsigned, const TSLanguage *); Subtree ts_subtree_new_missing_leaf(SubtreePool *, t_symbol, Length, t_u32,
Subtree ts_subtree_new_error_node(SubtreeArray *, bool, const TSLanguage *); const t_language *);
Subtree ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, Length, uint32_t, const TSLanguage *);
MutableSubtree ts_subtree_make_mut(SubtreePool *, Subtree); MutableSubtree ts_subtree_make_mut(SubtreePool *, Subtree);
void ts_subtree_retain(Subtree); void ts_subtree_retain(Subtree);
void ts_subtree_release(SubtreePool *, Subtree); void ts_subtree_release(SubtreePool *, Subtree);
int ts_subtree_compare(Subtree, Subtree, SubtreePool *); int ts_subtree_compare(Subtree, Subtree, SubtreePool *);
void ts_subtree_set_symbol(MutableSubtree *, TSSymbol, const TSLanguage *); void ts_subtree_set_symbol(MutableSubtree *, t_symbol, const t_language *);
void ts_subtree_summarize(MutableSubtree, const Subtree *, uint32_t, const TSLanguage *); void ts_subtree_summarize(MutableSubtree, const Subtree *, t_u32,
void ts_subtree_summarize_children(MutableSubtree, const TSLanguage *); const t_language *);
void ts_subtree_balance(Subtree, SubtreePool *, const TSLanguage *); void ts_subtree_summarize_children(MutableSubtree, const t_language *);
Subtree ts_subtree_edit(Subtree, const TSInputEdit *edit, SubtreePool *); void ts_subtree_balance(Subtree, SubtreePool *, const t_language *);
char *ts_subtree_string(Subtree, TSSymbol, bool, const TSLanguage *, bool include_all); Subtree ts_subtree_edit(Subtree, const t_input_edit *edit, SubtreePool *);
void ts_subtree_print_dot_graph(Subtree, const TSLanguage *, FILE *); char *ts_subtree_string(Subtree, t_symbol, bool, const t_language *,
bool include_all);
void ts_subtree_print_dot_graph(Subtree, const t_language *, FILE *);
Subtree ts_subtree_last_external_token(Subtree); Subtree ts_subtree_last_external_token(Subtree);
const ExternalScannerState *ts_subtree_external_scanner_state(Subtree self); const ExternalScannerState *ts_subtree_external_scanner_state(Subtree self);
bool ts_subtree_external_scanner_state_eq(Subtree, Subtree); bool ts_subtree_external_scanner_state_eq(Subtree, Subtree);
#define SUBTREE_GET(self, name) ((self).data.is_inline ? (self).data.name : (self).ptr->name) #define SUBTREE_GET(self, name) \
((self).data.is_inline ? (self).data.name : (self).ptr->name)
static inline TSSymbol ts_subtree_symbol(Subtree self) { return SUBTREE_GET(self, symbol); } static inline t_symbol ts_subtree_symbol(Subtree self)
static inline bool ts_subtree_visible(Subtree self) { return SUBTREE_GET(self, visible); } {
static inline bool ts_subtree_named(Subtree self) { return SUBTREE_GET(self, named); } return SUBTREE_GET(self, symbol);
static inline bool ts_subtree_extra(Subtree self) { return SUBTREE_GET(self, extra); } }
static inline bool ts_subtree_has_changes(Subtree self) { return SUBTREE_GET(self, has_changes); } static inline bool ts_subtree_visible(Subtree self)
static inline bool ts_subtree_missing(Subtree self) { return SUBTREE_GET(self, is_missing); } {
static inline bool ts_subtree_is_keyword(Subtree self) { return SUBTREE_GET(self, is_keyword); } return SUBTREE_GET(self, visible);
static inline TSStateId ts_subtree_parse_state(Subtree self) { return SUBTREE_GET(self, parse_state); } }
static inline uint32_t ts_subtree_lookahead_bytes(Subtree self) { return SUBTREE_GET(self, lookahead_bytes); } static inline bool ts_subtree_named(Subtree self)
{
return SUBTREE_GET(self, named);
}
static inline bool ts_subtree_extra(Subtree self)
{
return SUBTREE_GET(self, extra);
}
static inline bool ts_subtree_has_changes(Subtree self)
{
return SUBTREE_GET(self, has_changes);
}
static inline bool ts_subtree_missing(Subtree self)
{
return SUBTREE_GET(self, is_missing);
}
static inline bool ts_subtree_is_keyword(Subtree self)
{
return SUBTREE_GET(self, is_keyword);
}
static inline t_state_id ts_subtree_parse_state(Subtree self)
{
return SUBTREE_GET(self, parse_state);
}
static inline t_u32 ts_subtree_lookahead_bytes(Subtree self)
{
return SUBTREE_GET(self, lookahead_bytes);
}
#undef SUBTREE_GET #undef SUBTREE_GET
// Get the size needed to store a heap-allocated subtree with the given // Get the size needed to store a heap-allocated subtree with the given
// number of children. // number of children.
static inline size_t ts_subtree_alloc_size(uint32_t child_count) { static inline size_t ts_subtree_alloc_size(t_u32 child_count)
return child_count * sizeof(Subtree) + sizeof(SubtreeHeapData); {
return child_count * sizeof(Subtree) + sizeof(SubtreeHeapData);
} }
// Get a subtree's children, which are allocated immediately before the // Get a subtree's children, which are allocated immediately before the
// tree's own heap data. // tree's own heap data.
#define ts_subtree_children(self) \ #define ts_subtree_children(self) \
((self).data.is_inline ? NULL : (Subtree *)((self).ptr) - (self).ptr->child_count) ((self).data.is_inline \
? NULL \
: (Subtree *)((self).ptr) - (self).ptr->child_count)
static inline void ts_subtree_set_extra(MutableSubtree *self, bool is_extra) { static inline void ts_subtree_set_extra(MutableSubtree *self, bool is_extra)
if (self->data.is_inline) { {
self->data.extra = is_extra; if (self->data.is_inline)
} else { {
self->ptr->extra = is_extra; self->data.extra = is_extra;
} }
else
{
self->ptr->extra = is_extra;
}
} }
static inline TSSymbol ts_subtree_leaf_symbol(Subtree self) { static inline t_symbol ts_subtree_leaf_symbol(Subtree self)
if (self.data.is_inline) return self.data.symbol; {
if (self.ptr->child_count == 0) return self.ptr->symbol; if (self.data.is_inline)
return self.ptr->first_leaf.symbol; return self.data.symbol;
if (self.ptr->child_count == 0)
return self.ptr->symbol;
return self.ptr->first_leaf.symbol;
} }
static inline TSStateId ts_subtree_leaf_parse_state(Subtree self) { static inline t_state_id ts_subtree_leaf_parse_state(Subtree self)
if (self.data.is_inline) return self.data.parse_state; {
if (self.ptr->child_count == 0) return self.ptr->parse_state; if (self.data.is_inline)
return self.ptr->first_leaf.parse_state; return self.data.parse_state;
if (self.ptr->child_count == 0)
return self.ptr->parse_state;
return self.ptr->first_leaf.parse_state;
} }
static inline Length ts_subtree_padding(Subtree self) { static inline Length ts_subtree_padding(Subtree self)
if (self.data.is_inline) { {
Length result = {self.data.padding_bytes, {self.data.padding_rows, self.data.padding_columns}}; if (self.data.is_inline)
return result; {
} else { Length result = {self.data.padding_bytes,
return self.ptr->padding; {self.data.padding_rows, self.data.padding_columns}};
} return result;
}
else
{
return self.ptr->padding;
}
} }
static inline Length ts_subtree_size(Subtree self) { static inline Length ts_subtree_size(Subtree self)
if (self.data.is_inline) { {
Length result = {self.data.size_bytes, {0, self.data.size_bytes}}; if (self.data.is_inline)
return result; {
} else { Length result = {self.data.size_bytes, {0, self.data.size_bytes}};
return self.ptr->size; return result;
} }
else
{
return self.ptr->size;
}
} }
static inline Length ts_subtree_total_size(Subtree self) { static inline Length ts_subtree_total_size(Subtree self)
return length_add(ts_subtree_padding(self), ts_subtree_size(self)); {
return length_add(ts_subtree_padding(self), ts_subtree_size(self));
} }
static inline uint32_t ts_subtree_total_bytes(Subtree self) { static inline t_u32 ts_subtree_total_bytes(Subtree self)
return ts_subtree_total_size(self).bytes; {
return ts_subtree_total_size(self).bytes;
} }
static inline uint32_t ts_subtree_child_count(Subtree self) { static inline t_u32 ts_subtree_child_count(Subtree self)
return self.data.is_inline ? 0 : self.ptr->child_count; {
return self.data.is_inline ? 0 : self.ptr->child_count;
} }
static inline uint32_t ts_subtree_repeat_depth(Subtree self) { static inline t_u32 ts_subtree_repeat_depth(Subtree self)
return self.data.is_inline ? 0 : self.ptr->repeat_depth; {
return self.data.is_inline ? 0 : self.ptr->repeat_depth;
} }
static inline uint32_t ts_subtree_is_repetition(Subtree self) { static inline t_u32 ts_subtree_is_repetition(Subtree self)
return self.data.is_inline {
? 0 return self.data.is_inline ? 0
: !self.ptr->named && !self.ptr->visible && self.ptr->child_count != 0; : !self.ptr->named && !self.ptr->visible &&
self.ptr->child_count != 0;
} }
static inline uint32_t ts_subtree_visible_descendant_count(Subtree self) { static inline t_u32 ts_subtree_visible_descendant_count(Subtree self)
return (self.data.is_inline || self.ptr->child_count == 0) {
? 0 return (self.data.is_inline || self.ptr->child_count == 0)
: self.ptr->visible_descendant_count; ? 0
: self.ptr->visible_descendant_count;
} }
static inline uint32_t ts_subtree_visible_child_count(Subtree self) { static inline t_u32 ts_subtree_visible_child_count(Subtree self)
if (ts_subtree_child_count(self) > 0) { {
return self.ptr->visible_child_count; if (ts_subtree_child_count(self) > 0)
} else { {
return 0; return self.ptr->visible_child_count;
} }
else
{
return 0;
}
} }
static inline uint32_t ts_subtree_error_cost(Subtree self) { static inline t_u32 ts_subtree_error_cost(Subtree self)
if (ts_subtree_missing(self)) { {
return ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY; if (ts_subtree_missing(self))
} else { {
return self.data.is_inline ? 0 : self.ptr->error_cost; return ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY;
} }
else
{
return self.data.is_inline ? 0 : self.ptr->error_cost;
}
} }
static inline int32_t ts_subtree_dynamic_precedence(Subtree self) { static inline t_i32 ts_subtree_dynamic_precedence(Subtree self)
return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->dynamic_precedence; {
return (self.data.is_inline || self.ptr->child_count == 0)
? 0
: self.ptr->dynamic_precedence;
} }
static inline uint16_t ts_subtree_production_id(Subtree self) { static inline t_u16 ts_subtree_production_id(Subtree self)
if (ts_subtree_child_count(self) > 0) { {
return self.ptr->production_id; if (ts_subtree_child_count(self) > 0)
} else { {
return 0; return self.ptr->production_id;
} }
else
{
return 0;
}
} }
static inline bool ts_subtree_fragile_left(Subtree self) { static inline bool ts_subtree_fragile_left(Subtree self)
return self.data.is_inline ? false : self.ptr->fragile_left; {
return self.data.is_inline ? false : self.ptr->fragile_left;
} }
static inline bool ts_subtree_fragile_right(Subtree self) { static inline bool ts_subtree_fragile_right(Subtree self)
return self.data.is_inline ? false : self.ptr->fragile_right; {
return self.data.is_inline ? false : self.ptr->fragile_right;
} }
static inline bool ts_subtree_has_external_tokens(Subtree self) { static inline bool ts_subtree_has_external_tokens(Subtree self)
return self.data.is_inline ? false : self.ptr->has_external_tokens; {
return self.data.is_inline ? false : self.ptr->has_external_tokens;
} }
static inline bool ts_subtree_has_external_scanner_state_change(Subtree self) { static inline bool ts_subtree_has_external_scanner_state_change(Subtree self)
return self.data.is_inline ? false : self.ptr->has_external_scanner_state_change; {
return self.data.is_inline ? false
: self.ptr->has_external_scanner_state_change;
} }
static inline bool ts_subtree_depends_on_column(Subtree self) { static inline bool ts_subtree_depends_on_column(Subtree self)
return self.data.is_inline ? false : self.ptr->depends_on_column; {
return self.data.is_inline ? false : self.ptr->depends_on_column;
} }
static inline bool ts_subtree_is_fragile(Subtree self) { static inline bool ts_subtree_is_fragile(Subtree self)
return self.data.is_inline ? false : (self.ptr->fragile_left || self.ptr->fragile_right); {
return self.data.is_inline
? false
: (self.ptr->fragile_left || self.ptr->fragile_right);
} }
static inline bool ts_subtree_is_error(Subtree self) { static inline bool ts_subtree_is_error(Subtree self)
return ts_subtree_symbol(self) == ts_builtin_sym_error; {
return ts_subtree_symbol(self) == ts_builtin_sym_error;
} }
static inline bool ts_subtree_is_eof(Subtree self) { static inline bool ts_subtree_is_eof(Subtree self)
return ts_subtree_symbol(self) == ts_builtin_sym_end; {
return ts_subtree_symbol(self) == ts_builtin_sym_end;
} }
static inline Subtree ts_subtree_from_mut(MutableSubtree self) { static inline Subtree ts_subtree_from_mut(MutableSubtree self)
Subtree result; {
result.data = self.data; Subtree result;
return result; result.data = self.data;
return result;
} }
static inline MutableSubtree ts_subtree_to_mut_unsafe(Subtree self) { static inline MutableSubtree ts_subtree_to_mt_unsafe(Subtree self)
MutableSubtree result; {
result.data = self.data; MutableSubtree result;
return result; result.data = self.data;
return result;
} }
#ifdef __cplusplus #endif // TREE_SITTER_SUBTREE_H_
}
#endif
#endif // TREE_SITTER_SUBTREE_H_

28
parser/src/tree.c
View file

@ -8,11 +8,11 @@
#include "./tree_cursor.h" #include "./tree_cursor.h"
#include "./tree.h" #include "./tree.h"
TSTree *ts_tree_new( t_parse_tree *ts_tree_new(
Subtree root, const TSLanguage *language, Subtree root, const t_language *language,
const t_parser_range *included_ranges, unsigned included_range_count const t_parser_range *included_ranges, unsigned included_range_count
) { ) {
TSTree *result = malloc(sizeof(TSTree)); t_parse_tree *result = malloc(sizeof(t_parse_tree));
result->root = root; result->root = root;
result->language = ts_language_copy(language); result->language = ts_language_copy(language);
result->included_ranges = calloc(included_range_count, sizeof(t_parser_range)); result->included_ranges = calloc(included_range_count, sizeof(t_parser_range));
@ -21,12 +21,12 @@ TSTree *ts_tree_new(
return result; return result;
} }
TSTree *ts_tree_copy(const TSTree *self) { t_parse_tree *ts_tree_copy(const t_parse_tree *self) {
ts_subtree_retain(self->root); ts_subtree_retain(self->root);
return ts_tree_new(self->root, self->language, self->included_ranges, self->included_range_count); return ts_tree_new(self->root, self->language, self->included_ranges, self->included_range_count);
} }
void ts_tree_delete(TSTree *self) { void ts_tree_delete(t_parse_tree *self) {
if (!self) return; if (!self) return;
SubtreePool pool = ts_subtree_pool_new(0); SubtreePool pool = ts_subtree_pool_new(0);
@ -37,24 +37,24 @@ void ts_tree_delete(TSTree *self) {
free(self); free(self);
} }
TSNode ts_tree_root_node(const TSTree *self) { t_parse_node ts_tree_root_node(const t_parse_tree *self) {
return ts_node_new(self, &self->root, ts_subtree_padding(self->root), 0); return ts_node_new(self, &self->root, ts_subtree_padding(self->root), 0);
} }
TSNode ts_tree_root_node_with_offset( t_parse_node ts_tree_root_node_with_offset(
const TSTree *self, const t_parse_tree *self,
uint32_t offset_bytes, t_u32 offset_bytes,
t_point offset_extent t_point offset_extent
) { ) {
Length offset = {offset_bytes, offset_extent}; Length offset = {offset_bytes, offset_extent};
return ts_node_new(self, &self->root, length_add(offset, ts_subtree_padding(self->root)), 0); return ts_node_new(self, &self->root, length_add(offset, ts_subtree_padding(self->root)), 0);
} }
const TSLanguage *ts_tree_language(const TSTree *self) { const t_language *ts_tree_language(const t_parse_tree *self) {
return self->language; return self->language;
} }
void ts_tree_edit(TSTree *self, const TSInputEdit *edit) { void ts_tree_edit(t_parse_tree *self, const t_input_edit *edit) {
for (unsigned i = 0; i < self->included_range_count; i++) { for (unsigned i = 0; i < self->included_range_count; i++) {
t_parser_range *range = &self->included_ranges[i]; t_parser_range *range = &self->included_ranges[i];
if (range->end_byte >= edit->old_end_byte) { if (range->end_byte >= edit->old_end_byte) {
@ -94,7 +94,7 @@ void ts_tree_edit(TSTree *self, const TSInputEdit *edit) {
ts_subtree_pool_delete(&pool); ts_subtree_pool_delete(&pool);
} }
t_parser_range *ts_tree_included_ranges(const TSTree *self, uint32_t *length) { t_parser_range *ts_tree_included_ranges(const t_parse_tree *self, t_u32 *length) {
*length = self->included_range_count; *length = self->included_range_count;
t_parser_range *ranges = calloc(self->included_range_count, sizeof(t_parser_range)); t_parser_range *ranges = calloc(self->included_range_count, sizeof(t_parser_range));
memcpy(ranges, self->included_ranges, self->included_range_count * sizeof(t_parser_range)); memcpy(ranges, self->included_ranges, self->included_range_count * sizeof(t_parser_range));
@ -117,7 +117,7 @@ int _ts_dup(HANDLE handle) {
return _open_osfhandle((intptr_t)dup_handle, 0); return _open_osfhandle((intptr_t)dup_handle, 0);
} }
void ts_tree_print_dot_graph(const TSTree *self, int fd) { void ts_tree_print_dot_graph(const t_parse_tree *self, int fd) {
FILE *file = _fdopen(_ts_dup((HANDLE)_get_osfhandle(fd)), "a"); FILE *file = _fdopen(_ts_dup((HANDLE)_get_osfhandle(fd)), "a");
ts_subtree_print_dot_graph(self->root, self->language, file); ts_subtree_print_dot_graph(self->root, self->language, file);
fclose(file); fclose(file);
@ -131,7 +131,7 @@ int _ts_dup(int file_descriptor) {
return dup(file_descriptor); return dup(file_descriptor);
} }
void ts_tree_print_dot_graph(const TSTree *self, int file_descriptor) { void ts_tree_print_dot_graph(const t_parse_tree *self, int file_descriptor) {
FILE *file = fdopen(_ts_dup(file_descriptor), "a"); FILE *file = fdopen(_ts_dup(file_descriptor), "a");
ts_subtree_print_dot_graph(self->root, self->language, file); ts_subtree_print_dot_graph(self->root, self->language, file);
fclose(file); fclose(file);

10
parser/src/tree.h
View file

@ -11,18 +11,18 @@ typedef struct {
const Subtree *child; const Subtree *child;
const Subtree *parent; const Subtree *parent;
Length position; Length position;
TSSymbol alias_symbol; t_symbol alias_symbol;
} ParentCacheEntry; } ParentCacheEntry;
struct TSTree { struct t_parse_tree {
Subtree root; Subtree root;
const TSLanguage *language; const t_language *language;
t_parser_range *included_ranges; t_parser_range *included_ranges;
unsigned included_range_count; unsigned included_range_count;
}; };
TSTree *ts_tree_new(Subtree root, const TSLanguage *language, const t_parser_range *, unsigned); t_parse_tree *ts_tree_new(Subtree root, const t_language *language, const t_parser_range *, unsigned);
TSNode ts_node_new(const TSTree *, const Subtree *, Length, TSSymbol); t_parse_node ts_node_new(const t_parse_tree *, const Subtree *, Length, t_symbol);
#ifdef __cplusplus #ifdef __cplusplus
} }

132
parser/src/tree_cursor.c
View file

@ -6,17 +6,17 @@
typedef struct { typedef struct {
Subtree parent; Subtree parent;
const TSTree *tree; const t_parse_tree *tree;
Length position; Length position;
uint32_t child_index; t_u32 child_index;
uint32_t structural_child_index; t_u32 structural_child_index;
uint32_t descendant_index; t_u32 descendant_index;
const TSSymbol *alias_sequence; const t_symbol *alias_sequence;
} CursorChildIterator; } CursorChildIterator;
// CursorChildIterator // CursorChildIterator
static inline bool ts_tree_cursor_is_entry_visible(const TreeCursor *self, uint32_t index) { static inline bool ts_tree_cursor_is_entry_visible(const TreeCursor *self, t_u32 index) {
TreeCursorEntry *entry = &self->stack.contents[index]; TreeCursorEntry *entry = &self->stack.contents[index];
if (index == 0 || ts_subtree_visible(*entry->subtree)) { if (index == 0 || ts_subtree_visible(*entry->subtree)) {
return true; return true;
@ -37,12 +37,12 @@ static inline CursorChildIterator ts_tree_cursor_iterate_children(const TreeCurs
if (ts_subtree_child_count(*last_entry->subtree) == 0) { if (ts_subtree_child_count(*last_entry->subtree) == 0) {
return (CursorChildIterator) {NULL_SUBTREE, self->tree, length_zero(), 0, 0, 0, NULL}; return (CursorChildIterator) {NULL_SUBTREE, self->tree, length_zero(), 0, 0, 0, NULL};
} }
const TSSymbol *alias_sequence = ts_language_alias_sequence( const t_symbol *alias_sequence = ts_language_alias_sequence(
self->tree->language, self->tree->language,
last_entry->subtree->ptr->production_id last_entry->subtree->ptr->production_id
); );
uint32_t descendant_index = last_entry->descendant_index; t_u32 descendant_index = last_entry->descendant_index;
if (ts_tree_cursor_is_entry_visible(self, self->stack.size - 1)) { if (ts_tree_cursor_is_entry_visible(self, self->stack.size - 1)) {
descendant_index += 1; descendant_index += 1;
} }
@ -120,7 +120,7 @@ static inline bool ts_tree_cursor_child_iterator_previous(
) { ) {
// this is mostly a reverse `ts_tree_cursor_child_iterator_next` taking into // this is mostly a reverse `ts_tree_cursor_child_iterator_next` taking into
// account unsigned underflow // account unsigned underflow
if (!self->parent.ptr || (int8_t)self->child_index == -1) return false; if (!self->parent.ptr || (t_i8)self->child_index == -1) return false;
const Subtree *child = &ts_subtree_children(self->parent)[self->child_index]; const Subtree *child = &ts_subtree_children(self->parent)[self->child_index];
*result = (TreeCursorEntry) { *result = (TreeCursorEntry) {
.subtree = child, .subtree = child,
@ -148,19 +148,19 @@ static inline bool ts_tree_cursor_child_iterator_previous(
return true; return true;
} }
// TSTreeCursor - lifecycle // t_parse_tree_cursor - lifecycle
TSTreeCursor ts_tree_cursor_new(TSNode node) { t_parse_tree_cursor ts_tree_cursor_new(t_parse_node node) {
TSTreeCursor self = {NULL, NULL, {0, 0, 0}}; t_parse_tree_cursor self = {NULL, NULL, {0, 0, 0}};
ts_tree_cursor_init((TreeCursor *)&self, node); ts_tree_cursor_init((TreeCursor *)&self, node);
return self; return self;
} }
void ts_tree_cursor_reset(TSTreeCursor *_self, TSNode node) { void ts_tree_cursor_reset(t_parse_tree_cursor *_self, t_parse_node node) {
ts_tree_cursor_init((TreeCursor *)_self, node); ts_tree_cursor_init((TreeCursor *)_self, node);
} }
void ts_tree_cursor_init(TreeCursor *self, TSNode node) { void ts_tree_cursor_init(TreeCursor *self, t_parse_node node) {
self->tree = node.tree; self->tree = node.tree;
self->root_alias_symbol = node.context[3]; self->root_alias_symbol = node.context[3];
array_clear(&self->stack); array_clear(&self->stack);
@ -176,14 +176,14 @@ void ts_tree_cursor_init(TreeCursor *self, TSNode node) {
})); }));
} }
void ts_tree_cursor_delete(TSTreeCursor *_self) { void ts_tree_cursor_delete(t_parse_tree_cursor *_self) {
TreeCursor *self = (TreeCursor *)_self; TreeCursor *self = (TreeCursor *)_self;
array_delete(&self->stack); array_delete(&self->stack);
} }
// TSTreeCursor - walking the tree // t_parse_tree_cursor - walking the tree
TreeCursorStep ts_tree_cursor_goto_first_child_internal(TSTreeCursor *_self) { TreeCursorStep ts_tree_cursor_goto_first_child_internal(t_parse_tree_cursor *_self) {
TreeCursor *self = (TreeCursor *)_self; TreeCursor *self = (TreeCursor *)_self;
bool visible; bool visible;
TreeCursorEntry entry; TreeCursorEntry entry;
@ -201,7 +201,7 @@ TreeCursorStep ts_tree_cursor_goto_first_child_internal(TSTreeCursor *_self) {
return TreeCursorStepNone; return TreeCursorStepNone;
} }
bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { bool ts_tree_cursor_goto_first_child(t_parse_tree_cursor *self) {
for (;;) { for (;;) {
switch (ts_tree_cursor_goto_first_child_internal(self)) { switch (ts_tree_cursor_goto_first_child_internal(self)) {
case TreeCursorStepHidden: case TreeCursorStepHidden:
@ -215,7 +215,7 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) {
return false; return false;
} }
TreeCursorStep ts_tree_cursor_goto_last_child_internal(TSTreeCursor *_self) { TreeCursorStep ts_tree_cursor_goto_last_child_internal(t_parse_tree_cursor *_self) {
TreeCursor *self = (TreeCursor *)_self; TreeCursor *self = (TreeCursor *)_self;
bool visible; bool visible;
TreeCursorEntry entry; TreeCursorEntry entry;
@ -242,7 +242,7 @@ TreeCursorStep ts_tree_cursor_goto_last_child_internal(TSTreeCursor *_self) {
return TreeCursorStepNone; return TreeCursorStepNone;
} }
bool ts_tree_cursor_goto_last_child(TSTreeCursor *self) { bool ts_tree_cursor_goto_last_child(t_parse_tree_cursor *self) {
for (;;) { for (;;) {
switch (ts_tree_cursor_goto_last_child_internal(self)) { switch (ts_tree_cursor_goto_last_child_internal(self)) {
case TreeCursorStepHidden: case TreeCursorStepHidden:
@ -256,14 +256,14 @@ bool ts_tree_cursor_goto_last_child(TSTreeCursor *self) {
return false; return false;
} }
static inline int64_t ts_tree_cursor_goto_first_child_for_byte_and_point( static inline t_i64 ts_tree_cursor_goto_first_child_for_byte_and_point(
TSTreeCursor *_self, t_parse_tree_cursor *_self,
uint32_t goal_byte, t_u32 goal_byte,
t_point goal_point t_point goal_point
) { ) {
TreeCursor *self = (TreeCursor *)_self; TreeCursor *self = (TreeCursor *)_self;
uint32_t initial_size = self->stack.size; t_u32 initial_size = self->stack.size;
uint32_t visible_child_index = 0; t_u32 visible_child_index = 0;
bool did_descend; bool did_descend;
do { do {
@ -275,7 +275,7 @@ static inline int64_t ts_tree_cursor_goto_first_child_for_byte_and_point(
while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) {
Length entry_end = length_add(entry.position, ts_subtree_size(*entry.subtree)); Length entry_end = length_add(entry.position, ts_subtree_size(*entry.subtree));
bool at_goal = entry_end.bytes >= goal_byte && point_gte(entry_end.extent, goal_point); bool at_goal = entry_end.bytes >= goal_byte && point_gte(entry_end.extent, goal_point);
uint32_t visible_child_count = ts_subtree_visible_child_count(*entry.subtree); t_u32 visible_child_count = ts_subtree_visible_child_count(*entry.subtree);
if (at_goal) { if (at_goal) {
if (visible) { if (visible) {
array_push(&self->stack, entry); array_push(&self->stack, entry);
@ -298,19 +298,19 @@ static inline int64_t ts_tree_cursor_goto_first_child_for_byte_and_point(
return -1; return -1;
} }
int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *self, uint32_t goal_byte) { t_i64 ts_tree_cursor_goto_first_child_for_byte(t_parse_tree_cursor *self, t_u32 goal_byte) {
return ts_tree_cursor_goto_first_child_for_byte_and_point(self, goal_byte, POINT_ZERO); return ts_tree_cursor_goto_first_child_for_byte_and_point(self, goal_byte, POINT_ZERO);
} }
int64_t ts_tree_cursor_goto_first_child_for_point(TSTreeCursor *self, t_point goal_point) { t_i64 ts_tree_cursor_goto_first_child_for_point(t_parse_tree_cursor *self, t_point goal_point) {
return ts_tree_cursor_goto_first_child_for_byte_and_point(self, 0, goal_point); return ts_tree_cursor_goto_first_child_for_byte_and_point(self, 0, goal_point);
} }
TreeCursorStep ts_tree_cursor_goto_sibling_internal( TreeCursorStep ts_tree_cursor_goto_sibling_internal(
TSTreeCursor *_self, t_parse_tree_cursor *_self,
bool (*advance)(CursorChildIterator *, TreeCursorEntry *, bool *)) { bool (*advance)(CursorChildIterator *, TreeCursorEntry *, bool *)) {
TreeCursor *self = (TreeCursor *)_self; TreeCursor *self = (TreeCursor *)_self;
uint32_t initial_size = self->stack.size; t_u32 initial_size = self->stack.size;
while (self->stack.size > 1) { while (self->stack.size > 1) {
TreeCursorEntry entry = array_pop(&self->stack); TreeCursorEntry entry = array_pop(&self->stack);
@ -341,11 +341,11 @@ TreeCursorStep ts_tree_cursor_goto_sibling_internal(
return TreeCursorStepNone; return TreeCursorStepNone;
} }
TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *_self) { TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(t_parse_tree_cursor *_self) {
return ts_tree_cursor_goto_sibling_internal(_self, ts_tree_cursor_child_iterator_next); return ts_tree_cursor_goto_sibling_internal(_self, ts_tree_cursor_child_iterator_next);
} }
bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { bool ts_tree_cursor_goto_next_sibling(t_parse_tree_cursor *self) {
switch (ts_tree_cursor_goto_next_sibling_internal(self)) { switch (ts_tree_cursor_goto_next_sibling_internal(self)) {
case TreeCursorStepHidden: case TreeCursorStepHidden:
ts_tree_cursor_goto_first_child(self); ts_tree_cursor_goto_first_child(self);
@ -357,7 +357,7 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) {
} }
} }
TreeCursorStep ts_tree_cursor_goto_previous_sibling_internal(TSTreeCursor *_self) { TreeCursorStep ts_tree_cursor_goto_previous_sibling_internal(t_parse_tree_cursor *_self) {
// since subtracting across row loses column information, we may have to // since subtracting across row loses column information, we may have to
// restore it // restore it
TreeCursor *self = (TreeCursor *)_self; TreeCursor *self = (TreeCursor *)_self;
@ -375,13 +375,13 @@ TreeCursorStep ts_tree_cursor_goto_previous_sibling_internal(TSTreeCursor *_self
// restore position from the parent node // restore position from the parent node
const TreeCursorEntry *parent = &self->stack.contents[self->stack.size - 2]; const TreeCursorEntry *parent = &self->stack.contents[self->stack.size - 2];
Length position = parent->position; Length position = parent->position;
uint32_t child_index = array_back(&self->stack)->child_index; t_u32 child_index = array_back(&self->stack)->child_index;
const Subtree *children = ts_subtree_children((*(parent->subtree))); const Subtree *children = ts_subtree_children((*(parent->subtree)));
if (child_index > 0) { if (child_index > 0) {
// skip first child padding since its position should match the position of the parent // skip first child padding since its position should match the position of the parent
position = length_add(position, ts_subtree_size(children[0])); position = length_add(position, ts_subtree_size(children[0]));
for (uint32_t i = 1; i < child_index; ++i) { for (t_u32 i = 1; i < child_index; ++i) {
position = length_add(position, ts_subtree_total_size(children[i])); position = length_add(position, ts_subtree_total_size(children[i]));
} }
position = length_add(position, ts_subtree_padding(children[child_index])); position = length_add(position, ts_subtree_padding(children[child_index]));
@ -392,7 +392,7 @@ TreeCursorStep ts_tree_cursor_goto_previous_sibling_internal(TSTreeCursor *_self
return step; return step;
} }
bool ts_tree_cursor_goto_previous_sibling(TSTreeCursor *self) { bool ts_tree_cursor_goto_previous_sibling(t_parse_tree_cursor *self) {
switch (ts_tree_cursor_goto_previous_sibling_internal(self)) { switch (ts_tree_cursor_goto_previous_sibling_internal(self)) {
case TreeCursorStepHidden: case TreeCursorStepHidden:
ts_tree_cursor_goto_last_child(self); ts_tree_cursor_goto_last_child(self);
@ -404,7 +404,7 @@ bool ts_tree_cursor_goto_previous_sibling(TSTreeCursor *self) {
} }
} }
bool ts_tree_cursor_goto_parent(TSTreeCursor *_self) { bool ts_tree_cursor_goto_parent(t_parse_tree_cursor *_self) {
TreeCursor *self = (TreeCursor *)_self; TreeCursor *self = (TreeCursor *)_self;
for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) {
if (ts_tree_cursor_is_entry_visible(self, i)) { if (ts_tree_cursor_is_entry_visible(self, i)) {
@ -416,16 +416,16 @@ bool ts_tree_cursor_goto_parent(TSTreeCursor *_self) {
} }
void ts_tree_cursor_goto_descendant( void ts_tree_cursor_goto_descendant(
TSTreeCursor *_self, t_parse_tree_cursor *_self,
uint32_t goal_descendant_index t_u32 goal_descendant_index
) { ) {
TreeCursor *self = (TreeCursor *)_self; TreeCursor *self = (TreeCursor *)_self;
// Ascend to the lowest ancestor that contains the goal node. // Ascend to the lowest ancestor that contains the goal node.
for (;;) { for (;;) {
uint32_t i = self->stack.size - 1; t_u32 i = self->stack.size - 1;
TreeCursorEntry *entry = &self->stack.contents[i]; TreeCursorEntry *entry = &self->stack.contents[i];
uint32_t next_descendant_index = t_u32 next_descendant_index =
entry->descendant_index + entry->descendant_index +
(ts_tree_cursor_is_entry_visible(self, i) ? 1 : 0) + (ts_tree_cursor_is_entry_visible(self, i) ? 1 : 0) +
ts_subtree_visible_descendant_count(*entry->subtree); ts_subtree_visible_descendant_count(*entry->subtree);
@ -466,16 +466,16 @@ void ts_tree_cursor_goto_descendant(
} while (did_descend); } while (did_descend);
} }
uint32_t ts_tree_cursor_current_descendant_index(const TSTreeCursor *_self) { t_u32 ts_tree_cursor_current_descendant_index(const t_parse_tree_cursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self; const TreeCursor *self = (const TreeCursor *)_self;
TreeCursorEntry *last_entry = array_back(&self->stack); TreeCursorEntry *last_entry = array_back(&self->stack);
return last_entry->descendant_index; return last_entry->descendant_index;
} }
TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self) { t_parse_node ts_tree_cursor_current_node(const t_parse_tree_cursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self; const TreeCursor *self = (const TreeCursor *)_self;
TreeCursorEntry *last_entry = array_back(&self->stack); TreeCursorEntry *last_entry = array_back(&self->stack);
TSSymbol alias_symbol = self->root_alias_symbol; t_symbol alias_symbol = self->root_alias_symbol;
if (self->stack.size > 1 && !ts_subtree_extra(*last_entry->subtree)) { if (self->stack.size > 1 && !ts_subtree_extra(*last_entry->subtree)) {
TreeCursorEntry *parent_entry = &self->stack.contents[self->stack.size - 2]; TreeCursorEntry *parent_entry = &self->stack.contents[self->stack.size - 2];
alias_symbol = ts_language_alias_at( alias_symbol = ts_language_alias_at(
@ -495,12 +495,12 @@ TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self) {
// Private - Get various facts about the current node that are needed // Private - Get various facts about the current node that are needed
// when executing tree queries. // when executing tree queries.
void ts_tree_cursor_current_status( void ts_tree_cursor_current_status(
const TSTreeCursor *_self, const t_parse_tree_cursor *_self,
TSFieldId *field_id, t_field_id *field_id,
bool *has_later_siblings, bool *has_later_siblings,
bool *has_later_named_siblings, bool *has_later_named_siblings,
bool *can_have_later_siblings_with_this_field, bool *can_have_later_siblings_with_this_field,
TSSymbol *supertypes, t_symbol *supertypes,
unsigned *supertype_count unsigned *supertype_count
) { ) {
const TreeCursor *self = (const TreeCursor *)_self; const TreeCursor *self = (const TreeCursor *)_self;
@ -517,7 +517,7 @@ void ts_tree_cursor_current_status(
TreeCursorEntry *entry = &self->stack.contents[i]; TreeCursorEntry *entry = &self->stack.contents[i];
TreeCursorEntry *parent_entry = &self->stack.contents[i - 1]; TreeCursorEntry *parent_entry = &self->stack.contents[i - 1];
const TSSymbol *alias_sequence = ts_language_alias_sequence( const t_symbol *alias_sequence = ts_language_alias_sequence(
self->tree->language, self->tree->language,
parent_entry->subtree->ptr->production_id parent_entry->subtree->ptr->production_id
); );
@ -532,11 +532,11 @@ void ts_tree_cursor_current_status(
ts_subtree_symbol(subtree)) ts_subtree_symbol(subtree))
// Stop walking up when a visible ancestor is found. // Stop walking up when a visible ancestor is found.
TSSymbol entry_symbol = subtree_symbol( t_symbol entry_symbol = subtree_symbol(
*entry->subtree, *entry->subtree,
entry->structural_child_index entry->structural_child_index
); );
TSSymbolMetadata entry_metadata = ts_language_symbol_metadata( t_symbol_metadata entry_metadata = ts_language_symbol_metadata(
self->tree->language, self->tree->language,
entry_symbol entry_symbol
); );
@ -555,7 +555,7 @@ void ts_tree_cursor_current_status(
if (!ts_subtree_extra(*entry->subtree)) structural_child_index++; if (!ts_subtree_extra(*entry->subtree)) structural_child_index++;
for (unsigned j = entry->child_index + 1; j < sibling_count; j++) { for (unsigned j = entry->child_index + 1; j < sibling_count; j++) {
Subtree sibling = ts_subtree_children(*parent_entry->subtree)[j]; Subtree sibling = ts_subtree_children(*parent_entry->subtree)[j];
TSSymbolMetadata sibling_metadata = ts_language_symbol_metadata( t_symbol_metadata sibling_metadata = ts_language_symbol_metadata(
self->tree->language, self->tree->language,
subtree_symbol(sibling, structural_child_index) subtree_symbol(sibling, structural_child_index)
); );
@ -581,7 +581,7 @@ void ts_tree_cursor_current_status(
#undef subtree_symbol #undef subtree_symbol
if (!ts_subtree_extra(*entry->subtree)) { if (!ts_subtree_extra(*entry->subtree)) {
const TSFieldMapEntry *field_map, *field_map_end; const t_field_map_entry *field_map, *field_map_end;
ts_language_field_map( ts_language_field_map(
self->tree->language, self->tree->language,
parent_entry->subtree->ptr->production_id, parent_entry->subtree->ptr->production_id,
@ -590,7 +590,7 @@ void ts_tree_cursor_current_status(
// Look for a field name associated with the current node. // Look for a field name associated with the current node.
if (!*field_id) { if (!*field_id) {
for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) { for (const t_field_map_entry *map = field_map; map < field_map_end; map++) {
if (!map->inherited && map->child_index == entry->structural_child_index) { if (!map->inherited && map->child_index == entry->structural_child_index) {
*field_id = map->field_id; *field_id = map->field_id;
break; break;
@ -600,7 +600,7 @@ void ts_tree_cursor_current_status(
// Determine if the current node can have later siblings with the same field name. // Determine if the current node can have later siblings with the same field name.
if (*field_id) { if (*field_id) {
for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) { for (const t_field_map_entry *map = field_map; map < field_map_end; map++) {
if ( if (
map->field_id == *field_id && map->field_id == *field_id &&
map->child_index > entry->structural_child_index map->child_index > entry->structural_child_index
@ -614,9 +614,9 @@ void ts_tree_cursor_current_status(
} }
} }
uint32_t ts_tree_cursor_current_depth(const TSTreeCursor *_self) { t_u32 ts_tree_cursor_current_depth(const t_parse_tree_cursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self; const TreeCursor *self = (const TreeCursor *)_self;
uint32_t depth = 0; t_u32 depth = 0;
for (unsigned i = 1; i < self->stack.size; i++) { for (unsigned i = 1; i < self->stack.size; i++) {
if (ts_tree_cursor_is_entry_visible(self, i)) { if (ts_tree_cursor_is_entry_visible(self, i)) {
depth++; depth++;
@ -625,12 +625,12 @@ uint32_t ts_tree_cursor_current_depth(const TSTreeCursor *_self) {
return depth; return depth;
} }
TSNode ts_tree_cursor_parent_node(const TSTreeCursor *_self) { t_parse_node ts_tree_cursor_parent_node(const t_parse_tree_cursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self; const TreeCursor *self = (const TreeCursor *)_self;
for (int i = (int)self->stack.size - 2; i >= 0; i--) { for (int i = (int)self->stack.size - 2; i >= 0; i--) {
TreeCursorEntry *entry = &self->stack.contents[i]; TreeCursorEntry *entry = &self->stack.contents[i];
bool is_visible = true; bool is_visible = true;
TSSymbol alias_symbol = 0; t_symbol alias_symbol = 0;
if (i > 0) { if (i > 0) {
TreeCursorEntry *parent_entry = &self->stack.contents[i - 1]; TreeCursorEntry *parent_entry = &self->stack.contents[i - 1];
alias_symbol = ts_language_alias_at( alias_symbol = ts_language_alias_at(
@ -652,7 +652,7 @@ TSNode ts_tree_cursor_parent_node(const TSTreeCursor *_self) {
return ts_node_new(NULL, NULL, length_zero(), 0); return ts_node_new(NULL, NULL, length_zero(), 0);
} }
TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *_self) { t_field_id ts_tree_cursor_current_field_id(const t_parse_tree_cursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self; const TreeCursor *self = (const TreeCursor *)_self;
// Walk up the tree, visiting the current node and its invisible ancestors. // Walk up the tree, visiting the current node and its invisible ancestors.
@ -668,13 +668,13 @@ TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *_self) {
if (ts_subtree_extra(*entry->subtree)) break; if (ts_subtree_extra(*entry->subtree)) break;
const TSFieldMapEntry *field_map, *field_map_end; const t_field_map_entry *field_map, *field_map_end;
ts_language_field_map( ts_language_field_map(
self->tree->language, self->tree->language,
parent_entry->subtree->ptr->production_id, parent_entry->subtree->ptr->production_id,
&field_map, &field_map_end &field_map, &field_map_end
); );
for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) { for (const t_field_map_entry *map = field_map; map < field_map_end; map++) {
if (!map->inherited && map->child_index == entry->structural_child_index) { if (!map->inherited && map->child_index == entry->structural_child_index) {
return map->field_id; return map->field_id;
} }
@ -683,8 +683,8 @@ TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *_self) {
return 0; return 0;
} }
const char *ts_tree_cursor_current_field_name(const TSTreeCursor *_self) { const char *ts_tree_cursor_current_field_name(const t_parse_tree_cursor *_self) {
TSFieldId id = ts_tree_cursor_current_field_id(_self); t_field_id id = ts_tree_cursor_current_field_id(_self);
if (id) { if (id) {
const TreeCursor *self = (const TreeCursor *)_self; const TreeCursor *self = (const TreeCursor *)_self;
return self->tree->language->field_names[id]; return self->tree->language->field_names[id];
@ -693,9 +693,9 @@ const char *ts_tree_cursor_current_field_name(const TSTreeCursor *_self) {
} }
} }
TSTreeCursor ts_tree_cursor_copy(const TSTreeCursor *_cursor) { t_parse_tree_cursor ts_tree_cursor_copy(const t_parse_tree_cursor *_cursor) {
const TreeCursor *cursor = (const TreeCursor *)_cursor; const TreeCursor *cursor = (const TreeCursor *)_cursor;
TSTreeCursor res = {NULL, NULL, {0, 0}}; t_parse_tree_cursor res = {NULL, NULL, {0, 0}};
TreeCursor *copy = (TreeCursor *)&res; TreeCursor *copy = (TreeCursor *)&res;
copy->tree = cursor->tree; copy->tree = cursor->tree;
copy->root_alias_symbol = cursor->root_alias_symbol; copy->root_alias_symbol = cursor->root_alias_symbol;
@ -704,7 +704,7 @@ TSTreeCursor ts_tree_cursor_copy(const TSTreeCursor *_cursor) {
return res; return res;
} }
void ts_tree_cursor_reset_to(TSTreeCursor *_dst, const TSTreeCursor *_src) { void ts_tree_cursor_reset_to(t_parse_tree_cursor *_dst, const t_parse_tree_cursor *_src) {
const TreeCursor *cursor = (const TreeCursor *)_src; const TreeCursor *cursor = (const TreeCursor *)_src;
TreeCursor *copy = (TreeCursor *)_dst; TreeCursor *copy = (TreeCursor *)_dst;
copy->tree = cursor->tree; copy->tree = cursor->tree;

26
parser/src/tree_cursor.h
View file

@ -6,15 +6,15 @@
typedef struct { typedef struct {
const Subtree *subtree; const Subtree *subtree;
Length position; Length position;
uint32_t child_index; t_u32 child_index;
uint32_t structural_child_index; t_u32 structural_child_index;
uint32_t descendant_index; t_u32 descendant_index;
} TreeCursorEntry; } TreeCursorEntry;
typedef struct { typedef struct {
const TSTree *tree; const t_parse_tree *tree;
Array(TreeCursorEntry) stack; Array(TreeCursorEntry) stack;
TSSymbol root_alias_symbol; t_symbol root_alias_symbol;
} TreeCursor; } TreeCursor;
typedef enum { typedef enum {
@ -23,26 +23,26 @@ typedef enum {
TreeCursorStepVisible, TreeCursorStepVisible,
} TreeCursorStep; } TreeCursorStep;
void ts_tree_cursor_init(TreeCursor *, TSNode); void ts_tree_cursor_init(TreeCursor *, t_parse_node);
void ts_tree_cursor_current_status( void ts_tree_cursor_current_status(
const TSTreeCursor *, const t_parse_tree_cursor *,
TSFieldId *, t_field_id *,
bool *, bool *,
bool *, bool *,
bool *, bool *,
TSSymbol *, t_symbol *,
unsigned * unsigned *
); );
TreeCursorStep ts_tree_cursor_goto_first_child_internal(TSTreeCursor *); TreeCursorStep ts_tree_cursor_goto_first_child_internal(t_parse_tree_cursor *);
TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *); TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(t_parse_tree_cursor *);
static inline Subtree ts_tree_cursor_current_subtree(const TSTreeCursor *_self) { static inline Subtree ts_tree_cursor_current_subtree(const t_parse_tree_cursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self; const TreeCursor *self = (const TreeCursor *)_self;
TreeCursorEntry *last_entry = array_back(&self->stack); TreeCursorEntry *last_entry = array_back(&self->stack);
return *last_entry->subtree; return *last_entry->subtree;
} }
TSNode ts_tree_cursor_parent_node(const TSTreeCursor *); t_parse_node ts_tree_cursor_parent_node(const t_parse_tree_cursor *);
#endif // TREE_SITTER_TREE_CURSOR_H_ #endif // TREE_SITTER_TREE_CURSOR_H_