raphael/minishell
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

started to rename struct and stuff

Browse source
This commit is contained in:
Maieul BOYER 2024-04-29 16:57:49 +02:00
parent ff4b0c471f
commit 54cefca53f
No known key found for this signature in database
25 changed files with 2413 additions and 2431 deletions
Download patch file Download diff file Expand all files Collapse all files

372
parser/src/array.h
View file

@ -1,11 +1,7 @@
#ifndef TREE_SITTER_ARRAY_H_
#define TREE_SITTER_ARRAY_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "me/types.h"
#include <assert.h>
#include <stdbool.h>
@ -13,31 +9,27 @@ extern "C" {
#include <stdlib.h>
#include <string.h>
#ifdef _MSC_VER
#pragma warning(disable : 4101)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
#endif
#define Array(T) \
struct { \
T *contents; \
uint32_t size; \
uint32_t capacity; \
}
#define Array(T) \
struct \
{ \
T *contents; \
t_u32 size; \
t_u32 capacity; \
}
/// Initialize an array.
#define array_init(self) \
((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
#define array_init(self) \
((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
/// Create an empty array.
#define array_new() \
{ NULL, 0, 0 }
#define array_new() \
{ \
NULL, 0, 0 \
}
/// Get a pointer to the element at a given `index` in the array.
#define array_get(self, _index) \
(assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
#define array_get(self, _index) \
(assert((t_u32)(_index) < (self)->size), &(self)->contents[_index])
/// Get a pointer to the first element in the array.
#define array_front(self) array_get(self, 0)
@ -51,67 +43,67 @@ extern "C" {
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
/// less than the array's current capacity, this function has no effect.
#define array_reserve(self, new_capacity) \
_array__reserve((Array *)(self), array_elem_size(self), new_capacity)
#define array_reserve(self, new_capacity) \
_array__reserve((Array *)(self), array_elem_size(self), new_capacity)
/// Free any memory allocated for this array. Note that this does not free any
/// memory allocated for the array's contents.
#define array_delete(self) _array__delete((Array *)(self))
/// Push a new `element` onto the end of the array.
#define array_push(self, element) \
(_array__grow((Array *)(self), 1, array_elem_size(self)), \
(self)->contents[(self)->size++] = (element))
#define array_push(self, element) \
(_array__grow((Array *)(self), 1, array_elem_size(self)), \
(self)->contents[(self)->size++] = (element))
/// Increase the array's size by `count` elements.
/// New elements are zero-initialized.
#define array_grow_by(self, count) \
do { \
if ((count) == 0) break; \
_array__grow((Array *)(self), count, array_elem_size(self)); \
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
(self)->size += (count); \
} while (0)
#define array_grow_by(self, count) \
do \
{ \
if ((count) == 0) \
break; \
_array__grow((Array *)(self), count, array_elem_size(self)); \
memset((self)->contents + (self)->size, 0, \
(count) * array_elem_size(self)); \
(self)->size += (count); \
} while (0)
/// Append all elements from one array to the end of another.
#define array_push_all(self, other) \
array_extend((self), (other)->size, (other)->contents)
#define array_push_all(self, other) \
array_extend((self), (other)->size, (other)->contents)
/// Append `count` elements to the end of the array, reading their values from the
/// `contents` pointer.
#define array_extend(self, count, contents) \
_array__splice( \
(Array *)(self), array_elem_size(self), (self)->size, \
0, count, contents \
)
/// Append `count` elements to the end of the array, reading their values from
/// the `contents` pointer.
#define array_extend(self, count, contents) \
_array__splice((Array *)(self), array_elem_size(self), (self)->size, 0, \
count, contents)
/// Remove `old_count` elements from the array starting at the given `index`. At
/// the same index, insert `new_count` new elements, reading their values from the
/// `new_contents` pointer.
#define array_splice(self, _index, old_count, new_count, new_contents) \
_array__splice( \
(Array *)(self), array_elem_size(self), _index, \
old_count, new_count, new_contents \
)
/// the same index, insert `new_count` new elements, reading their values from
/// the `new_contents` pointer.
#define array_splice(self, _index, old_count, new_count, new_contents) \
_array__splice((Array *)(self), array_elem_size(self), _index, old_count, \
new_count, new_contents)
/// Insert one `element` into the array at the given `index`.
#define array_insert(self, _index, element) \
_array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
#define array_insert(self, _index, element) \
_array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, \
&(element))
/// Remove one element from the array at the given `index`.
#define array_erase(self, _index) \
_array__erase((Array *)(self), array_elem_size(self), _index)
#define array_erase(self, _index) \
_array__erase((Array *)(self), array_elem_size(self), _index)
/// Pop the last element off the array, returning the element by value.
#define array_pop(self) ((self)->contents[--(self)->size])
/// Assign the contents of one array to another, reallocating if necessary.
#define array_assign(self, other) \
_array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
#define array_assign(self, other) \
_array__assign((Array *)(self), (const Array *)(other), \
array_elem_size(self))
/// Swap one array with another
#define array_swap(self, other) \
_array__swap((Array *)(self), (Array *)(other))
#define array_swap(self, other) _array__swap((Array *)(self), (Array *)(other))
/// Get the size of the array contents
#define array_elem_size(self) (sizeof *(self)->contents)
@ -124,167 +116,187 @@ extern "C" {
/// out-parameter is set to true. Otherwise, `index` is set to an index where
/// `needle` should be inserted in order to preserve the sorting, and `exists`
/// is set to false.
#define array_search_sorted_with(self, compare, needle, _index, _exists) \
_array__search_sorted(self, 0, compare, , needle, _index, _exists)
#define array_search_sorted_with(self, compare, needle, _index, _exists) \
_array__search_sorted(self, 0, compare, , needle, _index, _exists)
/// Search a sorted array for a given `needle` value, using integer comparisons
/// of a given struct field (specified with a leading dot) to determine the order.
/// of a given struct field (specified with a leading dot) to determine the
/// order.
///
/// See also `array_search_sorted_with`.
#define array_search_sorted_by(self, field, needle, _index, _exists) \
_array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
#define array_search_sorted_by(self, field, needle, _index, _exists) \
_array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
/// Insert a given `value` into a sorted array, using the given `compare`
/// callback to determine the order.
#define array_insert_sorted_with(self, compare, value) \
do { \
unsigned _index, _exists; \
array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
if (!_exists) array_insert(self, _index, value); \
} while (0)
#define array_insert_sorted_with(self, compare, value) \
do \
{ \
unsigned _index, _exists; \
array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
if (!_exists) \
array_insert(self, _index, value); \
} while (0)
/// Insert a given `value` into a sorted array, using integer comparisons of
/// a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_by`.
#define array_insert_sorted_by(self, field, value) \
do { \
unsigned _index, _exists; \
array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
if (!_exists) array_insert(self, _index, value); \
} while (0)
#define array_insert_sorted_by(self, field, value) \
do \
{ \
unsigned _index, _exists; \
array_search_sorted_by(self, field, (value)field, &_index, &_exists); \
if (!_exists) \
array_insert(self, _index, value); \
} while (0)
// Private
typedef Array(void) Array;
/// This is not what you're looking for, see `array_delete`.
static inline void _array__delete(Array *self) {
if (self->contents) {
free(self->contents);
self->contents = NULL;
self->size = 0;
self->capacity = 0;
}
static inline void _array__delete(Array *self)
{
if (self->contents)
{
free(self->contents);
self->contents = NULL;
self->size = 0;
self->capacity = 0;
}
}
/// This is not what you're looking for, see `array_erase`.
static inline void _array__erase(Array *self, size_t element_size,
uint32_t index) {
assert(index < self->size);
char *contents = (char *)self->contents;
memmove(contents + index * element_size, contents + (index + 1) * element_size,
(self->size - index - 1) * element_size);
self->size--;
static inline void _array__erase(Array *self, size_t element_size, t_u32 index)
{
assert(index < self->size);
char *contents = (char *)self->contents;
memmove(contents + index * element_size,
contents + (index + 1) * element_size,
(self->size - index - 1) * element_size);
self->size--;
}
/// This is not what you're looking for, see `array_reserve`.
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
if (new_capacity > self->capacity) {
if (self->contents) {
self->contents = realloc(self->contents, new_capacity * element_size);
} else {
self->contents = malloc(new_capacity * element_size);
}
self->capacity = new_capacity;
}
static inline void _array__reserve(Array *self, size_t element_size,
t_u32 new_capacity)
{
if (new_capacity > self->capacity)
{
if (self->contents)
{
self->contents =
realloc(self->contents, new_capacity * element_size);
}
else
{
self->contents = malloc(new_capacity * element_size);
}
self->capacity = new_capacity;
}
}
/// This is not what you're looking for, see `array_assign`.
static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
_array__reserve(self, element_size, other->size);
self->size = other->size;
memcpy(self->contents, other->contents, self->size * element_size);
static inline void _array__assign(Array *self, const Array *other,
size_t element_size)
{
_array__reserve(self, element_size, other->size);
self->size = other->size;
memcpy(self->contents, other->contents, self->size * element_size);
}
/// This is not what you're looking for, see `array_swap`.
static inline void _array__swap(Array *self, Array *other) {
Array swap = *other;
*other = *self;
*self = swap;
static inline void _array__swap(Array *self, Array *other)
{
Array swap = *other;
*other = *self;
*self = swap;
}
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
uint32_t new_size = self->size + count;
if (new_size > self->capacity) {
uint32_t new_capacity = self->capacity * 2;
if (new_capacity < 8) new_capacity = 8;
if (new_capacity < new_size) new_capacity = new_size;
_array__reserve(self, element_size, new_capacity);
}
static inline void _array__grow(Array *self, t_u32 count, size_t element_size)
{
t_u32 new_size = self->size + count;
if (new_size > self->capacity)
{
t_u32 new_capacity = self->capacity * 2;
if (new_capacity < 8)
new_capacity = 8;
if (new_capacity < new_size)
new_capacity = new_size;
_array__reserve(self, element_size, new_capacity);
}
}
/// This is not what you're looking for, see `array_splice`.
static inline void _array__splice(Array *self, size_t element_size,
uint32_t index, uint32_t old_count,
uint32_t new_count, const void *elements) {
uint32_t new_size = self->size + new_count - old_count;
uint32_t old_end = index + old_count;
uint32_t new_end = index + new_count;
assert(old_end <= self->size);
static inline void _array__splice(Array *self, size_t element_size, t_u32 index,
t_u32 old_count, t_u32 new_count,
const void *elements)
{
t_u32 new_size = self->size + new_count - old_count;
t_u32 old_end = index + old_count;
t_u32 new_end = index + new_count;
assert(old_end <= self->size);
_array__reserve(self, element_size, new_size);
_array__reserve(self, element_size, new_size);
char *contents = (char *)self->contents;
if (self->size > old_end) {
memmove(
contents + new_end * element_size,
contents + old_end * element_size,
(self->size - old_end) * element_size
);
}
if (new_count > 0) {
if (elements) {
memcpy(
(contents + index * element_size),
elements,
new_count * element_size
);
} else {
memset(
(contents + index * element_size),
0,
new_count * element_size
);
}
}
self->size += new_count - old_count;
char *contents = (char *)self->contents;
if (self->size > old_end)
{
memmove(contents + new_end * element_size,
contents + old_end * element_size,
(self->size - old_end) * element_size);
}
if (new_count > 0)
{
if (elements)
{
memcpy((contents + index * element_size), elements,
new_count * element_size);
}
else
{
memset((contents + index * element_size), 0,
new_count * element_size);
}
}
self->size += new_count - old_count;
}
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
do { \
*(_index) = start; \
*(_exists) = false; \
uint32_t size = (self)->size - *(_index); \
if (size == 0) break; \
int comparison; \
while (size > 1) { \
uint32_t half_size = size / 2; \
uint32_t mid_index = *(_index) + half_size; \
comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
if (comparison <= 0) *(_index) = mid_index; \
size -= half_size; \
} \
comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
if (comparison == 0) *(_exists) = true; \
else if (comparison < 0) *(_index) += 1; \
} while (0)
/// This is not what you're looking for, see `array_search_sorted_with` or
/// `array_search_sorted_by`.
#define _array__search_sorted(self, start, compare, suffix, needle, _index, \
_exists) \
do \
{ \
*(_index) = start; \
*(_exists) = false; \
t_u32 size = (self)->size - *(_index); \
if (size == 0) \
break; \
int comparison; \
while (size > 1) \
{ \
t_u32 half_size = size / 2; \
t_u32 mid_index = *(_index) + half_size; \
comparison = \
compare(&((self)->contents[mid_index] suffix), (needle)); \
if (comparison <= 0) \
*(_index) = mid_index; \
size -= half_size; \
} \
comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
if (comparison == 0) \
*(_exists) = true; \
else if (comparison < 0) \
*(_index) += 1; \
} while (0)
/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
/// parameter by reference in order to work with the generic sorting function above.
/// Helper macro for the `_sorted_by` routines below. This takes the left
/// (existing) parameter by reference in order to work with the generic sorting
/// function above.
#define _compare_int(a, b) ((int)*(a) - (int)(b))
#ifdef _MSC_VER
#pragma warning(default : 4101)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic pop
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ARRAY_H_
#endif // TREE_SITTER_ARRAY_H_

112
parser/src/language.c
View file

@ -2,34 +2,34 @@
#include "parser/api.h"
#include <string.h>
const TSLanguage *ts_language_copy(const TSLanguage *self) {
const t_language *ts_language_copy(const t_language *self) {
return self;
}
void ts_language_delete(const TSLanguage *self) {
void ts_language_delete(const t_language *self) {
(void)(self);
}
uint32_t ts_language_symbol_count(const TSLanguage *self) {
t_u32 ts_language_symbol_count(const t_language *self) {
return self->symbol_count + self->alias_count;
}
uint32_t ts_language_state_count(const TSLanguage *self) {
t_u32 ts_language_state_count(const t_language *self) {
return self->state_count;
}
uint32_t ts_language_version(const TSLanguage *self) {
t_u32 ts_language_version(const t_language *self) {
return self->version;
}
uint32_t ts_language_field_count(const TSLanguage *self) {
t_u32 ts_language_field_count(const t_language *self) {
return self->field_count;
}
void ts_language_table_entry(
const TSLanguage *self,
TSStateId state,
TSSymbol symbol,
const t_language *self,
t_state_id state,
t_symbol symbol,
TableEntry *result
) {
if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
@ -38,48 +38,48 @@ void ts_language_table_entry(
result->actions = NULL;
} else {
assert(symbol < self->token_count);
uint32_t action_index = ts_language_lookup(self, state, symbol);
const TSParseActionEntry *entry = &self->parse_actions[action_index];
t_u32 action_index = ts_language_lookup(self, state, symbol);
const t_parse_action_entry *entry = &self->parse_actions[action_index];
result->action_count = entry->entry.count;
result->is_reusable = entry->entry.reusable;
result->actions = (const TSParseAction *)(entry + 1);
result->actions = (const t_parse_actions *)(entry + 1);
}
}
TSSymbolMetadata ts_language_symbol_metadata(
const TSLanguage *self,
TSSymbol symbol
t_symbol_metadata ts_language_symbol_metadata(
const t_language *self,
t_symbol symbol
) {
if (symbol == ts_builtin_sym_error) {
return (TSSymbolMetadata) {.visible = true, .named = true};
return (t_symbol_metadata) {.visible = true, .named = true};
} else if (symbol == ts_builtin_sym_error_repeat) {
return (TSSymbolMetadata) {.visible = false, .named = false};
return (t_symbol_metadata) {.visible = false, .named = false};
} else {
return self->symbol_metadata[symbol];
}
}
TSSymbol ts_language_public_symbol(
const TSLanguage *self,
TSSymbol symbol
t_symbol ts_language_public_symbol(
const t_language *self,
t_symbol symbol
) {
if (symbol == ts_builtin_sym_error) return symbol;
return self->public_symbol_map[symbol];
}
TSStateId ts_language_next_state(
const TSLanguage *self,
TSStateId state,
TSSymbol symbol
t_state_id ts_language_next_state(
const t_language *self,
t_state_id state,
t_symbol symbol
) {
if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
return 0;
} else if (symbol < self->token_count) {
uint32_t count;
const TSParseAction *actions = ts_language_actions(self, state, symbol, &count);
t_u32 count;
const t_parse_actions *actions = ts_language_actions(self, state, symbol, &count);
if (count > 0) {
TSParseAction action = actions[count - 1];
if (action.type == TSParseActionTypeShift) {
t_parse_actions action = actions[count - 1];
if (action.type == ActionTypeShift) {
return action.shift.extra ? state : action.shift.state;
}
}
@ -90,8 +90,8 @@ TSStateId ts_language_next_state(
}
const char *ts_language_symbol_name(
const TSLanguage *self,
TSSymbol symbol
const t_language *self,
t_symbol symbol
) {
if (symbol == ts_builtin_sym_error) {
return "ERROR";
@ -104,16 +104,16 @@ const char *ts_language_symbol_name(
}
}
TSSymbol ts_language_symbol_for_name(
const TSLanguage *self,
t_symbol ts_language_symbol_for_name(
const t_language *self,
const char *string,
uint32_t length,
t_u32 length,
bool is_named
) {
if (!strncmp(string, "ERROR", length)) return ts_builtin_sym_error;
uint16_t count = (uint16_t)ts_language_symbol_count(self);
for (TSSymbol i = 0; i < count; i++) {
TSSymbolMetadata metadata = ts_language_symbol_metadata(self, i);
t_u16 count = (t_u16)ts_language_symbol_count(self);
for (t_symbol i = 0; i < count; i++) {
t_symbol_metadata metadata = ts_language_symbol_metadata(self, i);
if ((!metadata.visible && !metadata.supertype) || metadata.named != is_named) continue;
const char *symbol_name = self->symbol_names[i];
if (!strncmp(symbol_name, string, length) && !symbol_name[length]) {
@ -123,25 +123,25 @@ TSSymbol ts_language_symbol_for_name(
return 0;
}
TSSymbolType ts_language_symbol_type(
const TSLanguage *self,
TSSymbol symbol
t_symbol_type ts_language_symbol_type(
const t_language *self,
t_symbol symbol
) {
TSSymbolMetadata metadata = ts_language_symbol_metadata(self, symbol);
t_symbol_metadata metadata = ts_language_symbol_metadata(self, symbol);
if (metadata.named && metadata.visible) {
return TSSymbolTypeRegular;
return SymbolTypeRegular;
} else if (metadata.visible) {
return TSSymbolTypeAnonymous;
return SymbolTypeAnonymous;
} else {
return TSSymbolTypeAuxiliary;
return SymbolTypeAuxiliary;
}
}
const char *ts_language_field_name_for_id(
const TSLanguage *self,
TSFieldId id
const t_language *self,
t_field_id id
) {
uint32_t count = ts_language_field_count(self);
t_u32 count = ts_language_field_count(self);
if (count && id <= count) {
return self->field_names[id];
} else {
@ -149,13 +149,13 @@ const char *ts_language_field_name_for_id(
}
}
TSFieldId ts_language_field_id_for_name(
const TSLanguage *self,
t_field_id ts_language_field_id_for_name(
const t_language *self,
const char *name,
uint32_t name_length
t_u32 name_length
) {
uint16_t count = (uint16_t)ts_language_field_count(self);
for (TSSymbol i = 1; i < count + 1; i++) {
t_u16 count = (t_u16)ts_language_field_count(self);
for (t_symbol i = 1; i < count + 1; i++) {
switch (strncmp(name, self->field_names[i], name_length)) {
case 0:
if (self->field_names[i][name_length] == 0) return i;
@ -169,7 +169,7 @@ TSFieldId ts_language_field_id_for_name(
return 0;
}
TSLookaheadIterator *ts_lookahead_iterator_new(const TSLanguage *self, TSStateId state) {
TSLookaheadIterator *ts_lookahead_iterator_new(const t_language *self, t_state_id state) {
if (state >= self->state_count) return NULL;
LookaheadIterator *iterator = malloc(sizeof(LookaheadIterator));
*iterator = ts_language_lookaheads(self, state);
@ -180,19 +180,19 @@ void ts_lookahead_iterator_delete(TSLookaheadIterator *self) {
free(self);
}
bool ts_lookahead_iterator_reset_state(TSLookaheadIterator * self, TSStateId state) {
bool ts_lookahead_iterator_reset_state(TSLookaheadIterator * self, t_state_id state) {
LookaheadIterator *iterator = (LookaheadIterator *)self;
if (state >= iterator->language->state_count) return false;
*iterator = ts_language_lookaheads(iterator->language, state);
return true;
}
const TSLanguage *ts_lookahead_iterator_language(const TSLookaheadIterator *self) {
const t_language *ts_lookahead_iterator_language(const TSLookaheadIterator *self) {
const LookaheadIterator *iterator = (const LookaheadIterator *)self;
return iterator->language;
}
bool ts_lookahead_iterator_reset(TSLookaheadIterator *self, const TSLanguage *language, TSStateId state) {
bool ts_lookahead_iterator_reset(TSLookaheadIterator *self, const t_language *language, t_state_id state) {
if (state >= language->state_count) return false;
LookaheadIterator *iterator = (LookaheadIterator *)self;
*iterator = ts_language_lookaheads(language, state);
@ -204,7 +204,7 @@ bool ts_lookahead_iterator_next(TSLookaheadIterator *self) {
return ts_lookahead_iterator__next(iterator);
}
TSSymbol ts_lookahead_iterator_current_symbol(const TSLookaheadIterator *self) {
t_symbol ts_lookahead_iterator_current_symbol(const TSLookaheadIterator *self) {
const LookaheadIterator *iterator = (const LookaheadIterator *)self;
return iterator->symbol;
}

493
parser/src/language.h
View file

@ -1,72 +1,75 @@
#ifndef TREE_SITTER_LANGUAGE_H_
#define TREE_SITTER_LANGUAGE_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "./subtree.h"
#include "./parser.h"
#include "./subtree.h"
#include "parser/types/types_parse_action_type.h"
#include "parser/types/types_state_id.h"
#include "parser/types/types_symbol.h"
#define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1)
#define LANGUAGE_VERSION_WITH_PRIMARY_STATES 14
#define LANGUAGE_VERSION_USABLE_VIA_WASM 13
typedef struct {
const TSParseAction *actions;
uint32_t action_count;
bool is_reusable;
typedef struct
{
const t_parse_actions *actions;
t_u32 action_count;
bool is_reusable;
} TableEntry;
typedef struct {
const TSLanguage *language;
const uint16_t *data;
const uint16_t *group_end;
TSStateId state;
uint16_t table_value;
uint16_t section_index;
uint16_t group_count;
bool is_small_state;
typedef struct
{
const t_language *language;
const t_u16 *data;
const t_u16 *group_end;
t_state_id state;
t_u16 table_value;
t_u16 section_index;
t_u16 group_count;
bool is_small_state;
const TSParseAction *actions;
TSSymbol symbol;
TSStateId next_state;
uint16_t action_count;
const t_parse_actions *actions;
t_symbol symbol;
t_state_id next_state;
t_u16 action_count;
} LookaheadIterator;
void ts_language_table_entry(const TSLanguage *, TSStateId, TSSymbol, TableEntry *);
void ts_language_table_entry(const t_language *, t_state_id, t_symbol,
TableEntry *);
TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *, TSSymbol);
t_symbol_metadata ts_language_symbol_metadata(const t_language *, t_symbol);
TSSymbol ts_language_public_symbol(const TSLanguage *, TSSymbol);
t_symbol ts_language_public_symbol(const t_language *, t_symbol);
TSStateId ts_language_next_state(const TSLanguage *self, TSStateId state, TSSymbol symbol);
t_state_id ts_language_next_state(const t_language *self, t_state_id state,
t_symbol symbol);
static inline bool ts_language_is_symbol_external(const TSLanguage *self, TSSymbol symbol) {
return 0 < symbol && symbol < self->external_token_count + 1;
static inline bool ts_language_is_symbol_external(const t_language *self,
t_symbol symbol)
{
return 0 < symbol && symbol < self->external_token_count + 1;
}
static inline const TSParseAction *ts_language_actions(
const TSLanguage *self,
TSStateId state,
TSSymbol symbol,
uint32_t *count
) {
TableEntry entry;
ts_language_table_entry(self, state, symbol, &entry);
*count = entry.action_count;
return entry.actions;
static inline const t_parse_actions *ts_language_actions(const t_language *self,
t_state_id state,
t_symbol symbol,
t_u32 *count)
{
TableEntry entry;
ts_language_table_entry(self, state, symbol, &entry);
*count = entry.action_count;
return entry.actions;
}
static inline bool ts_language_has_reduce_action(
const TSLanguage *self,
TSStateId state,
TSSymbol symbol
) {
TableEntry entry;
ts_language_table_entry(self, state, symbol, &entry);
return entry.action_count > 0 && entry.actions[0].type == TSParseActionTypeReduce;
static inline bool ts_language_has_reduce_action(const t_language *self,
t_state_id state,
t_symbol symbol)
{
TableEntry entry;
ts_language_table_entry(self, state, symbol, &entry);
return entry.action_count > 0 && entry.actions[0].type == ActionTypeReduce;
}
// Lookup the table value for a given symbol and state.
@ -76,34 +79,37 @@ static inline bool ts_language_has_reduce_action(
// For 'large' parse states, this is a direct lookup. For 'small' parse
// states, this requires searching through the symbol groups to find
// the given symbol.
static inline uint16_t ts_language_lookup(
const TSLanguage *self,
TSStateId state,
TSSymbol symbol
) {
if (state >= self->large_state_count) {
uint32_t index = self->small_parse_table_map[state - self->large_state_count];
const uint16_t *data = &self->small_parse_table[index];
uint16_t group_count = *(data++);
for (unsigned i = 0; i < group_count; i++) {
uint16_t section_value = *(data++);
uint16_t symbol_count = *(data++);
for (unsigned j = 0; j < symbol_count; j++) {
if (*(data++) == symbol) return section_value;
}
}
return 0;
} else {
return self->parse_table[state * self->symbol_count + symbol];
}
static inline t_u16 ts_language_lookup(const t_language *self, t_state_id state,
t_symbol symbol)
{
if (state >= self->large_state_count)
{
t_u32 index =
self->small_parse_table_map[state - self->large_state_count];
const t_u16 *data = &self->small_parse_table[index];
t_u16 group_count = *(data++);
for (unsigned i = 0; i < group_count; i++)
{
t_u16 section_value = *(data++);
t_u16 symbol_count = *(data++);
for (unsigned j = 0; j < symbol_count; j++)
{
if (*(data++) == symbol)
return section_value;
}
}
return 0;
}
else
{
return self->parse_table[state * self->symbol_count + symbol];
}
}
static inline bool ts_language_has_actions(
const TSLanguage *self,
TSStateId state,
TSSymbol symbol
) {
return ts_language_lookup(self, state, symbol) != 0;
static inline bool ts_language_has_actions(const t_language *self,
t_state_id state, t_symbol symbol)
{
return ts_language_lookup(self, state, symbol) != 0;
}
// Iterate over all of the symbols that are valid in the given state.
@ -112,188 +118,209 @@ static inline bool ts_language_has_actions(
// all possible symbols and checking the parse table for each one.
// For 'small' parse states, this exploits the structure of the
// table to only visit the valid symbols.
static inline LookaheadIterator ts_language_lookaheads(
const TSLanguage *self,
TSStateId state
) {
bool is_small_state = state >= self->large_state_count;
const uint16_t *data;
const uint16_t *group_end = NULL;
uint16_t group_count = 0;
if (is_small_state) {
uint32_t index = self->small_parse_table_map[state - self->large_state_count];
data = &self->small_parse_table[index];
group_end = data + 1;
group_count = *data;
} else {
data = &self->parse_table[state * self->symbol_count] - 1;
}
return (LookaheadIterator) {
.language = self,
.data = data,
.group_end = group_end,
.group_count = group_count,
.is_small_state = is_small_state,
.symbol = UINT16_MAX,
.next_state = 0,
};
static inline LookaheadIterator ts_language_lookaheads(const t_language *self,
t_state_id state)
{
bool is_small_state = state >= self->large_state_count;
const t_u16 *data;
const t_u16 *group_end = NULL;
t_u16 group_count = 0;
if (is_small_state)
{
t_u32 index =
self->small_parse_table_map[state - self->large_state_count];
data = &self->small_parse_table[index];
group_end = data + 1;
group_count = *data;
}
else
{
data = &self->parse_table[state * self->symbol_count] - 1;
}
return (LookaheadIterator){
.language = self,
.data = data,
.group_end = group_end,
.group_count = group_count,
.is_small_state = is_small_state,
.symbol = UINT16_MAX,
.next_state = 0,
};
}
static inline bool ts_lookahead_iterator__next(LookaheadIterator *self) {
// For small parse states, valid symbols are listed explicitly,
// grouped by their value. There's no need to look up the actions
// again until moving to the next group.
if (self->is_small_state) {
self->data++;
if (self->data == self->group_end) {
if (self->group_count == 0) return false;
self->group_count--;
self->table_value = *(self->data++);
unsigned symbol_count = *(self->data++);
self->group_end = self->data + symbol_count;
self->symbol = *self->data;
} else {
self->symbol = *self->data;
return true;
}
}
static inline bool ts_lookahead_iterator__next(LookaheadIterator *self)
{
// For small parse states, valid symbols are listed explicitly,
// grouped by their value. There's no need to look up the actions
// again until moving to the next group.
if (self->is_small_state)
{
self->data++;
if (self->data == self->group_end)
{
if (self->group_count == 0)
return false;
self->group_count--;
self->table_value = *(self->data++);
unsigned symbol_count = *(self->data++);
self->group_end = self->data + symbol_count;
self->symbol = *self->data;
}
else
{
self->symbol = *self->data;
return true;
}
}
// For large parse states, iterate through every symbol until one
// is found that has valid actions.
else {
do {
self->data++;
self->symbol++;
if (self->symbol >= self->language->symbol_count) return false;
self->table_value = *self->data;
} while (!self->table_value);
}
// For large parse states, iterate through every symbol until one
// is found that has valid actions.
else
{
do
{
self->data++;
self->symbol++;
if (self->symbol >= self->language->symbol_count)
return false;
self->table_value = *self->data;
} while (!self->table_value);
}
// Depending on if the symbols is terminal or non-terminal, the table value either
// represents a list of actions or a successor state.
if (self->symbol < self->language->token_count) {
const TSParseActionEntry *entry = &self->language->parse_actions[self->table_value];
self->action_count = entry->entry.count;
self->actions = (const TSParseAction *)(entry + 1);
self->next_state = 0;
} else {
self->action_count = 0;
self->next_state = self->table_value;
}
return true;
// Depending on if the symbols is terminal or non-terminal, the table
// value either represents a list of actions or a successor state.
if (self->symbol < self->language->token_count)
{
const t_parse_action_entry *entry =
&self->language->parse_actions[self->table_value];
self->action_count = entry->entry.count;
self->actions = (const t_parse_actions *)(entry + 1);
self->next_state = 0;
}
else
{
self->action_count = 0;
self->next_state = self->table_value;
}
return true;
}
// Whether the state is a "primary state". If this returns false, it indicates that there exists
// another state that behaves identically to this one with respect to query analysis.
static inline bool ts_language_state_is_primary(
const TSLanguage *self,
TSStateId state
) {
if (self->version >= LANGUAGE_VERSION_WITH_PRIMARY_STATES) {
return state == self->primary_state_ids[state];
} else {
return true;
}
// Whether the state is a "primary state". If this returns false, it
// indicates that there exists another state that behaves identically to
// this one with respect to query analysis.
static inline bool ts_language_state_is_primary(const t_language *self,
t_state_id state)
{
if (self->version >= LANGUAGE_VERSION_WITH_PRIMARY_STATES)
{
return state == self->primary_state_ids[state];
}
else
{
return true;
}
}
static inline const bool *ts_language_enabled_external_tokens(
const TSLanguage *self,
unsigned external_scanner_state
) {
if (external_scanner_state == 0) {
return NULL;
} else {
return self->external_scanner.states + self->external_token_count * external_scanner_state;
}
const t_language *self, unsigned external_scanner_state)
{
if (external_scanner_state == 0)
{
return NULL;
}
else
{
return self->external_scanner.states +
self->external_token_count * external_scanner_state;
}
}
static inline const TSSymbol *ts_language_alias_sequence(
const TSLanguage *self,
uint32_t production_id
) {
return production_id ?
&self->alias_sequences[production_id * self->max_alias_sequence_length] :
NULL;
static inline const t_symbol *ts_language_alias_sequence(const t_language *self,
t_u32 production_id)
{
return production_id
? &self->alias_sequences[production_id *
self->max_alias_sequence_length]
: NULL;
}
static inline TSSymbol ts_language_alias_at(
const TSLanguage *self,
uint32_t production_id,
uint32_t child_index
) {
return production_id ?
self->alias_sequences[production_id * self->max_alias_sequence_length + child_index] :
0;
static inline t_symbol ts_language_alias_at(const t_language *self,
t_u32 production_id,
t_u32 child_index)
{
return production_id
? self->alias_sequences[production_id *
self->max_alias_sequence_length +
child_index]
: 0;
}
static inline void ts_language_field_map(
const TSLanguage *self,
uint32_t production_id,
const TSFieldMapEntry **start,
const TSFieldMapEntry **end
) {
if (self->field_count == 0) {
*start = NULL;
*end = NULL;
return;
}
static inline void ts_language_field_map(const t_language *self,
t_u32 production_id,
const t_field_map_entry **start,
const t_field_map_entry **end)
{
if (self->field_count == 0)
{
*start = NULL;
*end = NULL;
return;
}
TSFieldMapSlice slice = self->field_map_slices[production_id];
*start = &self->field_map_entries[slice.index];
*end = &self->field_map_entries[slice.index] + slice.length;
t_field_map_slice slice = self->field_map_slices[production_id];
*start = &self->field_map_entries[slice.index];
*end = &self->field_map_entries[slice.index] + slice.length;
}
static inline void ts_language_aliases_for_symbol(
const TSLanguage *self,
TSSymbol original_symbol,
const TSSymbol **start,
const TSSymbol **end
) {
*start = &self->public_symbol_map[original_symbol];
*end = *start + 1;
static inline void ts_language_aliases_for_symbol(const t_language *self,
t_symbol original_symbol,
const t_symbol **start,
const t_symbol **end)
{
*start = &self->public_symbol_map[original_symbol];
*end = *start + 1;
unsigned idx = 0;
for (;;) {
TSSymbol symbol = self->alias_map[idx++];
if (symbol == 0 || symbol > original_symbol) break;
uint16_t count = self->alias_map[idx++];
if (symbol == original_symbol) {
*start = &self->alias_map[idx];
*end = &self->alias_map[idx + count];
break;
}
idx += count;
}
unsigned idx = 0;
for (;;)
{
t_symbol symbol = self->alias_map[idx++];
if (symbol == 0 || symbol > original_symbol)
break;
t_u16 count = self->alias_map[idx++];
if (symbol == original_symbol)
{
*start = &self->alias_map[idx];
*end = &self->alias_map[idx + count];
break;
}
idx += count;
}
}
static inline void ts_language_write_symbol_as_dot_string(
const TSLanguage *self,
FILE *f,
TSSymbol symbol
) {
const char *name = ts_language_symbol_name(self, symbol);
for (const char *chr = name; *chr; chr++) {
switch (*chr) {
case '"':
case '\\':
fputc('\\', f);
fputc(*chr, f);
break;
case '\n':
fputs("\\n", f);
break;
case '\t':
fputs("\\t", f);
break;
default:
fputc(*chr, f);
break;
}
}
const t_language *self, FILE *f, t_symbol symbol)
{
const char *name = ts_language_symbol_name(self, symbol);
for (const char *chr = name; *chr; chr++)
{
switch (*chr)
{
case '"':
case '\\':
fputc('\\', f);
fputc(*chr, f);
break;
case '\n':
fputs("\\n", f);
break;
case '\t':
fputs("\\t", f);
break;
default:
fputc(*chr, f);
break;
}
}
}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_LANGUAGE_H_
#endif // TREE_SITTER_LANGUAGE_H_

2
parser/src/length.h
View file

@ -7,7 +7,7 @@
#include "parser/api.h"
typedef struct {
uint32_t bytes;
t_u32 bytes;
t_point extent;
} Length;

104
parser/src/lexer.c
View file

@ -16,33 +16,33 @@
self->debug_buffer); \
}
static const int32_t BYTE_ORDER_MARK = 0xFEFF;
static const t_i32 BYTE_ORDER_MARK = 0xFEFF;
static const t_parser_range DEFAULT_RANGE = {.start_point =
{
.row = 0,
.column = 0,
},
.end_point =
{
.row = UINT32_MAX,
.column = UINT32_MAX,
},
.start_byte = 0,
.end_byte = UINT32_MAX};
{
.row = 0,
.column = 0,
},
.end_point =
{
.row = UINT32_MAX,
.column = UINT32_MAX,
},
.start_byte = 0,
.end_byte = UINT32_MAX};
// Check if the lexer has reached EOF. This state is stored
// by setting the lexer's `current_included_range_index` such that
// it has consumed all of its available ranges.
static bool ts_lexer__eof(const TSLexer *_self)
static bool ts_lexer__eof(const t_lexer *_self)
{
Lexer *self = (Lexer *)_self;
t_liblexer *self = (t_liblexer *)_self;
return self->current_included_range_index == self->included_range_count;
}
// Clear the currently stored chunk of source code, because the lexer's
// position has changed.
static void ts_lexer__clear_chunk(Lexer *self)
static void ts_lexer__clear_chunk(t_liblexer *self)
{
self->chunk = NULL;
self->chunk_size = 0;
@ -51,7 +51,7 @@ static void ts_lexer__clear_chunk(Lexer *self)
// Call the lexer's input callback to obtain a new chunk of source code
// for the current position.
static void ts_lexer__get_chunk(Lexer *self)
static void ts_lexer__get_chunk(t_liblexer *self)
{
self->chunk_start = self->current_position.bytes;
self->chunk =
@ -64,13 +64,13 @@ static void ts_lexer__get_chunk(Lexer *self)
}
}
typedef uint32_t (*UnicodeDecodeFunction)(const uint8_t *chunk, uint32_t size,
int32_t *lookahead);
typedef t_i32 (*UnicodeDecodeFunction)(const t_i8 *chunk, t_i32 size,
t_i32 *lookahead);
uint32_t my_decode(const uint8_t *chunk, uint32_t size, int32_t *lookahead)
t_i32 my_decode(const t_i8 *chunk, t_i32 size, t_i32 *lookahead)
{
(void)(size);
*((uint32_t *)lookahead) = *chunk;
*((t_i32 *)lookahead) = *chunk;
return (1);
}
@ -79,11 +79,11 @@ uint32_t my_decode(const uint8_t *chunk, uint32_t size, int32_t *lookahead)
// Decode the next unicode character in the current chunk of source code.
// This assumes that the lexer has already retrieved a chunk of source
// code that spans the current position.
static void ts_lexer__get_lookahead(Lexer *self)
static void ts_lexer__get_lookahead(t_liblexer *self)
{
uint32_t position_in_chunk =
t_i32 position_in_chunk =
self->current_position.bytes - self->chunk_start;
uint32_t size = self->chunk_size - position_in_chunk;
t_i32 size = self->chunk_size - position_in_chunk;
if (size == 0)
{
@ -92,7 +92,7 @@ static void ts_lexer__get_lookahead(Lexer *self)
return;
}
const uint8_t *chunk = (const uint8_t *)self->chunk + position_in_chunk;
const t_i8 *chunk = (const t_i8 *)self->chunk + position_in_chunk;
UnicodeDecodeFunction decode = my_decode;
self->lookahead_size = decode(chunk, size, &self->data.lookahead);
@ -102,7 +102,7 @@ static void ts_lexer__get_lookahead(Lexer *self)
if (self->data.lookahead == TS_DECODE_ERROR && size < 4)
{
ts_lexer__get_chunk(self);
chunk = (const uint8_t *)self->chunk;
chunk = (const t_i8 *)self->chunk;
size = self->chunk_size;
self->lookahead_size = decode(chunk, size, &self->data.lookahead);
}
@ -113,7 +113,7 @@ static void ts_lexer__get_lookahead(Lexer *self)
}
}
static void ts_lexer_goto(Lexer *self, Length position)
static void ts_lexer_goto(t_liblexer *self, Length position)
{
self->current_position = position;
@ -172,7 +172,7 @@ static void ts_lexer_goto(Lexer *self, Length position)
}
// Intended to be called only from functions that control logging.
static void ts_lexer__do_advance(Lexer *self, bool skip)
static void ts_lexer__do_advance(t_liblexer *self, bool skip)
{
if (self->lookahead_size)
{
@ -235,9 +235,9 @@ static void ts_lexer__do_advance(Lexer *self, bool skip)
// Advance to the next character in the source code, retrieving a new
// chunk of source code if needed.
static void ts_lexer__advance(TSLexer *_self, bool skip)
static void ts_lexer__advance(t_lexer *_self, bool skip)
{
Lexer *self = (Lexer *)_self;
t_liblexer *self = (t_liblexer *)_self;
if (!self->chunk)
return;
ts_lexer__do_advance(self, skip);
@ -245,9 +245,9 @@ static void ts_lexer__advance(TSLexer *_self, bool skip)
// Mark that a token match has completed. This can be called multiple
// times if a longer match is found later.
static void ts_lexer__mark_end(TSLexer *_self)
static void ts_lexer__mark_end(t_lexer *_self)
{
Lexer *self = (Lexer *)_self;
t_liblexer *self = (t_liblexer *)_self;
if (!ts_lexer__eof(&self->data))
{
// If the lexer is right at the beginning of included range,
@ -258,7 +258,8 @@ static void ts_lexer__mark_end(TSLexer *_self)
if (self->current_included_range_index > 0 &&
self->current_position.bytes == current_included_range->start_byte)
{
t_parser_range *previous_included_range = current_included_range - 1;
t_parser_range *previous_included_range =
current_included_range - 1;
self->token_end_position = (Length){
previous_included_range->end_byte,
previous_included_range->end_point,
@ -269,11 +270,11 @@ static void ts_lexer__mark_end(TSLexer *_self)
self->token_end_position = self->current_position;
}
static uint32_t ts_lexer__get_column(TSLexer *_self)
static t_i32 ts_lexer__get_column(t_lexer *_self)
{
Lexer *self = (Lexer *)_self;
t_liblexer *self = (t_liblexer *)_self;
uint32_t goal_byte = self->current_position.bytes;
t_u32 goal_byte = self->current_position.bytes;
self->did_get_column = true;
self->current_position.bytes -= self->current_position.extent.column;
@ -284,7 +285,7 @@ static uint32_t ts_lexer__get_column(TSLexer *_self)
ts_lexer__get_chunk(self);
}
uint32_t result = 0;
t_i32 result = 0;
if (!ts_lexer__eof(_self))
{
ts_lexer__get_lookahead(self);
@ -303,9 +304,9 @@ static uint32_t ts_lexer__get_column(TSLexer *_self)
// Is the lexer at a boundary between two disjoint included ranges of
// source code? This is exposed as an API because some languages' external
// scanners need to perform custom actions at these boundaries.
static bool ts_lexer__is_at_included_range_start(const TSLexer *_self)
static bool ts_lexer__is_at_included_range_start(const t_lexer *_self)
{
const Lexer *self = (const Lexer *)_self;
const t_liblexer *self = (const t_liblexer *)_self;
if (self->current_included_range_index < self->included_range_count)
{
t_parser_range *current_range =
@ -318,9 +319,9 @@ static bool ts_lexer__is_at_included_range_start(const TSLexer *_self)
}
}
void ts_lexer_init(Lexer *self)
void ts_lexer_init(t_liblexer *self)
{
*self = (Lexer){
*self = (t_liblexer){
.data =
{
// The lexer's methods are stored as struct fields so that
@ -349,12 +350,12 @@ void ts_lexer_init(Lexer *self)
ts_lexer_set_included_ranges(self, NULL, 0);
}
void ts_lexer_delete(Lexer *self)
void ts_lexer_delete(t_liblexer *self)
{
free(self->included_ranges);
}
void ts_lexer_set_input(Lexer *self, TSInput input)
void ts_lexer_set_input(t_liblexer *self, TSInput input)
{
self->input = input;
ts_lexer__clear_chunk(self);
@ -363,7 +364,7 @@ void ts_lexer_set_input(Lexer *self, TSInput input)
// Move the lexer to the given position. This doesn't do any work
// if the parser is already at the given position.
void ts_lexer_reset(Lexer *self, Length position)
void ts_lexer_reset(t_liblexer *self, Length position)
{
if (position.bytes != self->current_position.bytes)
{
@ -371,7 +372,7 @@ void ts_lexer_reset(Lexer *self, Length position)
}
}
void ts_lexer_start(Lexer *self)
void ts_lexer_start(t_liblexer *self)
{
self->token_start_position = self->current_position;
self->token_end_position = LENGTH_UNDEFINED;
@ -389,7 +390,7 @@ void ts_lexer_start(Lexer *self)
}
}
void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte)
void ts_lexer_finish(t_liblexer *self, t_i32 *lookahead_end_byte)
{
if (length_is_undefined(self->token_end_position))
{
@ -404,7 +405,7 @@ void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte)
self->token_start_position = self->token_end_position;
}
uint32_t current_lookahead_end_byte = self->current_position.bytes + 1;
t_i32 current_lookahead_end_byte = self->current_position.bytes + 1;
// In order to determine that a byte sequence is invalid UTF8 or UTF16,
// the character decoding algorithm may have looked at the following byte.
@ -421,7 +422,7 @@ void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte)
}
}
void ts_lexer_advance_to_end(Lexer *self)
void ts_lexer_advance_to_end(t_liblexer *self)
{
while (self->chunk)
{
@ -429,13 +430,13 @@ void ts_lexer_advance_to_end(Lexer *self)
}
}
void ts_lexer_mark_end(Lexer *self)
void ts_lexer_mark_end(t_liblexer *self)
{
ts_lexer__mark_end(&self->data);
}
bool ts_lexer_set_included_ranges(Lexer *self, const t_parser_range *ranges,
uint32_t count)
bool ts_lexer_set_included_ranges(t_liblexer *self,
const t_parser_range *ranges, t_u32 count)
{
ranges = &DEFAULT_RANGE;
count = 1;
@ -447,7 +448,8 @@ bool ts_lexer_set_included_ranges(Lexer *self, const t_parser_range *ranges,
return true;
}
t_parser_range *ts_lexer_included_ranges(const Lexer *self, uint32_t *count)
t_parser_range *ts_lexer_included_ranges(const t_liblexer *self,
t_u32 *count)
{
*count = self->included_range_count;
return self->included_ranges;

83
parser/src/lexer.h
View file

@ -1,49 +1,60 @@
/* ************************************************************************** */
/* */
/* ::: :::::::: */
/* lexer.h :+: :+: :+: */
/* +:+ +:+ +:+ */
/* By: maiboyer <maiboyer@student.42.fr> +#+ +:+ +#+ */
/* +#+#+#+#+#+ +#+ */
/* Created: 2024/04/29 16:17:29 by maiboyer #+# #+# */
/* Updated: 2024/04/29 16:55:37 by maiboyer ### ########.fr */
/* */
/* ************************************************************************** */
#ifndef TREE_SITTER_LEXER_H_
#define TREE_SITTER_LEXER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "./length.h"
#include "./parser.h"
#include "./subtree.h"
#include "parser/api.h"
#include "./parser.h"
typedef struct {
TSLexer data;
Length current_position;
Length token_start_position;
Length token_end_position;
#include "me/types.h"
#include "parser/types/types_lexer.h"
t_parser_range *included_ranges;
const char *chunk;
TSInput input;
TSLogger logger;
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
uint32_t included_range_count;
uint32_t current_included_range_index;
uint32_t chunk_start;
uint32_t chunk_size;
uint32_t lookahead_size;
bool did_get_column;
typedef struct s_liblexer
{
t_lexer data;
Length current_position;
Length token_start_position;
Length token_end_position;
char debug_buffer[TREE_SITTER_SERIALIZATION_BUFFER_SIZE];
} Lexer;
t_parser_range *included_ranges;
const char *chunk;
TSInput input;
TSLogger logger;
void ts_lexer_init(Lexer *);
void ts_lexer_delete(Lexer *);
void ts_lexer_set_input(Lexer *, TSInput);
void ts_lexer_reset(Lexer *, Length);
void ts_lexer_start(Lexer *);
void ts_lexer_finish(Lexer *, uint32_t *);
void ts_lexer_advance_to_end(Lexer *);
void ts_lexer_mark_end(Lexer *);
bool ts_lexer_set_included_ranges(Lexer *self, const t_parser_range *ranges, uint32_t count);
t_parser_range *ts_lexer_included_ranges(const Lexer *self, uint32_t *count);
t_u32 included_range_count;
t_u32 current_included_range_index;
t_u32 chunk_start;
t_u32 chunk_size;
t_u32 lookahead_size;
bool did_get_column;
#ifdef __cplusplus
}
#endif
char debug_buffer[TREE_SITTER_SERIALIZATION_BUFFER_SIZE];
} t_liblexer;
#endif // TREE_SITTER_LEXER_H_
void ts_lexer_init(t_liblexer *);
void ts_lexer_delete(t_liblexer *);
void ts_lexer_set_input(t_liblexer *, TSInput);
void ts_lexer_reset(t_liblexer *, Length);
void ts_lexer_start(t_liblexer *);
void ts_lexer_finish(t_liblexer *, t_i32 *);
void ts_lexer_advance_to_end(t_liblexer *);
void ts_lexer_mark_end(t_liblexer *);
bool ts_lexer_set_included_ranges(t_liblexer *self,
const t_parser_range *ranges, t_u32 count);
t_parser_range *ts_lexer_included_ranges(const t_liblexer *self, t_u32 *count);
#endif // TREE_SITTER_LEXER_H_

1351
parser/src/node.c
View file

File diff suppressed because it is too large Load diff

354
parser/src/parser.c
View file

@ -18,9 +18,10 @@
#include <time.h>
#include "me/vec/vec_parser_range.h"
#include "parser/types/types_language.h"
typedef uint64_t TSDuration;
typedef uint64_t TSClock;
typedef t_u64 t_duration;
typedef t_u64 t_clock;
#define LOG(...) \
if (self->lexer.logger.log || self->dot_graph_file) \
@ -102,70 +103,70 @@ static const unsigned MAX_SUMMARY_DEPTH = 16;
static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE;
// static const unsigned OP_COUNT_PER_TIMEOUT_CHECK = 100;
typedef struct
typedef struct s_token_cache
{
Subtree token;
Subtree last_external_token;
uint32_t byte_index;
} TokenCache;
Subtree token;
Subtree last_external_token;
t_u32 byte_index;
} t_token_cache;
struct TSParser
typedef struct s_parser
{
Lexer lexer;
t_liblexer lexer;
Stack *stack;
SubtreePool tree_pool;
const TSLanguage *language;
const t_language *language;
void *wasm_store;
ReduceActionSet reduce_actions;
Subtree finished_tree;
SubtreeArray trailing_extras;
SubtreeArray trailing_extras2;
SubtreeArray scratch_trees;
TokenCache token_cache;
t_token_cache token_cache;
ReusableNode reusable_node;
void *external_scanner_payload;
FILE *dot_graph_file;
TSClock end_clock;
TSDuration timeout_duration;
unsigned accept_count;
unsigned operation_count;
t_clock end_clock;
t_duration timeot_duration;
t_u32 accept_count;
t_u32 operation_count;
const volatile size_t *cancellation_flag;
Subtree old_tree;
t_vec_parser_range included_range_differences;
unsigned included_range_difference_index;
t_u32 included_range_difference_index;
bool has_scanner_error;
};
} t_parser;
typedef struct
typedef struct s_error_status
{
unsigned cost;
unsigned node_count;
int dynamic_precedence;
bool is_in_error;
} ErrorStatus;
t_u32 cost;
t_u32 node_count;
t_i32 dynamic_precedence;
bool is_in_error;
} t_error_status;
typedef enum
typedef enum e_error_comparison
{
ErrorComparisonTakeLeft,
ErrorComparisonPreferLeft,
ErrorComparisonNone,
ErrorComparisonPreferRight,
ErrorComparisonTakeRight,
} ErrorComparison;
} t_error_comparison;
typedef struct
typedef struct s_string_input
{
const char *string;
uint32_t length;
} TSStringInput;
t_u32 length;
} t_string_input;
// StringInput
static const char *ts_string_input_read(void *_self, uint32_t byte,
t_point point, uint32_t *length)
static const char *ts_string_inpt_read(void *_self, t_u32 byte,
t_point point, t_u32 *length)
{
(void)point;
TSStringInput *self = (TSStringInput *)_self;
t_string_input *self = (t_string_input *)_self;
if (byte >= self->length)
{
*length = 0;
@ -180,7 +181,7 @@ static const char *ts_string_input_read(void *_self, uint32_t byte,
// Parser - Private
static void ts_parser__log(TSParser *self)
static void ts_parser__log(t_parser *self)
{
if (self->lexer.logger.log)
{
@ -201,7 +202,7 @@ static void ts_parser__log(TSParser *self)
}
}
static bool ts_parser__breakdown_top_of_stack(TSParser *self,
static bool ts_parser__breakdown_top_of_stack(t_parser *self,
StackVersion version)
{
bool did_break_down = false;
@ -215,13 +216,13 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self,
did_break_down = true;
pending = false;
for (uint32_t i = 0; i < pop.size; i++)
for (t_u32 i = 0; i < pop.size; i++)
{
StackSlice slice = pop.contents[i];
TSStateId state = ts_stack_state(self->stack, slice.version);
t_state_id state = ts_stack_state(self->stack, slice.version);
Subtree parent = *array_front(&slice.subtrees);
for (uint32_t j = 0, n = ts_subtree_child_count(parent); j < n; j++)
for (t_u32 j = 0, n = ts_subtree_child_count(parent); j < n; j++)
{
Subtree child = ts_subtree_children(parent)[j];
pending = ts_subtree_child_count(child) > 0;
@ -241,7 +242,7 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self,
state);
}
for (uint32_t j = 1; j < slice.subtrees.size; j++)
for (t_u32 j = 1; j < slice.subtrees.size; j++)
{
Subtree tree = slice.subtrees.contents[j];
ts_stack_push(self->stack, slice.version, tree, false, state);
@ -258,8 +259,8 @@ static bool ts_parser__breakdown_top_of_stack(TSParser *self,
return did_break_down;
}
static void ts_parser__breakdown_lookahead(TSParser *self, Subtree *lookahead,
TSStateId state,
static void ts_parser__breakdown_lookahead(t_parser *self, Subtree *lookahead,
t_state_id state,
ReusableNode *reusable_node)
{
bool did_descend = false;
@ -281,8 +282,9 @@ static void ts_parser__breakdown_lookahead(TSParser *self, Subtree *lookahead,
}
}
static ErrorComparison ts_parser__compare_versions(TSParser *self,
ErrorStatus a, ErrorStatus b)
static t_error_comparison ts_parser__compare_versions(t_parser *self,
t_error_status a,
t_error_status b)
{
(void)self;
if (!a.is_in_error && b.is_in_error)
@ -340,14 +342,14 @@ static ErrorComparison ts_parser__compare_versions(TSParser *self,
return ErrorComparisonNone;
}
static ErrorStatus ts_parser__version_status(TSParser *self,
StackVersion version)
static t_error_status ts_parser__version_status(t_parser *self,
StackVersion version)
{
unsigned cost = ts_stack_error_cost(self->stack, version);
bool is_paused = ts_stack_is_paused(self->stack, version);
if (is_paused)
cost += ERROR_COST_PER_SKIPPED_TREE;
return (ErrorStatus){
return (t_error_status){
.cost = cost,
.node_count = ts_stack_node_count_since_error(self->stack, version),
.dynamic_precedence = ts_stack_dynamic_precedence(self->stack, version),
@ -355,7 +357,7 @@ static ErrorStatus ts_parser__version_status(TSParser *self,
is_paused || ts_stack_state(self->stack, version) == ERROR_STATE};
}
static bool ts_parser__better_version_exists(TSParser *self,
static bool ts_parser__better_version_exists(t_parser *self,
StackVersion version,
bool is_in_error, unsigned cost)
{
@ -365,8 +367,8 @@ static bool ts_parser__better_version_exists(TSParser *self,
return true;
}
Length position = ts_stack_position(self->stack, version);
ErrorStatus status = {
Length position = ts_stack_position(self->stack, version);
t_error_status status = {
.cost = cost,
.is_in_error = is_in_error,
.dynamic_precedence = ts_stack_dynamic_precedence(self->stack, version),
@ -379,7 +381,7 @@ static bool ts_parser__better_version_exists(TSParser *self,
if (i == version || !ts_stack_is_active(self->stack, i) ||
ts_stack_position(self->stack, i).bytes < position.bytes)
continue;
ErrorStatus status_i = ts_parser__version_status(self, i);
t_error_status status_i = ts_parser__version_status(self, i);
switch (ts_parser__compare_versions(self, status, status_i))
{
case ErrorComparisonTakeRight:
@ -396,19 +398,19 @@ static bool ts_parser__better_version_exists(TSParser *self,
return false;
}
static bool ts_parser__call_main_lex_fn(TSParser *self, TSLexMode lex_mode)
static bool ts_parser__call_main_lex_fn(t_parser *self, t_lex_modes lex_mode)
{
(void)(lex_mode);
return self->language->lex_fn(&self->lexer.data, lex_mode.lex_state);
}
static bool ts_parser__call_keyword_lex_fn(TSParser *self, TSLexMode lex_mode)
static bool ts_parser__call_keyword_lex_fn(t_parser *self, t_lex_modes lex_mode)
{
(void)(lex_mode);
return self->language->keyword_lex_fn(&self->lexer.data, 0);
}
static void ts_parser__external_scanner_create(TSParser *self)
static void ts_parser__external_scanner_create(t_parser *self)
{
if (self->language && self->language->external_scanner.states)
{
@ -417,7 +419,7 @@ static void ts_parser__external_scanner_create(TSParser *self)
}
}
static void ts_parser__external_scanner_destroy(TSParser *self)
static void ts_parser__external_scanner_destroy(t_parser *self)
{
if (self->language && self->external_scanner_payload &&
self->language->external_scanner.destroy)
@ -428,17 +430,17 @@ static void ts_parser__external_scanner_destroy(TSParser *self)
self->external_scanner_payload = NULL;
}
static unsigned ts_parser__external_scanner_serialize(TSParser *self)
static unsigned ts_parser__external_scanner_serialize(t_parser *self)
{
return self->language->external_scanner.serialize(
self->external_scanner_payload, self->lexer.debug_buffer);
}
static void ts_parser__external_scanner_deserialize(TSParser *self,
static void ts_parser__external_scanner_deserialize(t_parser *self,
Subtree external_token)
{
const char *data = NULL;
uint32_t length = 0;
t_u32 length = 0;
if (external_token.ptr)
{
data = ts_external_scanner_state_data(
@ -450,8 +452,8 @@ static void ts_parser__external_scanner_deserialize(TSParser *self,
data, length);
}
static bool ts_parser__external_scanner_scan(TSParser *self,
TSStateId external_lex_state)
static bool ts_parser__external_scanner_scan(t_parser *self,
t_state_id external_lex_state)
{
const bool *valid_external_tokens =
ts_language_enabled_external_tokens(self->language, external_lex_state);
@ -460,26 +462,26 @@ static bool ts_parser__external_scanner_scan(TSParser *self,
valid_external_tokens);
}
static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state,
static bool ts_parser__can_reuse_first_leaf(t_parser *self, t_state_id state,
Subtree tree,
TableEntry *table_entry)
{
TSLexMode current_lex_mode = self->language->lex_modes[state];
TSSymbol leaf_symbol = ts_subtree_leaf_symbol(tree);
TSStateId leaf_state = ts_subtree_leaf_parse_state(tree);
TSLexMode leaf_lex_mode = self->language->lex_modes[leaf_state];
t_lex_modes current_lex_mode = self->language->lex_modes[state];
t_symbol leaf_symbol = ts_subtree_leaf_symbol(tree);
t_state_id leaf_state = ts_subtree_leaf_parse_state(tree);
t_lex_modes leaf_lex_mode = self->language->lex_modes[leaf_state];
// At the end of a non-terminal extra node, the lexer normally returns
// NULL, which indicates that the parser should look for a reduce action
// at symbol `0`. Avoid reusing tokens in this situation to ensure that
// the same thing happens when incrementally reparsing.
if (current_lex_mode.lex_state == (uint16_t)(-1))
if (current_lex_mode.lex_state == (t_u16)(-1))
return false;
// If the token was created in a state with the same set of lookaheads, it
// is reusable.
if (table_entry->action_count > 0 &&
memcmp(&leaf_lex_mode, &current_lex_mode, sizeof(TSLexMode)) == 0 &&
memcmp(&leaf_lex_mode, &current_lex_mode, sizeof(t_lex_modes)) == 0 &&
(leaf_symbol != self->language->keyword_capture_token ||
(!ts_subtree_is_keyword(tree) &&
ts_subtree_parse_state(tree) == state)))
@ -494,11 +496,11 @@ static bool ts_parser__can_reuse_first_leaf(TSParser *self, TSStateId state,
return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable;
}
static Subtree ts_parser__lex(TSParser *self, StackVersion version,
TSStateId parse_state)
static Subtree ts_parser__lex(t_parser *self, StackVersion version,
t_state_id parse_state)
{
TSLexMode lex_mode = self->language->lex_modes[parse_state];
if (lex_mode.lex_state == (uint16_t)-1)
t_lex_modes lex_mode = self->language->lex_modes[parse_state];
if (lex_mode.lex_state == (t_u16)-1)
{
LOG("no_lookahead_after_non_terminal_extra");
return NULL_SUBTREE;
@ -512,11 +514,11 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version,
bool error_mode = parse_state == ERROR_STATE;
bool skipped_error = false;
bool called_get_column = false;
int32_t first_error_character = 0;
t_i32 first_error_character = 0;
Length error_start_position = length_zero();
Length error_end_position = length_zero();
uint32_t lookahead_end_byte = 0;
uint32_t external_scanner_state_len = 0;
t_i32 lookahead_end_byte = 0;
t_i32 external_scanner_state_len = 0;
bool external_scanner_state_changed = false;
ts_lexer_reset(&self->lexer, start_position);
@ -626,7 +628,7 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version,
{
Length padding = length_sub(error_start_position, start_position);
Length size = length_sub(error_end_position, error_start_position);
uint32_t lookahead_bytes =
t_u32 lookahead_bytes =
lookahead_end_byte - error_end_position.bytes;
result = ts_subtree_new_error(&self->tree_pool, first_error_character,
padding, size, lookahead_bytes,
@ -635,12 +637,12 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version,
else
{
bool is_keyword = false;
TSSymbol symbol = self->lexer.data.result_symbol;
t_symbol symbol = self->lexer.data.result_symbol;
Length padding =
length_sub(self->lexer.token_start_position, start_position);
Length size = length_sub(self->lexer.token_end_position,
self->lexer.token_start_position);
uint32_t lookahead_bytes =
t_u32 lookahead_bytes =
lookahead_end_byte - self->lexer.token_end_position.bytes;
if (found_external_token)
@ -649,7 +651,7 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version,
}
else if (symbol == self->language->keyword_capture_token && symbol != 0)
{
uint32_t end_byte = self->lexer.token_end_position.bytes;
t_u32 end_byte = self->lexer.token_end_position.bytes;
ts_lexer_reset(&self->lexer, self->lexer.token_start_position);
ts_lexer_start(&self->lexer);
@ -671,11 +673,11 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version,
if (found_external_token)
{
MutableSubtree mut_result = ts_subtree_to_mut_unsafe(result);
MutableSubtree mt_result = ts_subtree_to_mt_unsafe(result);
ts_external_scanner_state_init(
&mut_result.ptr->external_scanner_state,
&mt_result.ptr->external_scanner_state,
self->lexer.debug_buffer, external_scanner_state_len);
mut_result.ptr->has_external_scanner_state_change =
mt_result.ptr->has_external_scanner_state_change =
external_scanner_state_changed;
}
}
@ -685,12 +687,12 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version,
return result;
}
static Subtree ts_parser__get_cached_token(TSParser *self, TSStateId state,
static Subtree ts_parser__get_cached_token(t_parser *self, t_state_id state,
size_t position,
Subtree last_external_token,
TableEntry *table_entry)
{
TokenCache *cache = &self->token_cache;
t_token_cache *cache = &self->token_cache;
if (cache->token.ptr && cache->byte_index == position &&
ts_subtree_external_scanner_state_eq(cache->last_external_token,
last_external_token))
@ -707,11 +709,11 @@ static Subtree ts_parser__get_cached_token(TSParser *self, TSStateId state,
return NULL_SUBTREE;
}
static void ts_parser__set_cached_token(TSParser *self, uint32_t byte_index,
static void ts_parser__set_cached_token(t_parser *self, t_u32 byte_index,
Subtree last_external_token,
Subtree token)
{
TokenCache *cache = &self->token_cache;
t_token_cache *cache = &self->token_cache;
if (token.ptr)
ts_subtree_retain(token);
if (last_external_token.ptr)
@ -725,16 +727,16 @@ static void ts_parser__set_cached_token(TSParser *self, uint32_t byte_index,
cache->last_external_token = last_external_token;
}
static Subtree ts_parser__reuse_node(TSParser *self, StackVersion version,
TSStateId *state, uint32_t position,
static Subtree ts_parser__reuse_node(t_parser *self, StackVersion version,
t_state_id *state, t_u32 position,
Subtree last_external_token,
TableEntry *table_entry)
{
Subtree result;
while ((result = reusable_node_tree(&self->reusable_node)).ptr)
{
uint32_t byte_offset = reusable_node_byte_offset(&self->reusable_node);
uint32_t end_byte_offset = byte_offset + ts_subtree_total_bytes(result);
t_u32 byte_offset = reusable_node_byte_offset(&self->reusable_node);
t_u32 end_byte_offset = byte_offset + ts_subtree_total_bytes(result);
// Do not reuse an EOF node if the included ranges array has changes
// later on in the file.
@ -797,7 +799,7 @@ static Subtree ts_parser__reuse_node(TSParser *self, StackVersion version,
continue;
}
TSSymbol leaf_symbol = ts_subtree_leaf_symbol(result);
t_symbol leaf_symbol = ts_subtree_leaf_symbol(result);
ts_language_table_entry(self->language, *state, leaf_symbol,
table_entry);
if (!ts_parser__can_reuse_first_leaf(self, *state, result, table_entry))
@ -821,7 +823,7 @@ static Subtree ts_parser__reuse_node(TSParser *self, StackVersion version,
// The decision is based on the trees' error costs (if any), their dynamic
// precedence, and finally, as a default, by a recursive comparison of the
// trees' symbols.
static bool ts_parser__select_tree(TSParser *self, Subtree left, Subtree right)
static bool ts_parser__select_parse_tree(t_parser *self, Subtree left, Subtree right)
{
if (!left.ptr)
return true;
@ -886,7 +888,7 @@ static bool ts_parser__select_tree(TSParser *self, Subtree left, Subtree right)
// Determine if a given tree's children should be replaced by an alternative
// array of children.
static bool ts_parser__select_children(TSParser *self, Subtree left,
static bool ts_parser__select_children(t_parser *self, Subtree left,
const SubtreeArray *children)
{
array_assign(&self->scratch_trees, children);
@ -898,12 +900,12 @@ static bool ts_parser__select_children(TSParser *self, Subtree left,
MutableSubtree scratch_tree = ts_subtree_new_node(
ts_subtree_symbol(left), &self->scratch_trees, 0, self->language);
return ts_parser__select_tree(self, left,
return ts_parser__select_parse_tree(self, left,
ts_subtree_from_mut(scratch_tree));
}
static void ts_parser__shift(TSParser *self, StackVersion version,
TSStateId state, Subtree lookahead, bool extra)
static void ts_parser__shift(t_parser *self, StackVersion version,
t_state_id state, Subtree lookahead, bool extra)
{
bool is_leaf = ts_subtree_child_count(lookahead) == 0;
Subtree subtree_to_push = lookahead;
@ -924,13 +926,13 @@ static void ts_parser__shift(TSParser *self, StackVersion version,
}
}
static StackVersion ts_parser__reduce(TSParser *self, StackVersion version,
TSSymbol symbol, uint32_t count,
static StackVersion ts_parser__reduce(t_parser *self, StackVersion version,
t_symbol symbol, t_u32 count,
int dynamic_precedence,
uint16_t production_id, bool is_fragile,
t_u16 production_id, bool is_fragile,
bool end_of_non_terminal_extra)
{
uint32_t initial_version_count = ts_stack_version_count(self->stack);
t_u32 initial_version_count = ts_stack_version_count(self->stack);
// Pop the given number of nodes from the given version of the parse stack.
// If stack versions have previously merged, then there may be more than one
@ -938,8 +940,8 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version,
// contain the popped children, and push it onto the stack in place of the
// children.
StackSliceArray pop = ts_stack_pop_count(self->stack, version, count);
uint32_t removed_version_count = 0;
for (uint32_t i = 0; i < pop.size; i++)
t_u32 removed_version_count = 0;
for (t_u32 i = 0; i < pop.size; i++)
{
StackSlice slice = pop.contents[i];
StackVersion slice_version = slice.version - removed_version_count;
@ -1007,8 +1009,8 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version,
}
}
TSStateId state = ts_stack_state(self->stack, slice_version);
TSStateId next_state =
t_state_id state = ts_stack_state(self->stack, slice_version);
t_state_id next_state =
ts_language_next_state(self->language, state, symbol);
if (end_of_non_terminal_extra && next_state == state)
{
@ -1030,7 +1032,7 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version,
// were previously on top of the stack.
ts_stack_push(self->stack, slice_version, ts_subtree_from_mut(parent),
false, next_state);
for (uint32_t j = 0; j < self->trailing_extras.size; j++)
for (t_u32 j = 0; j < self->trailing_extras.size; j++)
{
ts_stack_push(self->stack, slice_version,
self->trailing_extras.contents[j], false, next_state);
@ -1054,27 +1056,27 @@ static StackVersion ts_parser__reduce(TSParser *self, StackVersion version,
: STACK_VERSION_NONE;
}
static void ts_parser__accept(TSParser *self, StackVersion version,
static void ts_parser__accept(t_parser *self, StackVersion version,
Subtree lookahead)
{
assert(ts_subtree_is_eof(lookahead));
ts_stack_push(self->stack, version, lookahead, false, 1);
StackSliceArray pop = ts_stack_pop_all(self->stack, version);
for (uint32_t i = 0; i < pop.size; i++)
for (t_u32 i = 0; i < pop.size; i++)
{
SubtreeArray trees = pop.contents[i].subtrees;
Subtree root = NULL_SUBTREE;
for (uint32_t j = trees.size - 1; j + 1 > 0; j--)
for (t_u32 j = trees.size - 1; j + 1 > 0; j--)
{
Subtree tree = trees.contents[j];
if (!ts_subtree_extra(tree))
{
assert(!tree.data.is_inline);
uint32_t child_count = ts_subtree_child_count(tree);
t_u32 child_count = ts_subtree_child_count(tree);
const Subtree *children = ts_subtree_children(tree);
for (uint32_t k = 0; k < child_count; k++)
for (t_u32 k = 0; k < child_count; k++)
{
ts_subtree_retain(children[k]);
}
@ -1092,7 +1094,7 @@ static void ts_parser__accept(TSParser *self, StackVersion version,
if (self->finished_tree.ptr)
{
if (ts_parser__select_tree(self, self->finished_tree, root))
if (ts_parser__select_parse_tree(self, self->finished_tree, root))
{
ts_subtree_release(&self->tree_pool, self->finished_tree);
self->finished_tree = root;
@ -1113,15 +1115,15 @@ static void ts_parser__accept(TSParser *self, StackVersion version,
}
static bool ts_parser__do_all_potential_reductions(
TSParser *self, StackVersion starting_version, TSSymbol lookahead_symbol)
t_parser *self, StackVersion starting_version, t_symbol lookahead_symbol)
{
uint32_t initial_version_count = ts_stack_version_count(self->stack);
t_u32 initial_version_count = ts_stack_version_count(self->stack);
bool can_shift_lookahead_symbol = false;
StackVersion version = starting_version;
for (unsigned i = 0; true; i++)
{
uint32_t version_count = ts_stack_version_count(self->stack);
t_u32 version_count = ts_stack_version_count(self->stack);
if (version >= version_count)
break;
@ -1137,11 +1139,11 @@ static bool ts_parser__do_all_potential_reductions(
if (merged)
continue;
TSStateId state = ts_stack_state(self->stack, version);
t_state_id state = ts_stack_state(self->stack, version);
bool has_shift_action = false;
array_clear(&self->reduce_actions);
TSSymbol first_symbol, end_symbol;
t_symbol first_symbol, end_symbol;
if (lookahead_symbol != 0)
{
first_symbol = lookahead_symbol;
@ -1153,21 +1155,21 @@ static bool ts_parser__do_all_potential_reductions(
end_symbol = self->language->token_count;
}
for (TSSymbol symbol = first_symbol; symbol < end_symbol; symbol++)
for (t_symbol symbol = first_symbol; symbol < end_symbol; symbol++)
{
TableEntry entry;
ts_language_table_entry(self->language, state, symbol, &entry);
for (uint32_t j = 0; j < entry.action_count; j++)
for (t_u32 j = 0; j < entry.action_count; j++)
{
TSParseAction action = entry.actions[j];
t_parse_actions action = entry.actions[j];
switch (action.type)
{
case TSParseActionTypeShift:
case TSParseActionTypeRecover:
case ActionTypeShift:
case ActionTypeRecover:
if (!action.shift.extra && !action.shift.repetition)
has_shift_action = true;
break;
case TSParseActionTypeReduce:
case ActionTypeReduce:
if (action.reduce.child_count > 0)
ts_reduce_action_set_add(
&self->reduce_actions,
@ -1186,7 +1188,7 @@ static bool ts_parser__do_all_potential_reductions(
}
StackVersion reduction_version = STACK_VERSION_NONE;
for (uint32_t j = 0; j < self->reduce_actions.size; j++)
for (t_u32 j = 0; j < self->reduce_actions.size; j++)
{
ReduceAction action = self->reduce_actions.contents[j];
@ -1223,8 +1225,8 @@ static bool ts_parser__do_all_potential_reductions(
return can_shift_lookahead_symbol;
}
static bool ts_parser__recover_to_state(TSParser *self, StackVersion version,
unsigned depth, TSStateId goal_state)
static bool ts_parser__recover_to_state(t_parser *self, StackVersion version,
unsigned depth, t_state_id goal_state)
{
StackSliceArray pop = ts_stack_pop_count(self->stack, version, depth);
StackVersion previous_version = STACK_VERSION_NONE;
@ -1254,7 +1256,7 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version,
{
assert(error_trees.size == 1);
Subtree error_tree = error_trees.contents[0];
uint32_t error_child_count = ts_subtree_child_count(error_tree);
t_u32 error_child_count = ts_subtree_child_count(error_tree);
if (error_child_count > 0)
{
array_splice(&slice.subtrees, 0, 0, error_child_count,
@ -1293,7 +1295,7 @@ static bool ts_parser__recover_to_state(TSParser *self, StackVersion version,
return previous_version != STACK_VERSION_NONE;
}
static void ts_parser__recover(TSParser *self, StackVersion version,
static void ts_parser__recover(t_parser *self, StackVersion version,
Subtree lookahead)
{
bool did_recover = false;
@ -1439,9 +1441,9 @@ static void ts_parser__recover(TSParser *self, StackVersion version,
// If the current lookahead token is an extra token, mark it as extra. This
// means it won't be counted in error cost calculations.
unsigned n;
const TSParseAction *actions = ts_language_actions(
const t_parse_actions *actions = ts_language_actions(
self->language, 1, ts_subtree_symbol(lookahead), &n);
if (n > 0 && actions[n - 1].type == TSParseActionTypeShift &&
if (n > 0 && actions[n - 1].type == ActionTypeShift &&
actions[n - 1].shift.extra)
{
MutableSubtree mutable_lookahead =
@ -1503,16 +1505,16 @@ static void ts_parser__recover(TSParser *self, StackVersion version,
}
}
static void ts_parser__handle_error(TSParser *self, StackVersion version,
static void ts_parser__handle_error(t_parser *self, StackVersion version,
Subtree lookahead)
{
uint32_t previous_version_count = ts_stack_version_count(self->stack);
t_u32 previous_version_count = ts_stack_version_count(self->stack);
// Perform any reductions that can happen in this state, regardless of the
// lookahead. After skipping one or more invalid tokens, the parser might
// find a token that would have allowed a reduction to take place.
ts_parser__do_all_potential_reductions(self, version, 0);
uint32_t version_count = ts_stack_version_count(self->stack);
t_u32 version_count = ts_stack_version_count(self->stack);
Length position = ts_stack_position(self->stack, version);
// Push a discontinuity onto the stack. Merge all of the stack versions that
@ -1522,12 +1524,12 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version,
{
if (!did_insert_missing_token)
{
TSStateId state = ts_stack_state(self->stack, v);
for (TSSymbol missing_symbol = 1;
missing_symbol < (uint16_t)self->language->token_count;
t_state_id state = ts_stack_state(self->stack, v);
for (t_symbol missing_symbol = 1;
missing_symbol < (t_u16)self->language->token_count;
missing_symbol++)
{
TSStateId state_after_missing_symbol = ts_language_next_state(
t_state_id state_after_missing_symbol = ts_language_next_state(
self->language, state, missing_symbol);
if (state_after_missing_symbol == 0 ||
state_after_missing_symbol == state)
@ -1547,7 +1549,7 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version,
ts_lexer_mark_end(&self->lexer);
Length padding =
length_sub(self->lexer.token_end_position, position);
uint32_t lookahead_bytes =
t_u32 lookahead_bytes =
ts_subtree_total_bytes(lookahead) +
ts_subtree_lookahead_bytes(lookahead);
@ -1604,11 +1606,11 @@ static void ts_parser__handle_error(TSParser *self, StackVersion version,
LOG_STACK();
}
static bool ts_parser__advance(TSParser *self, StackVersion version,
static bool ts_parser__advance(t_parser *self, StackVersion version,
bool allow_node_reuse)
{
TSStateId state = ts_stack_state(self->stack, version);
uint32_t position = ts_stack_position(self->stack, version).bytes;
t_state_id state = ts_stack_state(self->stack, version);
t_u32 position = ts_stack_position(self->stack, version).bytes;
Subtree last_external_token =
ts_stack_last_external_token(self->stack, version);
@ -1671,16 +1673,16 @@ static bool ts_parser__advance(TSParser *self, StackVersion version,
// version, whereas SHIFT actions update the existing stack version
// and terminate this loop.
StackVersion last_reduction_version = STACK_VERSION_NONE;
for (uint32_t i = 0; i < table_entry.action_count; i++)
for (t_u32 i = 0; i < table_entry.action_count; i++)
{
TSParseAction action = table_entry.actions[i];
t_parse_actions action = table_entry.actions[i];
switch (action.type)
{
case TSParseActionTypeShift: {
case ActionTypeShift: {
if (action.shift.repetition)
break;
TSStateId next_state;
t_state_id next_state;
if (action.shift.extra)
{
next_state = state;
@ -1707,7 +1709,7 @@ static bool ts_parser__advance(TSParser *self, StackVersion version,
return true;
}
case TSParseActionTypeReduce: {
case ActionTypeReduce: {
bool is_fragile = table_entry.action_count > 1;
bool end_of_non_terminal_extra = lookahead.ptr == NULL;
LOG("reduce sym:%s, child_count:%u",
@ -1724,13 +1726,13 @@ static bool ts_parser__advance(TSParser *self, StackVersion version,
break;
}
case TSParseActionTypeAccept: {
case ActionTypeAccept: {
LOG("accept");
ts_parser__accept(self, version, lookahead);
return true;
}
case TSParseActionTypeRecover: {
case ActionTypeRecover: {
if (ts_subtree_child_count(lookahead) > 0)
{
ts_parser__breakdown_lookahead(
@ -1843,7 +1845,7 @@ static bool ts_parser__advance(TSParser *self, StackVersion version,
}
}
static unsigned ts_parser__condense_stack(TSParser *self)
static unsigned ts_parser__condense_stack(t_parser *self)
{
bool made_changes = false;
unsigned min_error_cost = UINT_MAX;
@ -1859,7 +1861,7 @@ static unsigned ts_parser__condense_stack(TSParser *self)
// Keep track of the minimum error cost of any stack version so
// that it can be returned.
ErrorStatus status_i = ts_parser__version_status(self, i);
t_error_status status_i = ts_parser__version_status(self, i);
if (!status_i.is_in_error && status_i.cost < min_error_cost)
{
min_error_cost = status_i.cost;
@ -1870,7 +1872,7 @@ static unsigned ts_parser__condense_stack(TSParser *self)
// are ordered from most promising to least promising.
for (StackVersion j = 0; j < i; j++)
{
ErrorStatus status_j = ts_parser__version_status(self, j);
t_error_status status_j = ts_parser__version_status(self, j);
switch (ts_parser__compare_versions(self, status_j, status_i))
{
@ -1965,7 +1967,7 @@ static unsigned ts_parser__condense_stack(TSParser *self)
return min_error_cost;
}
static bool ts_parser_has_outstanding_parse(TSParser *self)
static bool ts_parser_has_outstanding_parse(t_parser *self)
{
return (self->external_scanner_payload ||
ts_stack_state(self->stack, 0) != 1 ||
@ -1974,9 +1976,9 @@ static bool ts_parser_has_outstanding_parse(TSParser *self)
// Parser - Public
TSParser *ts_parser_new(void)
t_parser *ts_parser_new(void)
{
TSParser *self = calloc(1, sizeof(TSParser));
t_parser *self = calloc(1, sizeof(t_parser));
ts_lexer_init(&self->lexer);
array_init(&self->reduce_actions);
array_reserve(&self->reduce_actions, 4);
@ -1986,7 +1988,7 @@ TSParser *ts_parser_new(void)
self->reusable_node = reusable_node_new();
self->dot_graph_file = NULL;
self->cancellation_flag = NULL;
self->timeout_duration = 0;
self->timeot_duration = 0;
self->language = NULL;
self->has_scanner_error = false;
self->external_scanner_payload = NULL;
@ -1999,7 +2001,7 @@ TSParser *ts_parser_new(void)
return self;
}
void ts_parser_delete(TSParser *self)
void ts_parser_delete(t_parser *self)
{
if (!self)
return;
@ -2029,12 +2031,12 @@ void ts_parser_delete(TSParser *self)
free(self);
}
const TSLanguage *ts_parser_language(const TSParser *self)
const t_language *ts_parser_language(const t_parser *self)
{
return self->language;
}
bool ts_parser_set_language(TSParser *self, const TSLanguage *language)
bool ts_parser_set_language(t_parser *self, const t_language *language)
{
ts_parser_reset(self);
ts_language_delete(self->language);
@ -2051,17 +2053,17 @@ bool ts_parser_set_language(TSParser *self, const TSLanguage *language)
return true;
}
TSLogger ts_parser_logger(const TSParser *self)
TSLogger ts_parser_logger(const t_parser *self)
{
return self->lexer.logger;
}
void ts_parser_set_logger(TSParser *self, TSLogger logger)
void ts_parser_set_logger(t_parser *self, TSLogger logger)
{
self->lexer.logger = logger;
}
void ts_parser_print_dot_graphs(TSParser *self, int fd)
void ts_parser_print_dot_graphs(t_parser *self, int fd)
{
if (self->dot_graph_file)
{
@ -2082,41 +2084,41 @@ void ts_parser_print_dot_graphs(TSParser *self, int fd)
}
}
const size_t *ts_parser_cancellation_flag(const TSParser *self)
const size_t *ts_parser_cancellation_flag(const t_parser *self)
{
return (const size_t *)self->cancellation_flag;
}
void ts_parser_set_cancellation_flag(TSParser *self, const size_t *flag)
void ts_parser_set_cancellation_flag(t_parser *self, const size_t *flag)
{
self->cancellation_flag = (const volatile size_t *)flag;
}
uint64_t ts_parser_timeout_micros(const TSParser *self)
t_u64 ts_parser_timeot_micros(const t_parser *self)
{
(void)(self);
return 0;
}
void ts_parser_set_timeout_micros(TSParser *self, uint64_t timeout_micros)
void ts_parser_set_timeot_micros(t_parser *self, t_u64 timeot_micros)
{
(void)(timeout_micros);
self->timeout_duration = 0;
(void)(timeot_micros);
self->timeot_duration = 0;
}
bool ts_parser_set_included_ranges(TSParser *self, const t_parser_range *ranges,
uint32_t count)
bool ts_parser_set_included_ranges(t_parser *self, const t_parser_range *ranges,
t_u32 count)
{
return ts_lexer_set_included_ranges(&self->lexer, ranges, count);
}
const t_parser_range *ts_parser_included_ranges(const TSParser *self,
uint32_t *count)
const t_parser_range *ts_parser_included_ranges(const t_parser *self,
t_u32 *count)
{
return ts_lexer_included_ranges(&self->lexer, count);
}
void ts_parser_reset(TSParser *self)
void ts_parser_reset(t_parser *self)
{
ts_parser__external_scanner_destroy(self);
@ -2139,9 +2141,9 @@ void ts_parser_reset(TSParser *self)
self->has_scanner_error = false;
}
TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input)
t_parse_tree *ts_parser_parse(t_parser *self, const t_parse_tree *old_tree, TSInput input)
{
TSTree *result = NULL;
t_parse_tree *result = NULL;
old_tree = NULL;
(void)(old_tree);
if (!self->language || !input.read)
@ -2167,7 +2169,7 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input)
self->operation_count = 0;
uint32_t position = 0, last_position = 0, version_count = 0;
t_u32 position = 0, last_position = 0, version_count = 0;
do
{
for (StackVersion version = 0;
@ -2252,22 +2254,22 @@ exit:
return result;
}
TSTree *ts_parser_parse_string(TSParser *self, const TSTree *old_tree,
const char *string, uint32_t length)
t_parse_tree *ts_parser_parse_string(t_parser *self, const t_parse_tree *old_tree,
const char *string, t_u32 length)
{
return ts_parser_parse_string_encoding(self, old_tree, string, length,
TSInputEncodingUTF8);
}
TSTree *ts_parser_parse_string_encoding(TSParser *self, const TSTree *old_tree,
const char *string, uint32_t length,
t_parse_tree *ts_parser_parse_string_encoding(t_parser *self, const t_parse_tree *old_tree,
const char *string, t_u32 length,
TSInputEncoding encoding)
{
TSStringInput input = {string, length};
t_string_input input = {string, length};
return ts_parser_parse(self, old_tree,
(TSInput){
&input,
ts_string_input_read,
ts_string_inpt_read,
encoding,
});
}

265
parser/src/parser.h
View file

@ -1,265 +0,0 @@
#ifndef TREE_SITTER_PARSER_H_
#define TREE_SITTER_PARSER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#define ts_builtin_sym_error ((TSSymbol)-1)
#define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
#ifndef TREE_SITTER_API_H_
typedef uint16_t TSStateId;
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
#endif
typedef struct {
TSFieldId field_id;
uint8_t child_index;
bool inherited;
} TSFieldMapEntry;
typedef struct {
uint16_t index;
uint16_t length;
} TSFieldMapSlice;
typedef struct {
bool visible;
bool named;
bool supertype;
} TSSymbolMetadata;
typedef struct TSLexer TSLexer;
struct TSLexer {
int32_t lookahead;
TSSymbol result_symbol;
void (*advance)(TSLexer *, bool);
void (*mark_end)(TSLexer *);
uint32_t (*get_column)(TSLexer *);
bool (*is_at_included_range_start)(const TSLexer *);
bool (*eof)(const TSLexer *);
};
typedef enum {
TSParseActionTypeShift,
TSParseActionTypeReduce,
TSParseActionTypeAccept,
TSParseActionTypeRecover,
} TSParseActionType;
typedef union {
struct {
uint8_t type;
TSStateId state;
bool extra;
bool repetition;
} shift;
struct {
uint8_t type;
uint8_t child_count;
TSSymbol symbol;
int16_t dynamic_precedence;
uint16_t production_id;
} reduce;
uint8_t type;
} TSParseAction;
typedef struct {
uint16_t lex_state;
uint16_t external_lex_state;
} TSLexMode;
typedef union {
TSParseAction action;
struct {
uint8_t count;
bool reusable;
} entry;
} TSParseActionEntry;
typedef struct {
int32_t start;
int32_t end;
} TSCharacterRange;
struct TSLanguage {
uint32_t version;
uint32_t symbol_count;
uint32_t alias_count;
uint32_t token_count;
uint32_t external_token_count;
uint32_t state_count;
uint32_t large_state_count;
uint32_t production_id_count;
uint32_t field_count;
uint16_t max_alias_sequence_length;
const uint16_t *parse_table;
const uint16_t *small_parse_table;
const uint32_t *small_parse_table_map;
const TSParseActionEntry *parse_actions;
const char * const *symbol_names;
const char * const *field_names;
const TSFieldMapSlice *field_map_slices;
const TSFieldMapEntry *field_map_entries;
const TSSymbolMetadata *symbol_metadata;
const TSSymbol *public_symbol_map;
const uint16_t *alias_map;
const TSSymbol *alias_sequences;
const TSLexMode *lex_modes;
bool (*lex_fn)(TSLexer *, TSStateId);
bool (*keyword_lex_fn)(TSLexer *, TSStateId);
TSSymbol keyword_capture_token;
struct {
const bool *states;
const TSSymbol *symbol_map;
void *(*create)(void);
void (*destroy)(void *);
bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
unsigned (*serialize)(void *, char *);
void (*deserialize)(void *, const char *, unsigned);
} external_scanner;
const TSStateId *primary_state_ids;
};
static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
uint32_t index = 0;
uint32_t size = len - index;
while (size > 1) {
uint32_t half_size = size / 2;
uint32_t mid_index = index + half_size;
TSCharacterRange *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end) {
return true;
} else if (lookahead > range->end) {
index = mid_index;
}
size -= half_size;
}
TSCharacterRange *range = &ranges[index];
return (lookahead >= range->start && lookahead <= range->end);
}
/*
* Lexer Macros
*/
#ifdef _MSC_VER
#define UNUSED __pragma(warning(suppress : 4101))
#else
#define UNUSED __attribute__((unused))
#endif
#define START_LEXER() \
bool result = false; \
bool skip = false; \
UNUSED \
bool eof = false; \
int32_t lookahead; \
goto start; \
next_state: \
lexer->advance(lexer, skip); \
start: \
skip = false; \
lookahead = lexer->lookahead;
#define ADVANCE(state_value) \
{ \
state = state_value; \
goto next_state; \
}
#define ADVANCE_MAP(...) \
{ \
static const uint16_t map[] = { __VA_ARGS__ }; \
for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \
if (map[i] == lookahead) { \
state = map[i + 1]; \
goto next_state; \
} \
} \
}
#define SKIP(state_value) \
{ \
skip = true; \
state = state_value; \
goto next_state; \
}
#define ACCEPT_TOKEN(symbol_value) \
result = true; \
lexer->result_symbol = symbol_value; \
lexer->mark_end(lexer);
#define END_STATE() return result;
/*
* Parse Table Macros
*/
#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
#define STATE(id) id
#define ACTIONS(id) id
#define SHIFT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = (state_value) \
} \
}}
#define SHIFT_REPEAT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = (state_value), \
.repetition = true \
} \
}}
#define SHIFT_EXTRA() \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.extra = true \
} \
}}
#define REDUCE(symbol_name, children, precedence, prod_id) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_name, \
.child_count = children, \
.dynamic_precedence = precedence, \
.production_id = prod_id \
}, \
}}
#define RECOVER() \
{{ \
.type = TSParseActionTypeRecover \
}}
#define ACCEPT_INPUT() \
{{ \
.type = TSParseActionTypeAccept \
}}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_PARSER_H_

6
parser/src/reduce_action.h
View file

@ -9,8 +9,8 @@ extern "C" {
#include "parser/api.h"
typedef struct {
uint32_t count;
TSSymbol symbol;
t_u32 count;
t_symbol symbol;
int dynamic_precedence;
unsigned short production_id;
} ReduceAction;
@ -19,7 +19,7 @@ typedef Array(ReduceAction) ReduceActionSet;
static inline void ts_reduce_action_set_add(ReduceActionSet *self,
ReduceAction new_action) {
for (uint32_t i = 0; i < self->size; i++) {
for (t_u32 i = 0; i < self->size; i++) {
ReduceAction action = self->contents[i];
if (action.symbol == new_action.symbol && action.count == new_action.count)
return;

10
parser/src/reusable_node.h
View file

@ -2,8 +2,8 @@
typedef struct {
Subtree tree;
uint32_t child_index;
uint32_t byte_offset;
t_u32 child_index;
t_u32 byte_offset;
} StackEntry;
typedef struct {
@ -26,7 +26,7 @@ static inline Subtree reusable_node_tree(ReusableNode *self) {
: NULL_SUBTREE;
}
static inline uint32_t reusable_node_byte_offset(ReusableNode *self) {
static inline t_u32 reusable_node_byte_offset(ReusableNode *self) {
return self->stack.size > 0
? self->stack.contents[self->stack.size - 1].byte_offset
: UINT32_MAX;
@ -38,13 +38,13 @@ static inline void reusable_node_delete(ReusableNode *self) {
static inline void reusable_node_advance(ReusableNode *self) {
StackEntry last_entry = *array_back(&self->stack);
uint32_t byte_offset = last_entry.byte_offset + ts_subtree_total_bytes(last_entry.tree);
t_u32 byte_offset = last_entry.byte_offset + ts_subtree_total_bytes(last_entry.tree);
if (ts_subtree_has_external_tokens(last_entry.tree)) {
self->last_external_token = ts_subtree_last_external_token(last_entry.tree);
}
Subtree tree;
uint32_t next_index;
t_u32 next_index;
do {
StackEntry popped_entry = array_pop(&self->stack);
next_index = popped_entry.child_index + 1;

69
parser/src/scanner.c
View file

@ -1,11 +1,14 @@
#include "array.h"
#include "parser.h"
#include "parser/types/types_lexer.h"
#include <assert.h>
#include <ctype.h>
#include <string.h>
#include <wctype.h>
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
enum TokenType
{
HEREDOC_START,
@ -64,18 +67,18 @@ static inline t_heredoc heredoc_new(void)
typedef struct s_scanner
{
uint8_t last_glob_paren_depth;
t_u8 last_glob_paren_depth;
bool ext_was_in_double_quote;
bool ext_saw_outside_quote;
Array(t_heredoc) heredocs;
} t_scanner;
static inline void advance(TSLexer *lexer)
static inline void advance(t_lexer *lexer)
{
lexer->advance(lexer, false);
}
static inline void skip(TSLexer *lexer)
static inline void skip(t_lexer *lexer)
{
lexer->advance(lexer, true);
}
@ -104,7 +107,7 @@ static inline void reset_heredoc(t_heredoc *heredoc)
static inline void reset(t_scanner *scanner)
{
uint32_t i;
t_u32 i;
i = 0;
while (i < scanner->heredocs.size)
@ -116,8 +119,8 @@ static inline void reset(t_scanner *scanner)
static unsigned serialize(t_scanner *scanner, char *buffer)
{
uint32_t size;
uint32_t i;
t_u32 size;
t_u32 i;
t_heredoc *heredoc;
size = 0;
@ -135,8 +138,8 @@ static unsigned serialize(t_scanner *scanner, char *buffer)
buffer[size++] = (char)heredoc->is_raw;
buffer[size++] = (char)heredoc->started;
buffer[size++] = (char)heredoc->allows_indent;
memcpy(&buffer[size], &heredoc->delimiter.size, sizeof(uint32_t));
size += sizeof(uint32_t);
memcpy(&buffer[size], &heredoc->delimiter.size, sizeof(t_u32));
size += sizeof(t_u32);
if (heredoc->delimiter.size > 0)
{
memcpy(&buffer[size], heredoc->delimiter.contents,
@ -150,10 +153,10 @@ static unsigned serialize(t_scanner *scanner, char *buffer)
static void deserialize(t_scanner *scanner, const char *buffer, unsigned length)
{
uint32_t size;
uint32_t heredoc_count;
t_u32 size;
t_u32 heredoc_count;
t_heredoc *heredoc;
uint32_t i;
t_u32 i;
size = 0;
if (length == 0)
@ -178,8 +181,8 @@ static void deserialize(t_scanner *scanner, const char *buffer, unsigned length)
heredoc->is_raw = buffer[size++];
heredoc->started = buffer[size++];
heredoc->allows_indent = buffer[size++];
memcpy(&heredoc->delimiter.size, &buffer[size], sizeof(uint32_t));
size += sizeof(uint32_t);
memcpy(&heredoc->delimiter.size, &buffer[size], sizeof(t_u32));
size += sizeof(t_u32);
array_reserve(&heredoc->delimiter, heredoc->delimiter.size);
if (heredoc->delimiter.size > 0)
{
@ -200,10 +203,10 @@ static void deserialize(t_scanner *scanner, const char *buffer, unsigned length)
* POSIX-mandated substitution, and assumes the default value for
* IFS.
*/
static bool advance_word(TSLexer *lexer, t_string *unquoted_word)
static bool advance_word(t_lexer *lexer, t_string *unquoted_word)
{
bool empty;
int32_t quote;
t_i32 quote;
quote = 0;
empty = true;
@ -230,7 +233,7 @@ static bool advance_word(TSLexer *lexer, t_string *unquoted_word)
return (!empty);
}
static inline bool scan_bare_dollar(TSLexer *lexer)
static inline bool scan_bare_dollar(t_lexer *lexer)
{
while (isspace(lexer->lookahead) && lexer->lookahead != '\n' &&
!lexer->eof(lexer))
@ -248,7 +251,7 @@ static inline bool scan_bare_dollar(TSLexer *lexer)
return (false);
}
static bool scan_heredoc_start(t_heredoc *heredoc, TSLexer *lexer)
static bool scan_heredoc_start(t_heredoc *heredoc, t_lexer *lexer)
{
bool found_delimiter;
@ -266,18 +269,18 @@ static bool scan_heredoc_start(t_heredoc *heredoc, TSLexer *lexer)
return found_delimiter;
}
static bool scan_heredoc_end_identifier(t_heredoc *heredoc, TSLexer *lexer)
static bool scan_heredoc_end_identifier(t_heredoc *heredoc, t_lexer *lexer)
{
reset_string(&heredoc->current_leading_word);
// Scan the first 'n' characters on this line, to see if they match the
// heredoc delimiter
int32_t size;
t_i32 size;
size = 0;
if (heredoc->delimiter.size > 0)
{
while (lexer->lookahead != '\0' && lexer->lookahead != '\n' &&
(int32_t)*array_get(&heredoc->delimiter, size) ==
(t_i32)*array_get(&heredoc->delimiter, size) ==
lexer->lookahead &&
heredoc->current_leading_word.size < heredoc->delimiter.size)
{
@ -293,7 +296,7 @@ static bool scan_heredoc_end_identifier(t_heredoc *heredoc, TSLexer *lexer)
heredoc->delimiter.contents) == 0;
}
static bool scan_heredoc_content(t_scanner *scanner, TSLexer *lexer,
static bool scan_heredoc_content(t_scanner *scanner, t_lexer *lexer,
enum TokenType middle_type,
enum TokenType end_type)
{
@ -422,7 +425,7 @@ static bool scan_heredoc_content(t_scanner *scanner, TSLexer *lexer,
}
}
}
static bool regex_scan(t_scanner *scanner, TSLexer *lexer,
static bool regex_scan(t_scanner *scanner, t_lexer *lexer,
const bool *valid_symbols)
{
(void)(scanner);
@ -451,9 +454,9 @@ static bool regex_scan(t_scanner *scanner, TSLexer *lexer,
bool found_non_alnumdollarunderdash;
bool last_was_escape;
bool in_single_quote;
uint32_t paren_depth;
uint32_t bracket_depth;
uint32_t brace_depth;
t_u32 paren_depth;
t_u32 bracket_depth;
t_u32 brace_depth;
} State;
if (lexer->lookahead == '$' && valid_symbols[REGEX_NO_SLASH])
@ -647,7 +650,7 @@ static bool regex_scan(t_scanner *scanner, TSLexer *lexer,
return (false);
}
static bool extglob_pattern_scan(t_scanner *scanner, TSLexer *lexer,
static bool extglob_pattern_scan(t_scanner *scanner, t_lexer *lexer,
const bool *valid_symbols)
{
if (valid_symbols[EXTGLOB_PATTERN] && !in_error_recovery(valid_symbols))
@ -792,9 +795,9 @@ static bool extglob_pattern_scan(t_scanner *scanner, TSLexer *lexer,
{
bool done;
bool saw_non_alphadot;
uint32_t paren_depth;
uint32_t bracket_depth;
uint32_t brace_depth;
t_u32 paren_depth;
t_u32 bracket_depth;
t_u32 brace_depth;
} State;
State state = {false, was_non_alpha, scanner->last_glob_paren_depth,
@ -923,7 +926,7 @@ static bool extglob_pattern_scan(t_scanner *scanner, TSLexer *lexer,
return (false);
}
static bool expansion_word_scan(t_scanner *scanner, TSLexer *lexer,
static bool expansion_word_scan(t_scanner *scanner, t_lexer *lexer,
const bool *valid_symbols)
{
(void)(scanner);
@ -1027,7 +1030,7 @@ static bool expansion_word_scan(t_scanner *scanner, TSLexer *lexer,
return (false);
}
static bool brace_start_scan(t_scanner *scanner, TSLexer *lexer,
static bool brace_start_scan(t_scanner *scanner, t_lexer *lexer,
const bool *valid_symbols)
{
(void)(scanner);
@ -1079,7 +1082,7 @@ static bool brace_start_scan(t_scanner *scanner, TSLexer *lexer,
}
return (false);
}
static bool scan(t_scanner *scanner, TSLexer *lexer, const bool *valid_symbols)
static bool scan(t_scanner *scanner, t_lexer *lexer, const bool *valid_symbols)
{
if (valid_symbols[CONCAT] && !in_error_recovery(valid_symbols))
{
@ -1485,7 +1488,7 @@ void *tree_sitter_bash_external_scanner_create()
return (scanner);
}
bool tree_sitter_bash_external_scanner_scan(void *payload, TSLexer *lexer,
bool tree_sitter_bash_external_scanner_scan(void *payload, t_lexer *lexer,
const bool *valid_symbols)
{
t_scanner *scanner = (t_scanner *)payload;

54
parser/src/stack.c
View file

@ -27,11 +27,11 @@ typedef struct {
} StackLink;
struct StackNode {
TSStateId state;
t_state_id state;
Length position;
StackLink links[MAX_LINK_COUNT];
short unsigned int link_count;
uint32_t ref_count;
t_u32 ref_count;
unsigned error_cost;
unsigned node_count;
int dynamic_precedence;
@ -40,7 +40,7 @@ struct StackNode {
typedef struct {
StackNode *node;
SubtreeArray subtrees;
uint32_t subtree_count;
t_u32 subtree_count;
bool is_pending;
} StackIterator;
@ -123,8 +123,8 @@ recur:
/// Get the number of nodes in the subtree, for the purpose of measuring
/// how much progress has been made by a given version of the stack.
static uint32_t stack__subtree_node_count(Subtree subtree) {
uint32_t count = ts_subtree_visible_descendant_count(subtree);
static t_u32 stack__subtree_node_count(Subtree subtree) {
t_u32 count = ts_subtree_visible_descendant_count(subtree);
if (ts_subtree_visible(subtree)) count++;
// Count intermediate error nodes even though they are not visible,
@ -139,7 +139,7 @@ static StackNode *stack_node_new(
StackNode *previous_node,
Subtree subtree,
bool is_pending,
TSStateId state,
t_state_id state,
StackNodeArray *pool
) {
StackNode *node = pool->size > 0
@ -234,7 +234,7 @@ static void stack_node_add_link(
for (int j = 0; j < link.node->link_count; j++) {
stack_node_add_link(existing_link->node, link.node->links[j], subtree_pool);
}
int32_t dynamic_precedence = link.node->dynamic_precedence;
t_i32 dynamic_precedence = link.node->dynamic_precedence;
if (link.subtree.ptr) {
dynamic_precedence += ts_subtree_dynamic_precedence(link.subtree);
}
@ -307,7 +307,7 @@ static void ts_stack__add_slice(
StackNode *node,
SubtreeArray *subtrees
) {
for (uint32_t i = self->slices.size - 1; i + 1 > 0; i--) {
for (t_u32 i = self->slices.size - 1; i + 1 > 0; i--) {
StackVersion version = self->slices.contents[i].version;
if (self->heads.contents[version].node == node) {
StackSlice slice = {*subtrees, version};
@ -342,13 +342,13 @@ static StackSliceArray stack__iter(
bool include_subtrees = false;
if (goal_subtree_count >= 0) {
include_subtrees = true;
array_reserve(&new_iterator.subtrees, (uint32_t)ts_subtree_alloc_size(goal_subtree_count) / sizeof(Subtree));
array_reserve(&new_iterator.subtrees, (t_u32)ts_subtree_alloc_size(goal_subtree_count) / sizeof(Subtree));
}
array_push(&self->iterators, new_iterator);
while (self->iterators.size > 0) {
for (uint32_t i = 0, size = self->iterators.size; i < size; i++) {
for (t_u32 i = 0, size = self->iterators.size; i < size; i++) {
StackIterator *iterator = &self->iterators.contents[i];
StackNode *node = iterator->node;
@ -379,7 +379,7 @@ static StackSliceArray stack__iter(
continue;
}
for (uint32_t j = 1; j <= node->link_count; j++) {
for (t_u32 j = 1; j <= node->link_count; j++) {
StackIterator *next_iterator;
StackLink link;
if (j == node->link_count) {
@ -443,12 +443,12 @@ void ts_stack_delete(Stack *self) {
if (self->iterators.contents)
array_delete(&self->iterators);
stack_node_release(self->base_node, &self->node_pool, self->subtree_pool);
for (uint32_t i = 0; i < self->heads.size; i++) {
for (t_u32 i = 0; i < self->heads.size; i++) {
stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool);
}
array_clear(&self->heads);
if (self->node_pool.contents) {
for (uint32_t i = 0; i < self->node_pool.size; i++)
for (t_u32 i = 0; i < self->node_pool.size; i++)
free(self->node_pool.contents[i]);
array_delete(&self->node_pool);
}
@ -456,11 +456,11 @@ void ts_stack_delete(Stack *self) {
free(self);
}
uint32_t ts_stack_version_count(const Stack *self) {
t_u32 ts_stack_version_count(const Stack *self) {
return self->heads.size;
}
TSStateId ts_stack_state(const Stack *self, StackVersion version) {
t_state_id ts_stack_state(const Stack *self, StackVersion version) {
return array_get(&self->heads, version)->node->state;
}
@ -503,7 +503,7 @@ void ts_stack_push(
StackVersion version,
Subtree subtree,
bool pending,
TSStateId state
t_state_id state
) {
StackHead *head = array_get(&self->heads, version);
StackNode *new_node = stack_node_new(head->node, subtree, pending, state, &self->node_pool);
@ -520,7 +520,7 @@ forceinline StackAction pop_count_callback(void *payload, const StackIterator *i
}
}
StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t count) {
StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, t_u32 count) {
return stack__iter(self, version, pop_count_callback, &count, (int)count);
}
@ -593,7 +593,7 @@ typedef struct {
forceinline StackAction summarize_stack_callback(void *payload, const StackIterator *iterator) {
SummarizeStackSession *session = payload;
TSStateId state = iterator->node->state;
t_state_id state = iterator->node->state;
unsigned depth = iterator->subtree_count;
if (depth > session->max_depth) return StackActionStop;
for (unsigned i = session->summary->size - 1; i + 1 > 0; i--) {
@ -664,7 +664,7 @@ void ts_stack_remove_version(Stack *self, StackVersion version) {
void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) {
if (v1 == v2) return;
assert(v2 < v1);
assert((uint32_t)v1 < self->heads.size);
assert((t_u32)v1 < self->heads.size);
StackHead *source_head = &self->heads.contents[v1];
StackHead *target_head = &self->heads.contents[v2];
if (target_head->summary && !source_head->summary) {
@ -696,7 +696,7 @@ bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2) {
if (!ts_stack_can_merge(self, version1, version2)) return false;
StackHead *head1 = &self->heads.contents[version1];
StackHead *head2 = &self->heads.contents[version2];
for (uint32_t i = 0; i < head2->node->link_count; i++) {
for (t_u32 i = 0; i < head2->node->link_count; i++) {
stack_node_add_link(head1->node, head2->node->links[i], self->subtree_pool);
}
if (head1->node->state == ERROR_STATE) {
@ -752,7 +752,7 @@ Subtree ts_stack_resume(Stack *self, StackVersion version) {
void ts_stack_clear(Stack *self) {
stack_node_retain(self->base_node);
for (uint32_t i = 0; i < self->heads.size; i++) {
for (t_u32 i = 0; i < self->heads.size; i++) {
stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool);
}
array_clear(&self->heads);
@ -764,7 +764,7 @@ void ts_stack_clear(Stack *self) {
}));
}
bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) {
bool ts_stack_print_dot_graph(Stack *self, const t_language *language, FILE *f) {
array_reserve(&self->iterators, 32);
if (!f) f = stderr;
@ -775,7 +775,7 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
Array(StackNode *) visited_nodes = array_new();
array_clear(&self->iterators);
for (uint32_t i = 0; i < self->heads.size; i++) {
for (t_u32 i = 0; i < self->heads.size; i++) {
StackHead *head = &self->heads.contents[i];
if (head->status == StackStatusHalted) continue;
@ -794,14 +794,14 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
if (head->summary) {
fprintf(f, "\nsummary:");
for (uint32_t j = 0; j < head->summary->size; j++) fprintf(f, " %u", head->summary->contents[j].state);
for (t_u32 j = 0; j < head->summary->size; j++) fprintf(f, " %u", head->summary->contents[j].state);
}
if (head->last_external_token.ptr) {
const ExternalScannerState *state = &head->last_external_token.ptr->external_scanner_state;
const char *data = ts_external_scanner_state_data(state);
fprintf(f, "\nexternal_scanner_state:");
for (uint32_t j = 0; j < state->length; j++) fprintf(f, " %2X", data[j]);
for (t_u32 j = 0; j < state->length; j++) fprintf(f, " %2X", data[j]);
}
fprintf(f, "\"]\n");
@ -814,11 +814,11 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
while (!all_iterators_done) {
all_iterators_done = true;
for (uint32_t i = 0; i < self->iterators.size; i++) {
for (t_u32 i = 0; i < self->iterators.size; i++) {
StackIterator iterator = self->iterators.contents[i];
StackNode *node = iterator.node;
for (uint32_t j = 0; j < visited_nodes.size; j++) {
for (t_u32 j = 0; j < visited_nodes.size; j++) {
if (visited_nodes.contents[j] == node) {
node = NULL;
break;

14
parser/src/stack.h
View file

@ -24,7 +24,7 @@ typedef Array(StackSlice) StackSliceArray;
typedef struct {
Length position;
unsigned depth;
TSStateId state;
t_state_id state;
} StackSummaryEntry;
typedef Array(StackSummaryEntry) StackSummary;
@ -35,11 +35,11 @@ Stack *ts_stack_new(SubtreePool *);
void ts_stack_delete(Stack *);
// Get the stack's current number of versions.
uint32_t ts_stack_version_count(const Stack *);
t_u32 ts_stack_version_count(const Stack *);
// Get the state at the top of the given version of the stack. If the stack is
// empty, this returns the initial state, 0.
TSStateId ts_stack_state(const Stack *, StackVersion);
t_state_id ts_stack_state(const Stack *, StackVersion);
// Get the last external token associated with a given version of the stack.
Subtree ts_stack_last_external_token(const Stack *, StackVersion);
@ -55,14 +55,14 @@ Length ts_stack_position(const Stack *, StackVersion);
// This transfers ownership of the tree to the Stack. Callers that
// need to retain ownership of the tree for their own purposes should
// first retain the tree.
void ts_stack_push(Stack *, StackVersion, Subtree , bool, TSStateId);
void ts_stack_push(Stack *, StackVersion, Subtree , bool, t_state_id);
// Pop the given number of entries from the given version of the stack. This
// operation can increase the number of stack versions by revealing multiple
// versions which had previously been merged. It returns an array that
// specifies the index of each revealed version and the trees that were
// removed from that version.
StackSliceArray ts_stack_pop_count(Stack *, StackVersion, uint32_t count);
StackSliceArray ts_stack_pop_count(Stack *, StackVersion, t_u32 count);
// Remove an error at the top of the given version of the stack.
SubtreeArray ts_stack_pop_error(Stack *, StackVersion);
@ -122,9 +122,9 @@ void ts_stack_remove_version(Stack *, StackVersion);
void ts_stack_clear(Stack *);
bool ts_stack_print_dot_graph(Stack *, const TSLanguage *, FILE *);
bool ts_stack_print_dot_graph(Stack *, const t_language *, FILE *);
typedef void (*StackIterateCallback)(void *, TSStateId, uint32_t);
typedef void (*StackIterateCallback)(void *, t_state_id, t_u32);
#ifdef __cplusplus
}

176
parser/src/subtree.c
View file

@ -90,7 +90,7 @@ void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest)
{
dest->contents = calloc(self.capacity, sizeof(Subtree));
memcpy(dest->contents, self.contents, self.size * sizeof(Subtree));
for (uint32_t i = 0; i < self.size; i++)
for (t_u32 i = 0; i < self.size; i++)
{
ts_subtree_retain(dest->contents[i]);
}
@ -99,7 +99,7 @@ void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest)
void ts_subtree_array_clear(SubtreePool *pool, SubtreeArray *self)
{
for (uint32_t i = 0; i < self->size; i++)
for (t_u32 i = 0; i < self->size; i++)
{
ts_subtree_release(pool, self->contents[i]);
}
@ -134,7 +134,7 @@ void ts_subtree_array_remove_trailing_extras(SubtreeArray *self,
void ts_subtree_array_reverse(SubtreeArray *self)
{
for (uint32_t i = 0, limit = self->size / 2; i < limit; i++)
for (t_u32 i = 0, limit = self->size / 2; i < limit; i++)
{
size_t reverse_index = self->size - 1 - i;
Subtree swap = self->contents[i];
@ -145,7 +145,7 @@ void ts_subtree_array_reverse(SubtreeArray *self)
// SubtreePool
SubtreePool ts_subtree_pool_new(uint32_t capacity)
SubtreePool ts_subtree_pool_new(t_u32 capacity)
{
SubtreePool self = {array_new(), array_new()};
array_reserve(&self.free_trees, capacity);
@ -194,7 +194,7 @@ static void ts_subtree_pool_free(SubtreePool *self, SubtreeHeapData *tree)
// Subtree
static inline bool ts_subtree_can_inline(Length padding, Length size,
uint32_t lookahead_bytes)
t_u32 lookahead_bytes)
{
return padding.bytes < TS_MAX_INLINE_TREE_LENGTH &&
padding.extent.row < 16 &&
@ -204,13 +204,13 @@ static inline bool ts_subtree_can_inline(Length padding, Length size,
lookahead_bytes < 16;
}
Subtree ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding,
Length size, uint32_t lookahead_bytes,
TSStateId parse_state, bool has_external_tokens,
Subtree ts_subtree_new_leaf(SubtreePool *pool, t_symbol symbol, Length padding,
Length size, t_u32 lookahead_bytes,
t_state_id parse_state, bool has_external_tokens,
bool depends_on_column, bool is_keyword,
const TSLanguage *language)
const t_language *language)
{
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol);
t_symbol_metadata metadata = ts_language_symbol_metadata(language, symbol);
bool extra = symbol == ts_builtin_sym_end;
bool is_inline = (symbol <= UINT8_MAX && !has_external_tokens &&
@ -263,10 +263,10 @@ Subtree ts_subtree_new_leaf(SubtreePool *pool, TSSymbol symbol, Length padding,
}
}
void ts_subtree_set_symbol(MutableSubtree *self, TSSymbol symbol,
const TSLanguage *language)
void ts_subtree_set_symbol(MutableSubtree *self, t_symbol symbol,
const t_language *language)
{
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol);
t_symbol_metadata metadata = ts_language_symbol_metadata(language, symbol);
if (self->data.is_inline)
{
assert(symbol < UINT8_MAX);
@ -282,10 +282,10 @@ void ts_subtree_set_symbol(MutableSubtree *self, TSSymbol symbol,
}
}
Subtree ts_subtree_new_error(SubtreePool *pool, int32_t lookahead_char,
Subtree ts_subtree_new_error(SubtreePool *pool, t_i32 lookahead_char,
Length padding, Length size,
uint32_t bytes_scanned, TSStateId parse_state,
const TSLanguage *language)
t_u32 bytes_scanned, t_state_id parse_state,
const t_language *language)
{
Subtree result = ts_subtree_new_leaf(pool, ts_builtin_sym_error, padding,
size, bytes_scanned, parse_state,
@ -308,7 +308,7 @@ MutableSubtree ts_subtree_clone(Subtree self)
(SubtreeHeapData *)&new_children[self.ptr->child_count];
if (self.ptr->child_count > 0)
{
for (uint32_t i = 0; i < self.ptr->child_count; i++)
for (t_u32 i = 0; i < self.ptr->child_count; i++)
{
ts_subtree_retain(new_children[i]);
}
@ -332,33 +332,33 @@ MutableSubtree ts_subtree_make_mut(SubtreePool *pool, Subtree self)
if (self.data.is_inline)
return (MutableSubtree){self.data};
if (self.ptr->ref_count == 1)
return ts_subtree_to_mut_unsafe(self);
return ts_subtree_to_mt_unsafe(self);
MutableSubtree result = ts_subtree_clone(self);
ts_subtree_release(pool, self);
return result;
}
static void ts_subtree__compress(MutableSubtree self, unsigned count,
const TSLanguage *language,
const t_language *language,
MutableSubtreeArray *stack)
{
unsigned initial_stack_size = stack->size;
MutableSubtree tree = self;
TSSymbol symbol = tree.ptr->symbol;
t_symbol symbol = tree.ptr->symbol;
for (unsigned i = 0; i < count; i++)
{
if (tree.ptr->ref_count > 1 || tree.ptr->child_count < 2)
break;
MutableSubtree child =
ts_subtree_to_mut_unsafe(ts_subtree_children(tree)[0]);
ts_subtree_to_mt_unsafe(ts_subtree_children(tree)[0]);
if (child.data.is_inline || child.ptr->child_count < 2 ||
child.ptr->ref_count > 1 || child.ptr->symbol != symbol)
break;
MutableSubtree grandchild =
ts_subtree_to_mut_unsafe(ts_subtree_children(child)[0]);
ts_subtree_to_mt_unsafe(ts_subtree_children(child)[0]);
if (grandchild.data.is_inline || grandchild.ptr->child_count < 2 ||
grandchild.ptr->ref_count > 1 || grandchild.ptr->symbol != symbol)
break;
@ -376,8 +376,8 @@ static void ts_subtree__compress(MutableSubtree self, unsigned count,
{
tree = array_pop(stack);
MutableSubtree child =
ts_subtree_to_mut_unsafe(ts_subtree_children(tree)[0]);
MutableSubtree grandchild = ts_subtree_to_mut_unsafe(
ts_subtree_to_mt_unsafe(ts_subtree_children(tree)[0]);
MutableSubtree grandchild = ts_subtree_to_mt_unsafe(
ts_subtree_children(child)[child.ptr->child_count - 1]);
ts_subtree_summarize_children(grandchild, language);
ts_subtree_summarize_children(child, language);
@ -386,13 +386,13 @@ static void ts_subtree__compress(MutableSubtree self, unsigned count,
}
void ts_subtree_balance(Subtree self, SubtreePool *pool,
const TSLanguage *language)
const t_language *language)
{
array_clear(&pool->tree_stack);
if (ts_subtree_child_count(self) > 0 && self.ptr->ref_count == 1)
{
array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(self));
array_push(&pool->tree_stack, ts_subtree_to_mt_unsafe(self));
}
while (pool->tree_stack.size > 0)
@ -417,12 +417,12 @@ void ts_subtree_balance(Subtree self, SubtreePool *pool,
}
}
for (uint32_t i = 0; i < tree.ptr->child_count; i++)
for (t_u32 i = 0; i < tree.ptr->child_count; i++)
{
Subtree child = ts_subtree_children(tree)[i];
if (ts_subtree_child_count(child) > 0 && child.ptr->ref_count == 1)
{
array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(child));
array_push(&pool->tree_stack, ts_subtree_to_mt_unsafe(child));
}
}
}
@ -430,7 +430,7 @@ void ts_subtree_balance(Subtree self, SubtreePool *pool,
// Assign all of the node's properties that depend on its children.
void ts_subtree_summarize_children(MutableSubtree self,
const TSLanguage *language)
const t_language *language)
{
assert(!self.data.is_inline);
@ -444,13 +444,13 @@ void ts_subtree_summarize_children(MutableSubtree self,
self.ptr->has_external_scanner_state_change = false;
self.ptr->dynamic_precedence = 0;
uint32_t structural_index = 0;
const TSSymbol *alias_sequence =
t_u32 structural_index = 0;
const t_symbol *alias_sequence =
ts_language_alias_sequence(language, self.ptr->production_id);
uint32_t lookahead_end_byte = 0;
t_u32 lookahead_end_byte = 0;
const Subtree *children = ts_subtree_children(self);
for (uint32_t i = 0; i < self.ptr->child_count; i++)
for (t_u32 i = 0; i < self.ptr->child_count; i++)
{
Subtree child = children[i];
@ -476,7 +476,7 @@ void ts_subtree_summarize_children(MutableSubtree self,
length_add(self.ptr->size, ts_subtree_total_size(child));
}
uint32_t child_lookahead_end_byte = self.ptr->padding.bytes +
t_u32 child_lookahead_end_byte = self.ptr->padding.bytes +
self.ptr->size.bytes +
ts_subtree_lookahead_bytes(child);
if (child_lookahead_end_byte > lookahead_end_byte)
@ -489,7 +489,7 @@ void ts_subtree_summarize_children(MutableSubtree self,
self.ptr->error_cost += ts_subtree_error_cost(child);
}
uint32_t grandchild_count = ts_subtree_child_count(child);
t_u32 grandchild_count = ts_subtree_child_count(child);
if (self.ptr->symbol == ts_builtin_sym_error ||
self.ptr->symbol == ts_builtin_sym_error_repeat)
{
@ -598,11 +598,11 @@ void ts_subtree_summarize_children(MutableSubtree self,
// Create a new parent node with the given children.
//
// This takes ownership of the children array.
MutableSubtree ts_subtree_new_node(TSSymbol symbol, SubtreeArray *children,
MutableSubtree ts_subtree_new_node(t_symbol symbol, SubtreeArray *children,
unsigned production_id,
const TSLanguage *language)
const t_language *language)
{
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol);
t_symbol_metadata metadata = ts_language_symbol_metadata(language, symbol);
bool fragile =
symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat;
@ -611,7 +611,7 @@ MutableSubtree ts_subtree_new_node(TSSymbol symbol, SubtreeArray *children,
if (children->capacity * sizeof(Subtree) < new_byte_size)
{
children->contents = realloc(children->contents, new_byte_size);
children->capacity = (uint32_t)(new_byte_size / sizeof(Subtree));
children->capacity = (t_u32)(new_byte_size / sizeof(Subtree));
}
SubtreeHeapData *data =
(SubtreeHeapData *)&children->contents[children->size];
@ -641,7 +641,7 @@ MutableSubtree ts_subtree_new_node(TSSymbol symbol, SubtreeArray *children,
// This node is treated as 'extra'. Its children are prevented from having
// having any effect on the parse state.
Subtree ts_subtree_new_error_node(SubtreeArray *children, bool extra,
const TSLanguage *language)
const t_language *language)
{
MutableSubtree result =
ts_subtree_new_node(ts_builtin_sym_error, children, 0, language);
@ -653,9 +653,9 @@ Subtree ts_subtree_new_error_node(SubtreeArray *children, bool extra,
//
// This node is treated as 'extra'. Its children are prevented from having
// having any effect on the parse state.
Subtree ts_subtree_new_missing_leaf(SubtreePool *pool, TSSymbol symbol,
Length padding, uint32_t lookahead_bytes,
const TSLanguage *language)
Subtree ts_subtree_new_missing_leaf(SubtreePool *pool, t_symbol symbol,
Length padding, t_u32 lookahead_bytes,
const t_language *language)
{
Subtree result =
ts_subtree_new_leaf(pool, symbol, padding, length_zero(),
@ -676,7 +676,7 @@ void ts_subtree_retain(Subtree self)
if (self.data.is_inline)
return;
assert(self.ptr->ref_count > 0);
*(uint32_t *)&self.ptr->ref_count += 1;
*(t_u32 *)&self.ptr->ref_count += 1;
assert(self.ptr->ref_count != 0);
}
@ -687,9 +687,9 @@ void ts_subtree_release(SubtreePool *pool, Subtree self)
array_clear(&pool->tree_stack);
assert(self.ptr->ref_count > 0);
if (--(*(uint32_t *)&self.ptr->ref_count) == 0)
if (--(*(t_u32 *)&self.ptr->ref_count) == 0)
{
array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(self));
array_push(&pool->tree_stack, ts_subtree_to_mt_unsafe(self));
}
while (pool->tree_stack.size > 0)
@ -698,16 +698,16 @@ void ts_subtree_release(SubtreePool *pool, Subtree self)
if (tree.ptr->child_count > 0)
{
Subtree *children = ts_subtree_children(tree);
for (uint32_t i = 0; i < tree.ptr->child_count; i++)
for (t_u32 i = 0; i < tree.ptr->child_count; i++)
{
Subtree child = children[i];
if (child.data.is_inline)
continue;
assert(child.ptr->ref_count > 0);
if (--*(uint32_t *)&child.ptr->ref_count == 0)
if (--*(t_u32 *)&child.ptr->ref_count == 0)
{
array_push(&pool->tree_stack,
ts_subtree_to_mut_unsafe(child));
ts_subtree_to_mt_unsafe(child));
}
}
free(children);
@ -726,8 +726,8 @@ void ts_subtree_release(SubtreePool *pool, Subtree self)
int ts_subtree_compare(Subtree left, Subtree right, SubtreePool *pool)
{
array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(left));
array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(right));
array_push(&pool->tree_stack, ts_subtree_to_mt_unsafe(left));
array_push(&pool->tree_stack, ts_subtree_to_mt_unsafe(right));
while (pool->tree_stack.size > 0)
{
@ -749,13 +749,13 @@ int ts_subtree_compare(Subtree left, Subtree right, SubtreePool *pool)
return result;
}
for (uint32_t i = ts_subtree_child_count(left); i > 0; i--)
for (t_u32 i = ts_subtree_child_count(left); i > 0; i--)
{
Subtree left_child = ts_subtree_children(left)[i - 1];
Subtree right_child = ts_subtree_children(right)[i - 1];
array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(left_child));
array_push(&pool->tree_stack, ts_subtree_to_mt_unsafe(left_child));
array_push(&pool->tree_stack,
ts_subtree_to_mut_unsafe(right_child));
ts_subtree_to_mt_unsafe(right_child));
}
}
@ -774,7 +774,7 @@ static inline void ts_subtree_set_has_changes(MutableSubtree *self)
}
}
Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit,
Subtree ts_subtree_edit(Subtree self, const t_input_edit *inpt_edit,
SubtreePool *pool)
{
typedef struct
@ -790,11 +790,11 @@ Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit,
.tree = &self,
.edit =
(Edit){
.start = {input_edit->start_byte, input_edit->start_point},
.old_end = {input_edit->old_end_byte,
input_edit->old_end_point},
.new_end = {input_edit->new_end_byte,
input_edit->new_end_point},
.start = {inpt_edit->start_byte, inpt_edit->start_point},
.old_end = {inpt_edit->old_end_byte,
inpt_edit->old_end_point},
.new_end = {inpt_edit->new_end_byte,
inpt_edit->new_end_point},
},
}));
@ -810,8 +810,8 @@ Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit,
Length size = ts_subtree_size(*entry.tree);
Length padding = ts_subtree_padding(*entry.tree);
Length total_size = length_add(padding, size);
uint32_t lookahead_bytes = ts_subtree_lookahead_bytes(*entry.tree);
uint32_t end_byte = total_size.bytes + lookahead_bytes;
t_u32 lookahead_bytes = ts_subtree_lookahead_bytes(*entry.tree);
t_u32 end_byte = total_size.bytes + lookahead_bytes;
if (edit.start.bytes > end_byte ||
(is_noop && edit.start.bytes == end_byte))
continue;
@ -896,7 +896,7 @@ Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit,
*entry.tree = ts_subtree_from_mut(result);
Length child_left, child_right = length_zero();
for (uint32_t i = 0, n = ts_subtree_child_count(*entry.tree); i < n;
for (t_u32 i = 0, n = ts_subtree_child_count(*entry.tree); i < n;
i++)
{
Subtree *child = &ts_subtree_children(*entry.tree)[i];
@ -964,7 +964,7 @@ Subtree ts_subtree_last_external_token(Subtree tree)
return NULL_SUBTREE;
while (tree.ptr->child_count > 0)
{
for (uint32_t i = tree.ptr->child_count - 1; i + 1 > 0; i--)
for (t_u32 i = tree.ptr->child_count - 1; i + 1 > 0; i--)
{
Subtree child = ts_subtree_children(tree)[i];
if (ts_subtree_has_external_tokens(child))
@ -977,7 +977,7 @@ Subtree ts_subtree_last_external_token(Subtree tree)
return tree;
}
static size_t ts_subtree__write_char_to_string(char *str, size_t n, int32_t chr)
static size_t ts_subtree__write_char_to_string(char *str, size_t n, t_i32 chr)
{
if (chr == -1)
return snprintf(str, n, "INVALID");
@ -998,8 +998,8 @@ static size_t ts_subtree__write_char_to_string(char *str, size_t n, int32_t chr)
static const char *const ROOT_FIELD = "__ROOT__";
static size_t ts_subtree__write_to_string(
Subtree self, char *string, size_t limit, const TSLanguage *language,
bool include_all, TSSymbol alias_symbol, bool alias_is_named,
Subtree self, char *string, size_t limit, const t_language *language,
bool include_all, t_symbol alias_symbol, bool alias_is_named,
const char *field_name)
{
if (!self.ptr)
@ -1033,7 +1033,7 @@ static size_t ts_subtree__write_to_string(
}
else
{
TSSymbol symbol =
t_symbol symbol =
alias_symbol ? alias_symbol : ts_subtree_symbol(self);
const char *symbol_name = ts_language_symbol_name(language, symbol);
if (ts_subtree_missing(self))
@ -1056,7 +1056,7 @@ static size_t ts_subtree__write_to_string(
}
else if (is_root)
{
TSSymbol symbol = alias_symbol ? alias_symbol : ts_subtree_symbol(self);
t_symbol symbol = alias_symbol ? alias_symbol : ts_subtree_symbol(self);
const char *symbol_name = ts_language_symbol_name(language, symbol);
if (ts_subtree_child_count(self) > 0)
{
@ -1074,14 +1074,14 @@ static size_t ts_subtree__write_to_string(
if (ts_subtree_child_count(self))
{
const TSSymbol *alias_sequence =
const t_symbol *alias_sequence =
ts_language_alias_sequence(language, self.ptr->production_id);
const TSFieldMapEntry *field_map, *field_map_end;
const t_field_map_entry *field_map, *field_map_end;
ts_language_field_map(language, self.ptr->production_id, &field_map,
&field_map_end);
uint32_t structural_child_index = 0;
for (uint32_t i = 0; i < self.ptr->child_count; i++)
t_u32 structural_child_index = 0;
for (t_u32 i = 0; i < self.ptr->child_count; i++)
{
Subtree child = ts_subtree_children(self)[i];
if (ts_subtree_extra(child))
@ -1092,7 +1092,7 @@ static size_t ts_subtree__write_to_string(
}
else
{
TSSymbol subtree_alias_symbol =
t_symbol subtree_alias_symbol =
alias_sequence ? alias_sequence[structural_child_index] : 0;
bool subtree_alias_is_named =
subtree_alias_symbol ? ts_language_symbol_metadata(
@ -1101,7 +1101,7 @@ static size_t ts_subtree__write_to_string(
: false;
const char *child_field_name = is_visible ? NULL : field_name;
for (const TSFieldMapEntry *map = field_map;
for (const t_field_map_entry *map = field_map;
map < field_map_end; map++)
{
if (!map->inherited &&
@ -1127,8 +1127,8 @@ static size_t ts_subtree__write_to_string(
return cursor - string;
}
char *ts_subtree_string(Subtree self, TSSymbol alias_symbol,
bool alias_is_named, const TSLanguage *language,
char *ts_subtree_string(Subtree self, t_symbol alias_symbol,
bool alias_is_named, const t_language *language,
bool include_all)
{
char scratch_string[1];
@ -1142,13 +1142,13 @@ char *ts_subtree_string(Subtree self, TSSymbol alias_symbol,
return result;
}
void ts_subtree__print_dot_graph(const Subtree *self, uint32_t start_offset,
const TSLanguage *language,
TSSymbol alias_symbol, FILE *f)
void ts_subtree__print_dot_graph(const Subtree *self, t_u32 start_offset,
const t_language *language,
t_symbol alias_symbol, FILE *f)
{
TSSymbol subtree_symbol = ts_subtree_symbol(*self);
TSSymbol symbol = alias_symbol ? alias_symbol : subtree_symbol;
uint32_t end_offset = start_offset + ts_subtree_total_bytes(*self);
t_symbol subtree_symbol = ts_subtree_symbol(*self);
t_symbol symbol = alias_symbol ? alias_symbol : subtree_symbol;
t_u32 end_offset = start_offset + ts_subtree_total_bytes(*self);
fprintf(f, "tree_%p [label=\"", (void *)self);
ts_language_write_symbol_as_dot_string(language, f, symbol);
fprintf(f, "\"");
@ -1182,13 +1182,13 @@ void ts_subtree__print_dot_graph(const Subtree *self, uint32_t start_offset,
fprintf(f, "\"]\n");
uint32_t child_start_offset = start_offset;
uint32_t child_info_offset =
t_u32 child_start_offset = start_offset;
t_u32 child_info_offset =
language->max_alias_sequence_length * ts_subtree_production_id(*self);
for (uint32_t i = 0, n = ts_subtree_child_count(*self); i < n; i++)
for (t_u32 i = 0, n = ts_subtree_child_count(*self); i < n; i++)
{
const Subtree *child = &ts_subtree_children(*self)[i];
TSSymbol subtree_alias_symbol = 0;
t_symbol subtree_alias_symbol = 0;
if (!ts_subtree_extra(*child) && child_info_offset)
{
subtree_alias_symbol = language->alias_sequences[child_info_offset];
@ -1202,7 +1202,7 @@ void ts_subtree__print_dot_graph(const Subtree *self, uint32_t start_offset,
}
}
void ts_subtree_print_dot_graph(Subtree self, const TSLanguage *language,
void ts_subtree_print_dot_graph(Subtree self, const t_language *language,
FILE *f)
{
fprintf(f, "digraph tree {\n");

545
parser/src/subtree.h
View file

@ -1,22 +1,21 @@
#ifndef TREE_SITTER_SUBTREE_H_
#define TREE_SITTER_SUBTREE_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "me/types.h"
#include "parser/types/types_symbol.h"
#include <limits.h>
#include <stdbool.h>
#include <stdio.h>
#include "./length.h"
#include "./array.h"
#include "./error_costs.h"
#include "./host.h"
#include "parser/api.h"
#include "./length.h"
#include "./parser.h"
#include "parser/api.h"
#include <limits.h>
#include <stdbool.h>
#include <stdio.h>
#define TS_TREE_STATE_NONE USHRT_MAX
#define NULL_SUBTREE ((Subtree) {.ptr = NULL})
#define NULL_SUBTREE ((Subtree){.ptr = NULL})
// The serialized state of an external scanner.
//
@ -28,12 +27,13 @@ extern "C" {
//
// Small byte arrays are stored inline, and long ones are allocated
// separately on the heap.
typedef struct {
union {
char *long_data;
char short_data[24];
};
uint32_t length;
typedef struct
{
union {
char *long_data;
char short_data[24];
};
t_u32 length;
} ExternalScannerState;
// A compact representation of a subtree.
@ -47,136 +47,108 @@ typedef struct {
// Because of alignment, for any valid pointer this will be 0, giving
// us the opportunity to make use of this bit to signify whether to use
// the pointer or the inline struct.
typedef struct SubtreeInlineData SubtreeInlineData;
typedef struct s_subtree_inline_data t_subtree_inline_data;
#define SUBTREE_BITS \
bool visible : 1; \
bool named : 1; \
bool extra : 1; \
bool has_changes : 1; \
bool is_missing : 1; \
bool is_keyword : 1;
#define SUBTREE_SIZE \
uint8_t padding_columns; \
uint8_t padding_rows : 4; \
uint8_t lookahead_bytes : 4; \
uint8_t padding_bytes; \
uint8_t size_bytes;
#if TS_BIG_ENDIAN
#if TS_PTR_SIZE == 32
struct SubtreeInlineData {
uint16_t parse_state;
uint8_t symbol;
SUBTREE_BITS
bool unused : 1;
bool is_inline : 1;
SUBTREE_SIZE
struct s_subtree_inline_data
{
bool is_inline : 1;
bool visible : 1;
bool named : 1;
bool extra : 1;
bool has_changes : 1;
bool is_missing : 1;
bool is_keyword : 1;
t_u8 symbol;
t_u16 parse_state;
t_u8 padding_columns;
t_u8 padding_rows : 4;
t_u8 lookahead_bytes : 4;
t_u8 padding_bytes;
t_u8 size_bytes;
};
#else
struct SubtreeInlineData {
SUBTREE_SIZE
uint16_t parse_state;
uint8_t symbol;
SUBTREE_BITS
bool unused : 1;
bool is_inline : 1;
};
#endif
#else
struct SubtreeInlineData {
bool is_inline : 1;
SUBTREE_BITS
uint8_t symbol;
uint16_t parse_state;
SUBTREE_SIZE
};
#endif
#undef SUBTREE_BITS
#undef SUBTREE_SIZE
// A heap-allocated representation of a subtree.
//
// This representation is used for parent nodes, external tokens,
// errors, and other leaf nodes whose data is too large to fit into
// the inline representation.
typedef struct {
volatile uint32_t ref_count;
Length padding;
Length size;
uint32_t lookahead_bytes;
uint32_t error_cost;
uint32_t child_count;
TSSymbol symbol;
TSStateId parse_state;
typedef struct
{
volatile t_u32 ref_count;
Length padding;
Length size;
t_u32 lookahead_bytes;
t_u32 error_cost;
t_u32 child_count;
t_symbol symbol;
t_state_id parse_state;
bool visible : 1;
bool named : 1;
bool extra : 1;
bool fragile_left : 1;
bool fragile_right : 1;
bool has_changes : 1;
bool has_external_tokens : 1;
bool has_external_scanner_state_change : 1;
bool depends_on_column: 1;
bool is_missing : 1;
bool is_keyword : 1;
bool visible : 1;
bool named : 1;
bool extra : 1;
bool fragile_left : 1;
bool fragile_right : 1;
bool has_changes : 1;
bool has_external_tokens : 1;
bool has_external_scanner_state_change : 1;
bool depends_on_column : 1;
bool is_missing : 1;
bool is_keyword : 1;
union {
// Non-terminal subtrees (`child_count > 0`)
struct {
uint32_t visible_child_count;
uint32_t named_child_count;
uint32_t visible_descendant_count;
int32_t dynamic_precedence;
uint16_t repeat_depth;
uint16_t production_id;
struct {
TSSymbol symbol;
TSStateId parse_state;
} first_leaf;
};
union {
// Non-terminal subtrees (`child_count > 0`)
struct
{
t_u32 visible_child_count;
t_u32 named_child_count;
t_u32 visible_descendant_count;
t_i32 dynamic_precedence;
t_u16 repeat_depth;
t_u16 production_id;
struct
{
t_symbol symbol;
t_state_id parse_state;
} first_leaf;
};
// External terminal subtrees (`child_count == 0 && has_external_tokens`)
ExternalScannerState external_scanner_state;
// External terminal subtrees (`child_count == 0 &&
// has_external_tokens`)
ExternalScannerState external_scanner_state;
// Error terminal subtrees (`child_count == 0 && symbol == ts_builtin_sym_error`)
int32_t lookahead_char;
};
// Error terminal subtrees (`child_count == 0 && symbol ==
// ts_builtin_sym_error`)
t_i32 lookahead_char;
};
} SubtreeHeapData;
// The fundamental building block of a syntax tree.
typedef union {
SubtreeInlineData data;
const SubtreeHeapData *ptr;
t_subtree_inline_data data;
const SubtreeHeapData *ptr;
} Subtree;
// Like Subtree, but mutable.
typedef union {
SubtreeInlineData data;
SubtreeHeapData *ptr;
t_subtree_inline_data data;
SubtreeHeapData *ptr;
} MutableSubtree;
typedef Array(Subtree) SubtreeArray;
typedef Array(MutableSubtree) MutableSubtreeArray;
typedef struct {
MutableSubtreeArray free_trees;
MutableSubtreeArray tree_stack;
typedef struct
{
MutableSubtreeArray free_trees;
MutableSubtreeArray tree_stack;
} SubtreePool;
void ts_external_scanner_state_init(ExternalScannerState *, const char *, unsigned);
void ts_external_scanner_state_init(ExternalScannerState *, const char *,
unsigned);
const char *ts_external_scanner_state_data(const ExternalScannerState *);
bool ts_external_scanner_state_eq(const ExternalScannerState *self, const char *, unsigned);
void ts_external_scanner_state_delete(ExternalScannerState *self);
bool ts_external_scanner_state_eq(const ExternalScannerState *self,
const char *, unsigned);
void ts_external_scanner_state_delete(ExternalScannerState *self);
void ts_subtree_array_copy(SubtreeArray, SubtreeArray *);
void ts_subtree_array_clear(SubtreePool *, SubtreeArray *);
@ -184,199 +156,280 @@ void ts_subtree_array_delete(SubtreePool *, SubtreeArray *);
void ts_subtree_array_remove_trailing_extras(SubtreeArray *, SubtreeArray *);
void ts_subtree_array_reverse(SubtreeArray *);
SubtreePool ts_subtree_pool_new(uint32_t capacity);
void ts_subtree_pool_delete(SubtreePool *);
SubtreePool ts_subtree_pool_new(t_u32 capacity);
void ts_subtree_pool_delete(SubtreePool *);
Subtree ts_subtree_new_leaf(
SubtreePool *, TSSymbol, Length, Length, uint32_t,
TSStateId, bool, bool, bool, const TSLanguage *
);
Subtree ts_subtree_new_error(
SubtreePool *, int32_t, Length, Length, uint32_t, TSStateId, const TSLanguage *
);
MutableSubtree ts_subtree_new_node(TSSymbol, SubtreeArray *, unsigned, const TSLanguage *);
Subtree ts_subtree_new_error_node(SubtreeArray *, bool, const TSLanguage *);
Subtree ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, Length, uint32_t, const TSLanguage *);
Subtree ts_subtree_new_leaf(SubtreePool *, t_symbol, Length, Length, t_u32,
t_state_id, bool, bool, bool, const t_language *);
Subtree ts_subtree_new_error(SubtreePool *, t_i32, Length, Length, t_u32,
t_state_id, const t_language *);
MutableSubtree ts_subtree_new_node(t_symbol, SubtreeArray *, unsigned,
const t_language *);
Subtree ts_subtree_new_error_node(SubtreeArray *, bool, const t_language *);
Subtree ts_subtree_new_missing_leaf(SubtreePool *, t_symbol, Length, t_u32,
const t_language *);
MutableSubtree ts_subtree_make_mut(SubtreePool *, Subtree);
void ts_subtree_retain(Subtree);
void ts_subtree_release(SubtreePool *, Subtree);
int ts_subtree_compare(Subtree, Subtree, SubtreePool *);
void ts_subtree_set_symbol(MutableSubtree *, TSSymbol, const TSLanguage *);
void ts_subtree_summarize(MutableSubtree, const Subtree *, uint32_t, const TSLanguage *);
void ts_subtree_summarize_children(MutableSubtree, const TSLanguage *);
void ts_subtree_balance(Subtree, SubtreePool *, const TSLanguage *);
Subtree ts_subtree_edit(Subtree, const TSInputEdit *edit, SubtreePool *);
char *ts_subtree_string(Subtree, TSSymbol, bool, const TSLanguage *, bool include_all);
void ts_subtree_print_dot_graph(Subtree, const TSLanguage *, FILE *);
void ts_subtree_retain(Subtree);
void ts_subtree_release(SubtreePool *, Subtree);
int ts_subtree_compare(Subtree, Subtree, SubtreePool *);
void ts_subtree_set_symbol(MutableSubtree *, t_symbol, const t_language *);
void ts_subtree_summarize(MutableSubtree, const Subtree *, t_u32,
const t_language *);
void ts_subtree_summarize_children(MutableSubtree, const t_language *);
void ts_subtree_balance(Subtree, SubtreePool *, const t_language *);
Subtree ts_subtree_edit(Subtree, const t_input_edit *edit, SubtreePool *);
char *ts_subtree_string(Subtree, t_symbol, bool, const t_language *,
bool include_all);
void ts_subtree_print_dot_graph(Subtree, const t_language *, FILE *);
Subtree ts_subtree_last_external_token(Subtree);
const ExternalScannerState *ts_subtree_external_scanner_state(Subtree self);
bool ts_subtree_external_scanner_state_eq(Subtree, Subtree);
#define SUBTREE_GET(self, name) ((self).data.is_inline ? (self).data.name : (self).ptr->name)
#define SUBTREE_GET(self, name) \
((self).data.is_inline ? (self).data.name : (self).ptr->name)
static inline TSSymbol ts_subtree_symbol(Subtree self) { return SUBTREE_GET(self, symbol); }
static inline bool ts_subtree_visible(Subtree self) { return SUBTREE_GET(self, visible); }
static inline bool ts_subtree_named(Subtree self) { return SUBTREE_GET(self, named); }
static inline bool ts_subtree_extra(Subtree self) { return SUBTREE_GET(self, extra); }
static inline bool ts_subtree_has_changes(Subtree self) { return SUBTREE_GET(self, has_changes); }
static inline bool ts_subtree_missing(Subtree self) { return SUBTREE_GET(self, is_missing); }
static inline bool ts_subtree_is_keyword(Subtree self) { return SUBTREE_GET(self, is_keyword); }
static inline TSStateId ts_subtree_parse_state(Subtree self) { return SUBTREE_GET(self, parse_state); }
static inline uint32_t ts_subtree_lookahead_bytes(Subtree self) { return SUBTREE_GET(self, lookahead_bytes); }
static inline t_symbol ts_subtree_symbol(Subtree self)
{
return SUBTREE_GET(self, symbol);
}
static inline bool ts_subtree_visible(Subtree self)
{
return SUBTREE_GET(self, visible);
}
static inline bool ts_subtree_named(Subtree self)
{
return SUBTREE_GET(self, named);
}
static inline bool ts_subtree_extra(Subtree self)
{
return SUBTREE_GET(self, extra);
}
static inline bool ts_subtree_has_changes(Subtree self)
{
return SUBTREE_GET(self, has_changes);
}
static inline bool ts_subtree_missing(Subtree self)
{
return SUBTREE_GET(self, is_missing);
}
static inline bool ts_subtree_is_keyword(Subtree self)
{
return SUBTREE_GET(self, is_keyword);
}
static inline t_state_id ts_subtree_parse_state(Subtree self)
{
return SUBTREE_GET(self, parse_state);
}
static inline t_u32 ts_subtree_lookahead_bytes(Subtree self)
{
return SUBTREE_GET(self, lookahead_bytes);
}
#undef SUBTREE_GET
// Get the size needed to store a heap-allocated subtree with the given
// number of children.
static inline size_t ts_subtree_alloc_size(uint32_t child_count) {
return child_count * sizeof(Subtree) + sizeof(SubtreeHeapData);
static inline size_t ts_subtree_alloc_size(t_u32 child_count)
{
return child_count * sizeof(Subtree) + sizeof(SubtreeHeapData);
}
// Get a subtree's children, which are allocated immediately before the
// tree's own heap data.
#define ts_subtree_children(self) \
((self).data.is_inline ? NULL : (Subtree *)((self).ptr) - (self).ptr->child_count)
#define ts_subtree_children(self) \
((self).data.is_inline \
? NULL \
: (Subtree *)((self).ptr) - (self).ptr->child_count)
static inline void ts_subtree_set_extra(MutableSubtree *self, bool is_extra) {
if (self->data.is_inline) {
self->data.extra = is_extra;
} else {
self->ptr->extra = is_extra;
}
static inline void ts_subtree_set_extra(MutableSubtree *self, bool is_extra)
{
if (self->data.is_inline)
{
self->data.extra = is_extra;
}
else
{
self->ptr->extra = is_extra;
}
}
static inline TSSymbol ts_subtree_leaf_symbol(Subtree self) {
if (self.data.is_inline) return self.data.symbol;
if (self.ptr->child_count == 0) return self.ptr->symbol;
return self.ptr->first_leaf.symbol;
static inline t_symbol ts_subtree_leaf_symbol(Subtree self)
{
if (self.data.is_inline)
return self.data.symbol;
if (self.ptr->child_count == 0)
return self.ptr->symbol;
return self.ptr->first_leaf.symbol;
}
static inline TSStateId ts_subtree_leaf_parse_state(Subtree self) {
if (self.data.is_inline) return self.data.parse_state;
if (self.ptr->child_count == 0) return self.ptr->parse_state;
return self.ptr->first_leaf.parse_state;
static inline t_state_id ts_subtree_leaf_parse_state(Subtree self)
{
if (self.data.is_inline)
return self.data.parse_state;
if (self.ptr->child_count == 0)
return self.ptr->parse_state;
return self.ptr->first_leaf.parse_state;
}
static inline Length ts_subtree_padding(Subtree self) {
if (self.data.is_inline) {
Length result = {self.data.padding_bytes, {self.data.padding_rows, self.data.padding_columns}};
return result;
} else {
return self.ptr->padding;
}
static inline Length ts_subtree_padding(Subtree self)
{
if (self.data.is_inline)
{
Length result = {self.data.padding_bytes,
{self.data.padding_rows, self.data.padding_columns}};
return result;
}
else
{
return self.ptr->padding;
}
}
static inline Length ts_subtree_size(Subtree self) {
if (self.data.is_inline) {
Length result = {self.data.size_bytes, {0, self.data.size_bytes}};
return result;
} else {
return self.ptr->size;
}
static inline Length ts_subtree_size(Subtree self)
{
if (self.data.is_inline)
{
Length result = {self.data.size_bytes, {0, self.data.size_bytes}};
return result;
}
else
{
return self.ptr->size;
}
}
static inline Length ts_subtree_total_size(Subtree self) {
return length_add(ts_subtree_padding(self), ts_subtree_size(self));
static inline Length ts_subtree_total_size(Subtree self)
{
return length_add(ts_subtree_padding(self), ts_subtree_size(self));
}
static inline uint32_t ts_subtree_total_bytes(Subtree self) {
return ts_subtree_total_size(self).bytes;
static inline t_u32 ts_subtree_total_bytes(Subtree self)
{
return ts_subtree_total_size(self).bytes;
}
static inline uint32_t ts_subtree_child_count(Subtree self) {
return self.data.is_inline ? 0 : self.ptr->child_count;
static inline t_u32 ts_subtree_child_count(Subtree self)
{
return self.data.is_inline ? 0 : self.ptr->child_count;
}
static inline uint32_t ts_subtree_repeat_depth(Subtree self) {
return self.data.is_inline ? 0 : self.ptr->repeat_depth;
static inline t_u32 ts_subtree_repeat_depth(Subtree self)
{
return self.data.is_inline ? 0 : self.ptr->repeat_depth;
}
static inline uint32_t ts_subtree_is_repetition(Subtree self) {
return self.data.is_inline
? 0
: !self.ptr->named && !self.ptr->visible && self.ptr->child_count != 0;
static inline t_u32 ts_subtree_is_repetition(Subtree self)
{
return self.data.is_inline ? 0
: !self.ptr->named && !self.ptr->visible &&
self.ptr->child_count != 0;
}
static inline uint32_t ts_subtree_visible_descendant_count(Subtree self) {
return (self.data.is_inline || self.ptr->child_count == 0)
? 0
: self.ptr->visible_descendant_count;
static inline t_u32 ts_subtree_visible_descendant_count(Subtree self)
{
return (self.data.is_inline || self.ptr->child_count == 0)
? 0
: self.ptr->visible_descendant_count;
}
static inline uint32_t ts_subtree_visible_child_count(Subtree self) {
if (ts_subtree_child_count(self) > 0) {
return self.ptr->visible_child_count;
} else {
return 0;
}
static inline t_u32 ts_subtree_visible_child_count(Subtree self)
{
if (ts_subtree_child_count(self) > 0)
{
return self.ptr->visible_child_count;
}
else
{
return 0;
}
}
static inline uint32_t ts_subtree_error_cost(Subtree self) {
if (ts_subtree_missing(self)) {
return ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY;
} else {
return self.data.is_inline ? 0 : self.ptr->error_cost;
}
static inline t_u32 ts_subtree_error_cost(Subtree self)
{
if (ts_subtree_missing(self))
{
return ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY;
}
else
{
return self.data.is_inline ? 0 : self.ptr->error_cost;
}
}
static inline int32_t ts_subtree_dynamic_precedence(Subtree self) {
return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->dynamic_precedence;
static inline t_i32 ts_subtree_dynamic_precedence(Subtree self)
{
return (self.data.is_inline || self.ptr->child_count == 0)
? 0
: self.ptr->dynamic_precedence;
}
static inline uint16_t ts_subtree_production_id(Subtree self) {
if (ts_subtree_child_count(self) > 0) {
return self.ptr->production_id;
} else {
return 0;
}
static inline t_u16 ts_subtree_production_id(Subtree self)
{
if (ts_subtree_child_count(self) > 0)
{
return self.ptr->production_id;
}
else
{
return 0;
}
}
static inline bool ts_subtree_fragile_left(Subtree self) {
return self.data.is_inline ? false : self.ptr->fragile_left;
static inline bool ts_subtree_fragile_left(Subtree self)
{
return self.data.is_inline ? false : self.ptr->fragile_left;
}
static inline bool ts_subtree_fragile_right(Subtree self) {
return self.data.is_inline ? false : self.ptr->fragile_right;
static inline bool ts_subtree_fragile_right(Subtree self)
{
return self.data.is_inline ? false : self.ptr->fragile_right;
}
static inline bool ts_subtree_has_external_tokens(Subtree self) {
return self.data.is_inline ? false : self.ptr->has_external_tokens;
static inline bool ts_subtree_has_external_tokens(Subtree self)
{
return self.data.is_inline ? false : self.ptr->has_external_tokens;
}
static inline bool ts_subtree_has_external_scanner_state_change(Subtree self) {
return self.data.is_inline ? false : self.ptr->has_external_scanner_state_change;
static inline bool ts_subtree_has_external_scanner_state_change(Subtree self)
{
return self.data.is_inline ? false
: self.ptr->has_external_scanner_state_change;
}
static inline bool ts_subtree_depends_on_column(Subtree self) {
return self.data.is_inline ? false : self.ptr->depends_on_column;
static inline bool ts_subtree_depends_on_column(Subtree self)
{
return self.data.is_inline ? false : self.ptr->depends_on_column;
}
static inline bool ts_subtree_is_fragile(Subtree self) {
return self.data.is_inline ? false : (self.ptr->fragile_left || self.ptr->fragile_right);
static inline bool ts_subtree_is_fragile(Subtree self)
{
return self.data.is_inline
? false
: (self.ptr->fragile_left || self.ptr->fragile_right);
}
static inline bool ts_subtree_is_error(Subtree self) {
return ts_subtree_symbol(self) == ts_builtin_sym_error;
static inline bool ts_subtree_is_error(Subtree self)
{
return ts_subtree_symbol(self) == ts_builtin_sym_error;
}
static inline bool ts_subtree_is_eof(Subtree self) {
return ts_subtree_symbol(self) == ts_builtin_sym_end;
static inline bool ts_subtree_is_eof(Subtree self)
{
return ts_subtree_symbol(self) == ts_builtin_sym_end;
}
static inline Subtree ts_subtree_from_mut(MutableSubtree self) {
Subtree result;
result.data = self.data;
return result;
static inline Subtree ts_subtree_from_mut(MutableSubtree self)
{
Subtree result;
result.data = self.data;
return result;
}
static inline MutableSubtree ts_subtree_to_mut_unsafe(Subtree self) {
MutableSubtree result;
result.data = self.data;
return result;
static inline MutableSubtree ts_subtree_to_mt_unsafe(Subtree self)
{
MutableSubtree result;
result.data = self.data;
return result;
}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_SUBTREE_H_
#endif // TREE_SITTER_SUBTREE_H_

28
parser/src/tree.c
View file

@ -8,11 +8,11 @@
#include "./tree_cursor.h"
#include "./tree.h"
TSTree *ts_tree_new(
Subtree root, const TSLanguage *language,
t_parse_tree *ts_tree_new(
Subtree root, const t_language *language,
const t_parser_range *included_ranges, unsigned included_range_count
) {
TSTree *result = malloc(sizeof(TSTree));
t_parse_tree *result = malloc(sizeof(t_parse_tree));
result->root = root;
result->language = ts_language_copy(language);
result->included_ranges = calloc(included_range_count, sizeof(t_parser_range));
@ -21,12 +21,12 @@ TSTree *ts_tree_new(
return result;
}
TSTree *ts_tree_copy(const TSTree *self) {
t_parse_tree *ts_tree_copy(const t_parse_tree *self) {
ts_subtree_retain(self->root);
return ts_tree_new(self->root, self->language, self->included_ranges, self->included_range_count);
}
void ts_tree_delete(TSTree *self) {
void ts_tree_delete(t_parse_tree *self) {
if (!self) return;
SubtreePool pool = ts_subtree_pool_new(0);
@ -37,24 +37,24 @@ void ts_tree_delete(TSTree *self) {
free(self);
}
TSNode ts_tree_root_node(const TSTree *self) {
t_parse_node ts_tree_root_node(const t_parse_tree *self) {
return ts_node_new(self, &self->root, ts_subtree_padding(self->root), 0);
}
TSNode ts_tree_root_node_with_offset(
const TSTree *self,
uint32_t offset_bytes,
t_parse_node ts_tree_root_node_with_offset(
const t_parse_tree *self,
t_u32 offset_bytes,
t_point offset_extent
) {
Length offset = {offset_bytes, offset_extent};
return ts_node_new(self, &self->root, length_add(offset, ts_subtree_padding(self->root)), 0);
}
const TSLanguage *ts_tree_language(const TSTree *self) {
const t_language *ts_tree_language(const t_parse_tree *self) {
return self->language;
}
void ts_tree_edit(TSTree *self, const TSInputEdit *edit) {
void ts_tree_edit(t_parse_tree *self, const t_input_edit *edit) {
for (unsigned i = 0; i < self->included_range_count; i++) {
t_parser_range *range = &self->included_ranges[i];
if (range->end_byte >= edit->old_end_byte) {
@ -94,7 +94,7 @@ void ts_tree_edit(TSTree *self, const TSInputEdit *edit) {
ts_subtree_pool_delete(&pool);
}
t_parser_range *ts_tree_included_ranges(const TSTree *self, uint32_t *length) {
t_parser_range *ts_tree_included_ranges(const t_parse_tree *self, t_u32 *length) {
*length = self->included_range_count;
t_parser_range *ranges = calloc(self->included_range_count, sizeof(t_parser_range));
memcpy(ranges, self->included_ranges, self->included_range_count * sizeof(t_parser_range));
@ -117,7 +117,7 @@ int _ts_dup(HANDLE handle) {
return _open_osfhandle((intptr_t)dup_handle, 0);
}
void ts_tree_print_dot_graph(const TSTree *self, int fd) {
void ts_tree_print_dot_graph(const t_parse_tree *self, int fd) {
FILE *file = _fdopen(_ts_dup((HANDLE)_get_osfhandle(fd)), "a");
ts_subtree_print_dot_graph(self->root, self->language, file);
fclose(file);
@ -131,7 +131,7 @@ int _ts_dup(int file_descriptor) {
return dup(file_descriptor);
}
void ts_tree_print_dot_graph(const TSTree *self, int file_descriptor) {
void ts_tree_print_dot_graph(const t_parse_tree *self, int file_descriptor) {
FILE *file = fdopen(_ts_dup(file_descriptor), "a");
ts_subtree_print_dot_graph(self->root, self->language, file);
fclose(file);

10
parser/src/tree.h
View file

@ -11,18 +11,18 @@ typedef struct {
const Subtree *child;
const Subtree *parent;
Length position;
TSSymbol alias_symbol;
t_symbol alias_symbol;
} ParentCacheEntry;
struct TSTree {
struct t_parse_tree {
Subtree root;
const TSLanguage *language;
const t_language *language;
t_parser_range *included_ranges;
unsigned included_range_count;
};
TSTree *ts_tree_new(Subtree root, const TSLanguage *language, const t_parser_range *, unsigned);
TSNode ts_node_new(const TSTree *, const Subtree *, Length, TSSymbol);
t_parse_tree *ts_tree_new(Subtree root, const t_language *language, const t_parser_range *, unsigned);
t_parse_node ts_node_new(const t_parse_tree *, const Subtree *, Length, t_symbol);
#ifdef __cplusplus
}

132
parser/src/tree_cursor.c
View file

@ -6,17 +6,17 @@
typedef struct {
Subtree parent;
const TSTree *tree;
const t_parse_tree *tree;
Length position;
uint32_t child_index;
uint32_t structural_child_index;
uint32_t descendant_index;
const TSSymbol *alias_sequence;
t_u32 child_index;
t_u32 structural_child_index;
t_u32 descendant_index;
const t_symbol *alias_sequence;
} CursorChildIterator;
// CursorChildIterator
static inline bool ts_tree_cursor_is_entry_visible(const TreeCursor *self, uint32_t index) {
static inline bool ts_tree_cursor_is_entry_visible(const TreeCursor *self, t_u32 index) {
TreeCursorEntry *entry = &self->stack.contents[index];
if (index == 0 || ts_subtree_visible(*entry->subtree)) {
return true;
@ -37,12 +37,12 @@ static inline CursorChildIterator ts_tree_cursor_iterate_children(const TreeCurs
if (ts_subtree_child_count(*last_entry->subtree) == 0) {
return (CursorChildIterator) {NULL_SUBTREE, self->tree, length_zero(), 0, 0, 0, NULL};
}
const TSSymbol *alias_sequence = ts_language_alias_sequence(
const t_symbol *alias_sequence = ts_language_alias_sequence(
self->tree->language,
last_entry->subtree->ptr->production_id
);
uint32_t descendant_index = last_entry->descendant_index;
t_u32 descendant_index = last_entry->descendant_index;
if (ts_tree_cursor_is_entry_visible(self, self->stack.size - 1)) {
descendant_index += 1;
}
@ -120,7 +120,7 @@ static inline bool ts_tree_cursor_child_iterator_previous(
) {
// this is mostly a reverse `ts_tree_cursor_child_iterator_next` taking into
// account unsigned underflow
if (!self->parent.ptr || (int8_t)self->child_index == -1) return false;
if (!self->parent.ptr || (t_i8)self->child_index == -1) return false;
const Subtree *child = &ts_subtree_children(self->parent)[self->child_index];
*result = (TreeCursorEntry) {
.subtree = child,
@ -148,19 +148,19 @@ static inline bool ts_tree_cursor_child_iterator_previous(
return true;
}
// TSTreeCursor - lifecycle
// t_parse_tree_cursor - lifecycle
TSTreeCursor ts_tree_cursor_new(TSNode node) {
TSTreeCursor self = {NULL, NULL, {0, 0, 0}};
t_parse_tree_cursor ts_tree_cursor_new(t_parse_node node) {
t_parse_tree_cursor self = {NULL, NULL, {0, 0, 0}};
ts_tree_cursor_init((TreeCursor *)&self, node);
return self;
}
void ts_tree_cursor_reset(TSTreeCursor *_self, TSNode node) {
void ts_tree_cursor_reset(t_parse_tree_cursor *_self, t_parse_node node) {
ts_tree_cursor_init((TreeCursor *)_self, node);
}
void ts_tree_cursor_init(TreeCursor *self, TSNode node) {
void ts_tree_cursor_init(TreeCursor *self, t_parse_node node) {
self->tree = node.tree;
self->root_alias_symbol = node.context[3];
array_clear(&self->stack);
@ -176,14 +176,14 @@ void ts_tree_cursor_init(TreeCursor *self, TSNode node) {
}));
}
void ts_tree_cursor_delete(TSTreeCursor *_self) {
void ts_tree_cursor_delete(t_parse_tree_cursor *_self) {
TreeCursor *self = (TreeCursor *)_self;
array_delete(&self->stack);
}
// TSTreeCursor - walking the tree
// t_parse_tree_cursor - walking the tree
TreeCursorStep ts_tree_cursor_goto_first_child_internal(TSTreeCursor *_self) {
TreeCursorStep ts_tree_cursor_goto_first_child_internal(t_parse_tree_cursor *_self) {
TreeCursor *self = (TreeCursor *)_self;
bool visible;
TreeCursorEntry entry;
@ -201,7 +201,7 @@ TreeCursorStep ts_tree_cursor_goto_first_child_internal(TSTreeCursor *_self) {
return TreeCursorStepNone;
}
bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) {
bool ts_tree_cursor_goto_first_child(t_parse_tree_cursor *self) {
for (;;) {
switch (ts_tree_cursor_goto_first_child_internal(self)) {
case TreeCursorStepHidden:
@ -215,7 +215,7 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) {
return false;
}
TreeCursorStep ts_tree_cursor_goto_last_child_internal(TSTreeCursor *_self) {
TreeCursorStep ts_tree_cursor_goto_last_child_internal(t_parse_tree_cursor *_self) {
TreeCursor *self = (TreeCursor *)_self;
bool visible;
TreeCursorEntry entry;
@ -242,7 +242,7 @@ TreeCursorStep ts_tree_cursor_goto_last_child_internal(TSTreeCursor *_self) {
return TreeCursorStepNone;
}
bool ts_tree_cursor_goto_last_child(TSTreeCursor *self) {
bool ts_tree_cursor_goto_last_child(t_parse_tree_cursor *self) {
for (;;) {
switch (ts_tree_cursor_goto_last_child_internal(self)) {
case TreeCursorStepHidden:
@ -256,14 +256,14 @@ bool ts_tree_cursor_goto_last_child(TSTreeCursor *self) {
return false;
}
static inline int64_t ts_tree_cursor_goto_first_child_for_byte_and_point(
TSTreeCursor *_self,
uint32_t goal_byte,
static inline t_i64 ts_tree_cursor_goto_first_child_for_byte_and_point(
t_parse_tree_cursor *_self,
t_u32 goal_byte,
t_point goal_point
) {
TreeCursor *self = (TreeCursor *)_self;
uint32_t initial_size = self->stack.size;
uint32_t visible_child_index = 0;
t_u32 initial_size = self->stack.size;
t_u32 visible_child_index = 0;
bool did_descend;
do {
@ -275,7 +275,7 @@ static inline int64_t ts_tree_cursor_goto_first_child_for_byte_and_point(
while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) {
Length entry_end = length_add(entry.position, ts_subtree_size(*entry.subtree));
bool at_goal = entry_end.bytes >= goal_byte && point_gte(entry_end.extent, goal_point);
uint32_t visible_child_count = ts_subtree_visible_child_count(*entry.subtree);
t_u32 visible_child_count = ts_subtree_visible_child_count(*entry.subtree);
if (at_goal) {
if (visible) {
array_push(&self->stack, entry);
@ -298,19 +298,19 @@ static inline int64_t ts_tree_cursor_goto_first_child_for_byte_and_point(
return -1;
}
int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *self, uint32_t goal_byte) {
t_i64 ts_tree_cursor_goto_first_child_for_byte(t_parse_tree_cursor *self, t_u32 goal_byte) {
return ts_tree_cursor_goto_first_child_for_byte_and_point(self, goal_byte, POINT_ZERO);
}
int64_t ts_tree_cursor_goto_first_child_for_point(TSTreeCursor *self, t_point goal_point) {
t_i64 ts_tree_cursor_goto_first_child_for_point(t_parse_tree_cursor *self, t_point goal_point) {
return ts_tree_cursor_goto_first_child_for_byte_and_point(self, 0, goal_point);
}
TreeCursorStep ts_tree_cursor_goto_sibling_internal(
TSTreeCursor *_self,
t_parse_tree_cursor *_self,
bool (*advance)(CursorChildIterator *, TreeCursorEntry *, bool *)) {
TreeCursor *self = (TreeCursor *)_self;
uint32_t initial_size = self->stack.size;
t_u32 initial_size = self->stack.size;
while (self->stack.size > 1) {
TreeCursorEntry entry = array_pop(&self->stack);
@ -341,11 +341,11 @@ TreeCursorStep ts_tree_cursor_goto_sibling_internal(
return TreeCursorStepNone;
}
TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *_self) {
TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(t_parse_tree_cursor *_self) {
return ts_tree_cursor_goto_sibling_internal(_self, ts_tree_cursor_child_iterator_next);
}
bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) {
bool ts_tree_cursor_goto_next_sibling(t_parse_tree_cursor *self) {
switch (ts_tree_cursor_goto_next_sibling_internal(self)) {
case TreeCursorStepHidden:
ts_tree_cursor_goto_first_child(self);
@ -357,7 +357,7 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) {
}
}
TreeCursorStep ts_tree_cursor_goto_previous_sibling_internal(TSTreeCursor *_self) {
TreeCursorStep ts_tree_cursor_goto_previous_sibling_internal(t_parse_tree_cursor *_self) {
// since subtracting across row loses column information, we may have to
// restore it
TreeCursor *self = (TreeCursor *)_self;
@ -375,13 +375,13 @@ TreeCursorStep ts_tree_cursor_goto_previous_sibling_internal(TSTreeCursor *_self
// restore position from the parent node
const TreeCursorEntry *parent = &self->stack.contents[self->stack.size - 2];
Length position = parent->position;
uint32_t child_index = array_back(&self->stack)->child_index;
t_u32 child_index = array_back(&self->stack)->child_index;
const Subtree *children = ts_subtree_children((*(parent->subtree)));
if (child_index > 0) {
// skip first child padding since its position should match the position of the parent
position = length_add(position, ts_subtree_size(children[0]));
for (uint32_t i = 1; i < child_index; ++i) {
for (t_u32 i = 1; i < child_index; ++i) {
position = length_add(position, ts_subtree_total_size(children[i]));
}
position = length_add(position, ts_subtree_padding(children[child_index]));
@ -392,7 +392,7 @@ TreeCursorStep ts_tree_cursor_goto_previous_sibling_internal(TSTreeCursor *_self
return step;
}
bool ts_tree_cursor_goto_previous_sibling(TSTreeCursor *self) {
bool ts_tree_cursor_goto_previous_sibling(t_parse_tree_cursor *self) {
switch (ts_tree_cursor_goto_previous_sibling_internal(self)) {
case TreeCursorStepHidden:
ts_tree_cursor_goto_last_child(self);
@ -404,7 +404,7 @@ bool ts_tree_cursor_goto_previous_sibling(TSTreeCursor *self) {
}
}
bool ts_tree_cursor_goto_parent(TSTreeCursor *_self) {
bool ts_tree_cursor_goto_parent(t_parse_tree_cursor *_self) {
TreeCursor *self = (TreeCursor *)_self;
for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) {
if (ts_tree_cursor_is_entry_visible(self, i)) {
@ -416,16 +416,16 @@ bool ts_tree_cursor_goto_parent(TSTreeCursor *_self) {
}
void ts_tree_cursor_goto_descendant(
TSTreeCursor *_self,
uint32_t goal_descendant_index
t_parse_tree_cursor *_self,
t_u32 goal_descendant_index
) {
TreeCursor *self = (TreeCursor *)_self;
// Ascend to the lowest ancestor that contains the goal node.
for (;;) {
uint32_t i = self->stack.size - 1;
t_u32 i = self->stack.size - 1;
TreeCursorEntry *entry = &self->stack.contents[i];
uint32_t next_descendant_index =
t_u32 next_descendant_index =
entry->descendant_index +
(ts_tree_cursor_is_entry_visible(self, i) ? 1 : 0) +
ts_subtree_visible_descendant_count(*entry->subtree);
@ -466,16 +466,16 @@ void ts_tree_cursor_goto_descendant(
} while (did_descend);
}
uint32_t ts_tree_cursor_current_descendant_index(const TSTreeCursor *_self) {
t_u32 ts_tree_cursor_current_descendant_index(const t_parse_tree_cursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self;
TreeCursorEntry *last_entry = array_back(&self->stack);
return last_entry->descendant_index;
}
TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self) {
t_parse_node ts_tree_cursor_current_node(const t_parse_tree_cursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self;
TreeCursorEntry *last_entry = array_back(&self->stack);
TSSymbol alias_symbol = self->root_alias_symbol;
t_symbol alias_symbol = self->root_alias_symbol;
if (self->stack.size > 1 && !ts_subtree_extra(*last_entry->subtree)) {
TreeCursorEntry *parent_entry = &self->stack.contents[self->stack.size - 2];
alias_symbol = ts_language_alias_at(
@ -495,12 +495,12 @@ TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self) {
// Private - Get various facts about the current node that are needed
// when executing tree queries.
void ts_tree_cursor_current_status(
const TSTreeCursor *_self,
TSFieldId *field_id,
const t_parse_tree_cursor *_self,
t_field_id *field_id,
bool *has_later_siblings,
bool *has_later_named_siblings,
bool *can_have_later_siblings_with_this_field,
TSSymbol *supertypes,
t_symbol *supertypes,
unsigned *supertype_count
) {
const TreeCursor *self = (const TreeCursor *)_self;
@ -517,7 +517,7 @@ void ts_tree_cursor_current_status(
TreeCursorEntry *entry = &self->stack.contents[i];
TreeCursorEntry *parent_entry = &self->stack.contents[i - 1];
const TSSymbol *alias_sequence = ts_language_alias_sequence(
const t_symbol *alias_sequence = ts_language_alias_sequence(
self->tree->language,
parent_entry->subtree->ptr->production_id
);
@ -532,11 +532,11 @@ void ts_tree_cursor_current_status(
ts_subtree_symbol(subtree))
// Stop walking up when a visible ancestor is found.
TSSymbol entry_symbol = subtree_symbol(
t_symbol entry_symbol = subtree_symbol(
*entry->subtree,
entry->structural_child_index
);
TSSymbolMetadata entry_metadata = ts_language_symbol_metadata(
t_symbol_metadata entry_metadata = ts_language_symbol_metadata(
self->tree->language,
entry_symbol
);
@ -555,7 +555,7 @@ void ts_tree_cursor_current_status(
if (!ts_subtree_extra(*entry->subtree)) structural_child_index++;
for (unsigned j = entry->child_index + 1; j < sibling_count; j++) {
Subtree sibling = ts_subtree_children(*parent_entry->subtree)[j];
TSSymbolMetadata sibling_metadata = ts_language_symbol_metadata(
t_symbol_metadata sibling_metadata = ts_language_symbol_metadata(
self->tree->language,
subtree_symbol(sibling, structural_child_index)
);
@ -581,7 +581,7 @@ void ts_tree_cursor_current_status(
#undef subtree_symbol
if (!ts_subtree_extra(*entry->subtree)) {
const TSFieldMapEntry *field_map, *field_map_end;
const t_field_map_entry *field_map, *field_map_end;
ts_language_field_map(
self->tree->language,
parent_entry->subtree->ptr->production_id,
@ -590,7 +590,7 @@ void ts_tree_cursor_current_status(
// Look for a field name associated with the current node.
if (!*field_id) {
for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) {
for (const t_field_map_entry *map = field_map; map < field_map_end; map++) {
if (!map->inherited && map->child_index == entry->structural_child_index) {
*field_id = map->field_id;
break;
@ -600,7 +600,7 @@ void ts_tree_cursor_current_status(
// Determine if the current node can have later siblings with the same field name.
if (*field_id) {
for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) {
for (const t_field_map_entry *map = field_map; map < field_map_end; map++) {
if (
map->field_id == *field_id &&
map->child_index > entry->structural_child_index
@ -614,9 +614,9 @@ void ts_tree_cursor_current_status(
}
}
uint32_t ts_tree_cursor_current_depth(const TSTreeCursor *_self) {
t_u32 ts_tree_cursor_current_depth(const t_parse_tree_cursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self;
uint32_t depth = 0;
t_u32 depth = 0;
for (unsigned i = 1; i < self->stack.size; i++) {
if (ts_tree_cursor_is_entry_visible(self, i)) {
depth++;
@ -625,12 +625,12 @@ uint32_t ts_tree_cursor_current_depth(const TSTreeCursor *_self) {
return depth;
}
TSNode ts_tree_cursor_parent_node(const TSTreeCursor *_self) {
t_parse_node ts_tree_cursor_parent_node(const t_parse_tree_cursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self;
for (int i = (int)self->stack.size - 2; i >= 0; i--) {
TreeCursorEntry *entry = &self->stack.contents[i];
bool is_visible = true;
TSSymbol alias_symbol = 0;
t_symbol alias_symbol = 0;
if (i > 0) {
TreeCursorEntry *parent_entry = &self->stack.contents[i - 1];
alias_symbol = ts_language_alias_at(
@ -652,7 +652,7 @@ TSNode ts_tree_cursor_parent_node(const TSTreeCursor *_self) {
return ts_node_new(NULL, NULL, length_zero(), 0);
}
TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *_self) {
t_field_id ts_tree_cursor_current_field_id(const t_parse_tree_cursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self;
// Walk up the tree, visiting the current node and its invisible ancestors.
@ -668,13 +668,13 @@ TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *_self) {
if (ts_subtree_extra(*entry->subtree)) break;
const TSFieldMapEntry *field_map, *field_map_end;
const t_field_map_entry *field_map, *field_map_end;
ts_language_field_map(
self->tree->language,
parent_entry->subtree->ptr->production_id,
&field_map, &field_map_end
);
for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) {
for (const t_field_map_entry *map = field_map; map < field_map_end; map++) {
if (!map->inherited && map->child_index == entry->structural_child_index) {
return map->field_id;
}
@ -683,8 +683,8 @@ TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *_self) {
return 0;
}
const char *ts_tree_cursor_current_field_name(const TSTreeCursor *_self) {
TSFieldId id = ts_tree_cursor_current_field_id(_self);
const char *ts_tree_cursor_current_field_name(const t_parse_tree_cursor *_self) {
t_field_id id = ts_tree_cursor_current_field_id(_self);
if (id) {
const TreeCursor *self = (const TreeCursor *)_self;
return self->tree->language->field_names[id];
@ -693,9 +693,9 @@ const char *ts_tree_cursor_current_field_name(const TSTreeCursor *_self) {
}
}
TSTreeCursor ts_tree_cursor_copy(const TSTreeCursor *_cursor) {
t_parse_tree_cursor ts_tree_cursor_copy(const t_parse_tree_cursor *_cursor) {
const TreeCursor *cursor = (const TreeCursor *)_cursor;
TSTreeCursor res = {NULL, NULL, {0, 0}};
t_parse_tree_cursor res = {NULL, NULL, {0, 0}};
TreeCursor *copy = (TreeCursor *)&res;
copy->tree = cursor->tree;
copy->root_alias_symbol = cursor->root_alias_symbol;
@ -704,7 +704,7 @@ TSTreeCursor ts_tree_cursor_copy(const TSTreeCursor *_cursor) {
return res;
}
void ts_tree_cursor_reset_to(TSTreeCursor *_dst, const TSTreeCursor *_src) {
void ts_tree_cursor_reset_to(t_parse_tree_cursor *_dst, const t_parse_tree_cursor *_src) {
const TreeCursor *cursor = (const TreeCursor *)_src;
TreeCursor *copy = (TreeCursor *)_dst;
copy->tree = cursor->tree;

26
parser/src/tree_cursor.h
View file

@ -6,15 +6,15 @@
typedef struct {
const Subtree *subtree;
Length position;
uint32_t child_index;
uint32_t structural_child_index;
uint32_t descendant_index;
t_u32 child_index;
t_u32 structural_child_index;
t_u32 descendant_index;
} TreeCursorEntry;
typedef struct {
const TSTree *tree;
const t_parse_tree *tree;
Array(TreeCursorEntry) stack;
TSSymbol root_alias_symbol;
t_symbol root_alias_symbol;
} TreeCursor;
typedef enum {
@ -23,26 +23,26 @@ typedef enum {
TreeCursorStepVisible,
} TreeCursorStep;
void ts_tree_cursor_init(TreeCursor *, TSNode);
void ts_tree_cursor_init(TreeCursor *, t_parse_node);
void ts_tree_cursor_current_status(
const TSTreeCursor *,
TSFieldId *,
const t_parse_tree_cursor *,
t_field_id *,
bool *,
bool *,
bool *,
TSSymbol *,
t_symbol *,
unsigned *
);
TreeCursorStep ts_tree_cursor_goto_first_child_internal(TSTreeCursor *);
TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *);
TreeCursorStep ts_tree_cursor_goto_first_child_internal(t_parse_tree_cursor *);
TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(t_parse_tree_cursor *);
static inline Subtree ts_tree_cursor_current_subtree(const TSTreeCursor *_self) {
static inline Subtree ts_tree_cursor_current_subtree(const t_parse_tree_cursor *_self) {
const TreeCursor *self = (const TreeCursor *)_self;
TreeCursorEntry *last_entry = array_back(&self->stack);
return *last_entry->subtree;
}
TSNode ts_tree_cursor_parent_node(const TSTreeCursor *);
t_parse_node ts_tree_cursor_parent_node(const t_parse_tree_cursor *);
#endif // TREE_SITTER_TREE_CURSOR_H_