raphael/minishell
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactoring some stuff in the parser lib, moving functions out of headers

Browse source
This commit is contained in:
Maix0 2024-07-04 00:43:43 +02:00
parent 4580d68951
commit fb3a2d94a0
19 changed files with 522 additions and 608 deletions
Download patch file Download diff file Expand all files Collapse all files

10
parser/src/input.c Normal file
View file

@ -0,0 +1,10 @@
#include "./input.h"
#include "me/types.h"
t_u32 ts_decode_ascii(const t_u8 *string, t_u32 length, t_i32 *code_point)
{
(void)(length);
*code_point = 0;
*(t_u8 *)code_point = *string;
return (1);
}

12
parser/src/input.h Normal file
View file

@ -0,0 +1,12 @@
#ifndef INPUT_H
#define INPUT_H
#include "me/types.h"
#define TS_DECODE_ERROR -1i
typedef t_u32 (*UnicodeDecodeFunction)(const t_u8 *string, t_u32 length, t_i32 *code_point);
t_u32 ts_decode_ascii(const t_u8 *string, t_u32 length, t_i32 *code_point);
#endif // INPUT_H

316
parser/src/language.c
View file

@ -1,12 +1,13 @@
#include "./language.h"
#include "./api.h"
#include "me/types.h"
#include "parser.h"
#include <assert.h>
#include <string.h>
const TSLanguage *ts_language_copy(const TSLanguage *self)
{
return self;
return (self);
}
void ts_language_delete(const TSLanguage *self)
@ -16,26 +17,29 @@ void ts_language_delete(const TSLanguage *self)
t_u32 ts_language_symbol_count(const TSLanguage *self)
{
return self->symbol_count + self->alias_count;
return (self->symbol_count + self->alias_count);
}
t_u32 ts_language_state_count(const TSLanguage *self)
{
return self->state_count;
return (self->state_count);
}
t_u32 ts_language_version(const TSLanguage *self)
{
return self->version;
return (self->version);
}
t_u32 ts_language_field_count(const TSLanguage *self)
{
return self->field_count;
return (self->field_count);
}
void ts_language_table_entry(const TSLanguage *self, TSStateId state, TSSymbol symbol, TableEntry *result)
{
t_u32 action_index;
const TSParseActionEntry *entry;
if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat)
{
result->action_count = 0;
@ -45,8 +49,8 @@ void ts_language_table_entry(const TSLanguage *self, TSStateId state, TSSymbol s
else
{
assert(symbol < self->token_count);
t_u32 action_index = ts_language_lookup(self, state, symbol);
const TSParseActionEntry *entry = &self->parse_actions[action_index];
action_index = ts_language_lookup(self, state, symbol);
entry = &self->parse_actions[action_index];
result->action_count = entry->entry.count;
result->is_reusable = entry->entry.reusable;
result->actions = (const TSParseAction *)(entry + 1);
@ -56,182 +60,252 @@ void ts_language_table_entry(const TSLanguage *self, TSStateId state, TSSymbol s
TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *self, TSSymbol symbol)
{
if (symbol == ts_builtin_sym_error)
{
return (TSSymbolMetadata){.visible = true, .named = true};
}
return ((TSSymbolMetadata){.visible = true, .named = true});
else if (symbol == ts_builtin_sym_error_repeat)
{
return (TSSymbolMetadata){.visible = false, .named = false};
}
return ((TSSymbolMetadata){.visible = false, .named = false});
else
{
return self->symbol_metadata[symbol];
}
return (self->symbol_metadata[symbol]);
}
TSSymbol ts_language_public_symbol(const TSLanguage *self, TSSymbol symbol)
{
if (symbol == ts_builtin_sym_error)
return symbol;
return self->public_symbol_map[symbol];
return (symbol);
return (self->public_symbol_map[symbol]);
}
TSStateId ts_language_next_state(const TSLanguage *self, TSStateId state, TSSymbol symbol)
{
t_u32 count;
const TSParseAction *actions;
TSParseAction action;
if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat)
{
return 0;
return (0);
}
else if (symbol < self->token_count)
{
t_u32 count;
const TSParseAction *actions = ts_language_actions(self, state, symbol, &count);
actions = ts_language_actions(self, state, symbol, &count);
if (count > 0)
{
TSParseAction action = actions[count - 1];
action = actions[count - 1];
if (action.type == TSParseActionTypeShift)
{
return action.shift.extra ? state : action.shift.state;
return (action.shift.extra ? state : action.shift.state);
}
}
return 0;
return (0);
}
else
{
return ts_language_lookup(self, state, symbol);
}
return (ts_language_lookup(self, state, symbol));
}
t_const_str ts_language_symbol_name(const TSLanguage *self, TSSymbol symbol)
{
if (symbol == ts_builtin_sym_error)
{
return "ERROR";
}
return ("ERROR");
else if (symbol == ts_builtin_sym_error_repeat)
{
return "_ERROR";
}
return ("_ERROR");
else if (symbol < ts_language_symbol_count(self))
{
return self->symbol_names[symbol];
}
return (self->symbol_names[symbol]);
else
{
return NULL;
}
return (NULL);
}
TSSymbol ts_language_symbol_for_name(const TSLanguage *self, t_const_str string, t_u32 length, bool is_named)
{
TSSymbolMetadata metadata;
t_u16 count;
TSSymbol i;
t_const_str symbol_name;
if (!strncmp(string, "ERROR", length))
return ts_builtin_sym_error;
t_u16 count = (t_u16)ts_language_symbol_count(self);
for (TSSymbol i = 0; i < count; i++)
return (ts_builtin_sym_error);
count = (t_u16)ts_language_symbol_count(self);
i = 0;
while (i < count)
{
TSSymbolMetadata metadata = ts_language_symbol_metadata(self, i);
metadata = ts_language_symbol_metadata(self, i);
if ((!metadata.visible && !metadata.supertype) || metadata.named != is_named)
continue;
t_const_str symbol_name = self->symbol_names[i];
if (!strncmp(symbol_name, string, length) && !symbol_name[length])
{
return self->public_symbol_map[i];
i++;
continue;
}
symbol_name = self->symbol_names[i];
if (!strncmp(symbol_name, string, length) && !symbol_name[length])
return (self->public_symbol_map[i]);
i++;
}
return 0;
return (0);
}
TSSymbolType ts_language_symbol_type(const TSLanguage *self, TSSymbol symbol)
{
TSSymbolMetadata metadata = ts_language_symbol_metadata(self, symbol);
TSSymbolMetadata metadata;
metadata = ts_language_symbol_metadata(self, symbol);
if (metadata.named && metadata.visible)
{
return TSSymbolTypeRegular;
}
return (TSSymbolTypeRegular);
else if (metadata.visible)
{
return TSSymbolTypeAnonymous;
}
return (TSSymbolTypeAnonymous);
else
{
return TSSymbolTypeAuxiliary;
}
return (TSSymbolTypeAuxiliary);
}
t_const_str ts_language_field_name_for_id(const TSLanguage *self, TSFieldId id)
{
t_u32 count = ts_language_field_count(self);
t_u32 count;
count = ts_language_field_count(self);
if (count && id <= count)
{
return self->field_names[id];
}
return (self->field_names[id]);
else
{
return NULL;
}
return (NULL);
}
TSFieldId ts_language_field_id_for_name(const TSLanguage *self, t_const_str name, t_u32 name_length)
{
t_u16 count = (t_u16)ts_language_field_count(self);
for (TSSymbol i = 1; i < count + 1; i++)
t_u16 count;
TSSymbol i;
t_i32 res;
count = (t_u16)ts_language_field_count(self);
i = 1;
while (i < count + 1)
{
switch (strncmp(name, self->field_names[i], name_length))
res = strncmp(name, self->field_names[i], name_length);
if (res == 0)
{
case 0:
if (self->field_names[i][name_length] == 0)
return i;
return (i);
}
else if (res == -1)
return (0);
i++;
}
return (0);
}
bool ts_language_is_symbol_external(const TSLanguage *self, TSSymbol symbol)
{
return (0 < symbol && symbol < self->external_token_count + 1);
}
const TSParseAction *ts_language_actions(const TSLanguage *self, TSStateId state, TSSymbol symbol, t_u32 *count)
{
TableEntry entry;
ts_language_table_entry(self, state, symbol, &entry);
*count = entry.action_count;
return (entry.actions);
}
bool ts_language_has_reduce_action(const TSLanguage *self, TSStateId state, TSSymbol symbol)
{
TableEntry entry;
ts_language_table_entry(self, state, symbol, &entry);
return (entry.action_count > 0 && entry.actions[0].type == TSParseActionTypeReduce);
}
// Lookup the table value for a given symbol and state.
//
// For non-terminal symbols, the table value represents a successor state.
// For terminal symbols, it represents an index in the actions table.
// For 'large' parse states, this is a direct lookup. For 'small' parse
// states, this requires searching through the symbol groups to find
// the given symbol.
t_u16 ts_language_lookup(const TSLanguage *self, TSStateId state, TSSymbol symbol)
{
t_u32 index;
const t_u16 *data;
t_u16 group_count;
t_u16 section_value;
t_u16 symbol_count;
t_u32 i;
t_u32 j;
if (state >= self->large_state_count)
{
index = self->small_parse_table_map[state - self->large_state_count];
data = &self->small_parse_table[index];
group_count = *(data++);
i = 0;
while (i++ < group_count)
{
section_value = *(data++);
symbol_count = *(data++);
j = 0;
while (j++ < symbol_count)
if (*(data++) == symbol)
return (section_value);
}
return (0);
}
else
return (self->parse_table[state * self->symbol_count + symbol]);
}
bool ts_language_has_actions(const TSLanguage *self, TSStateId state, TSSymbol symbol)
{
return (ts_language_lookup(self, state, symbol) != 0);
}
const bool *ts_language_enabled_external_tokens(const TSLanguage *self, t_u32 external_scanner_state)
{
if (external_scanner_state == 0)
return (NULL);
else
return (self->external_scanner.states + self->external_token_count * external_scanner_state);
}
const TSSymbol *ts_language_alias_sequence(const TSLanguage *self, t_u32 production_id)
{
return (production_id ? &self->alias_sequences[production_id * self->max_alias_sequence_length] : NULL);
}
TSSymbol ts_language_alias_at(const TSLanguage *self, t_u32 production_id, t_u32 child_index)
{
return (production_id ? self->alias_sequences[production_id * self->max_alias_sequence_length + child_index] : 0);
}
void ts_language_field_map(const TSLanguage *self, t_u32 production_id, const TSFieldMapEntry **start, const TSFieldMapEntry **end)
{
TSFieldMapSlice slice;
if (self->field_count == 0)
{
*start = NULL;
*end = NULL;
return;
}
slice = self->field_map_slices[production_id];
*start = &self->field_map_entries[slice.index];
*end = &self->field_map_entries[slice.index] + slice.length;
}
void ts_language_aliases_for_symbol(const TSLanguage *self, TSSymbol original_symbol, const TSSymbol **start, const TSSymbol **end)
{
t_u32 idx;
TSSymbol symbol;
t_u16 count;
*start = &self->public_symbol_map[original_symbol];
*end = *start + 1;
idx = 0;
while (true)
{
symbol = self->alias_map[idx++];
if (symbol == 0 || symbol > original_symbol)
break;
case -1:
return 0;
default:
count = self->alias_map[idx++];
if (symbol == original_symbol)
{
*start = &self->alias_map[idx];
*end = &self->alias_map[idx + count];
break;
}
idx += count;
}
return 0;
}
// TSLookaheadIterator *ts_lookahead_iterator_new(const TSLanguage *self, TSStateId state) {
// if (state >= self->state_count) return NULL;
// LookaheadIterator *iterator = mem_alloc(sizeof(LookaheadIterator));
// *iterator = ts_language_lookaheads(self, state);
// return (TSLookaheadIterator *)iterator;
// }
// void ts_lookahead_iterator_delete(TSLookaheadIterator *self) {
// mem_free(self);
// }
// bool ts_lookahead_iterator_reset_state(TSLookaheadIterator * self, TSStateId state) {
// LookaheadIterator *iterator = (LookaheadIterator *)self;
// if (state >= iterator->language->state_count) return false;
// *iterator = ts_language_lookaheads(iterator->language, state);
// return true;
// }
// const TSLanguage *ts_lookahead_iterator_language(const TSLookaheadIterator *self) {
// const LookaheadIterator *iterator = (const LookaheadIterator *)self;
// return iterator->language;
// }
// bool ts_lookahead_iterator_reset(TSLookaheadIterator *self, const TSLanguage *language, TSStateId state) {
// if (state >= language->state_count) return false;
// LookaheadIterator *iterator = (LookaheadIterator *)self;
// *iterator = ts_language_lookaheads(language, state);
// return true;
// }
// bool ts_lookahead_iterator_next(TSLookaheadIterator *self) {
// LookaheadIterator *iterator = (LookaheadIterator *)self;
// return ts_lookahead_iterator__next(iterator);
// }
// TSSymbol ts_lookahead_iterator_current_symbol(const TSLookaheadIterator *self) {
// const LookaheadIterator *iterator = (const LookaheadIterator *)self;
// return iterator->symbol;
// }
// t_const_str ts_lookahead_iterator_current_symbol_name(const TSLookaheadIterator *self) {
// const LookaheadIterator *iterator = (const LookaheadIterator *)self;
// return ts_language_symbol_name(iterator->language, iterator->symbol);
// }

142
parser/src/language.h
View file

@ -1,5 +1,5 @@
#ifndef TREE_SITTER_LANGUAGE_H_
#define TREE_SITTER_LANGUAGE_H_
#ifndef LANGUAGE_H
#define LANGUAGE_H
#include "./parser.h"
#include "me/types.h"
@ -9,136 +9,28 @@
#define LANGUAGE_VERSION_WITH_PRIMARY_STATES 14
#define LANGUAGE_VERSION_USABLE_VIA_WASM 13
typedef struct TableEntry
struct TableEntry
{
const TSParseAction *actions;
t_u32 action_count;
bool is_reusable;
} TableEntry;
};
typedef struct TableEntry TableEntry;
void ts_language_table_entry(const TSLanguage *, TSStateId, TSSymbol, TableEntry *);
TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *, TSSymbol);
TSSymbol ts_language_public_symbol(const TSLanguage *, TSSymbol);
TSStateId ts_language_next_state(const TSLanguage *self, TSStateId state, TSSymbol symbol);
bool ts_language_is_symbol_external(const TSLanguage *self, TSSymbol symbol);
const TSParseAction *ts_language_actions(const TSLanguage *self, TSStateId state, TSSymbol symbol, t_u32 *count);
bool ts_language_has_reduce_action(const TSLanguage *self, TSStateId state, TSSymbol symbol);
t_u16 ts_language_lookup(const TSLanguage *self, TSStateId state, TSSymbol symbol);
bool ts_language_has_actions(const TSLanguage *self, TSStateId state, TSSymbol symbol);
const bool *ts_language_enabled_external_tokens(const TSLanguage *self, t_u32 external_scanner_state);
const TSSymbol *ts_language_alias_sequence(const TSLanguage *self, t_u32 production_id);
TSSymbol ts_language_alias_at(const TSLanguage *self, t_u32 production_id, t_u32 child_index);
void ts_language_field_map(const TSLanguage *self, t_u32 production_id, const TSFieldMapEntry **start, const TSFieldMapEntry **end);
void ts_language_aliases_for_symbol(const TSLanguage *self, TSSymbol original_symbol, const TSSymbol **start, const TSSymbol **end);
static inline bool ts_language_is_symbol_external(const TSLanguage *self, TSSymbol symbol)
{
return 0 < symbol && symbol < self->external_token_count + 1;
}
static inline const TSParseAction *ts_language_actions(const TSLanguage *self, TSStateId state, TSSymbol symbol, t_u32 *count)
{
TableEntry entry;
ts_language_table_entry(self, state, symbol, &entry);
*count = entry.action_count;
return entry.actions;
}
static inline bool ts_language_has_reduce_action(const TSLanguage *self, TSStateId state, TSSymbol symbol)
{
TableEntry entry;
ts_language_table_entry(self, state, symbol, &entry);
return entry.action_count > 0 && entry.actions[0].type == TSParseActionTypeReduce;
}
// Lookup the table value for a given symbol and state.
//
// For non-terminal symbols, the table value represents a successor state.
// For terminal symbols, it represents an index in the actions table.
// For 'large' parse states, this is a direct lookup. For 'small' parse
// states, this requires searching through the symbol groups to find
// the given symbol.
static inline t_u16 ts_language_lookup(const TSLanguage *self, TSStateId state, TSSymbol symbol)
{
if (state >= self->large_state_count)
{
t_u32 index = self->small_parse_table_map[state - self->large_state_count];
const t_u16 *data = &self->small_parse_table[index];
t_u16 group_count = *(data++);
for (t_u32 i = 0; i < group_count; i++)
{
t_u16 section_value = *(data++);
t_u16 symbol_count = *(data++);
for (t_u32 j = 0; j < symbol_count; j++)
{
if (*(data++) == symbol)
return section_value;
}
}
return 0;
}
else
{
return self->parse_table[state * self->symbol_count + symbol];
}
}
static inline bool ts_language_has_actions(const TSLanguage *self, TSStateId state, TSSymbol symbol)
{
return ts_language_lookup(self, state, symbol) != 0;
}
static inline const bool *ts_language_enabled_external_tokens(const TSLanguage *self, t_u32 external_scanner_state)
{
if (external_scanner_state == 0)
{
return NULL;
}
else
{
return self->external_scanner.states + self->external_token_count * external_scanner_state;
}
}
static inline const TSSymbol *ts_language_alias_sequence(const TSLanguage *self, t_u32 production_id)
{
return production_id ? &self->alias_sequences[production_id * self->max_alias_sequence_length] : NULL;
}
static inline TSSymbol ts_language_alias_at(const TSLanguage *self, t_u32 production_id, t_u32 child_index)
{
return production_id ? self->alias_sequences[production_id * self->max_alias_sequence_length + child_index] : 0;
}
static inline void ts_language_field_map(const TSLanguage *self, t_u32 production_id, const TSFieldMapEntry **start,
const TSFieldMapEntry **end)
{
if (self->field_count == 0)
{
*start = NULL;
*end = NULL;
return;
}
TSFieldMapSlice slice = self->field_map_slices[production_id];
*start = &self->field_map_entries[slice.index];
*end = &self->field_map_entries[slice.index] + slice.length;
}
static inline void ts_language_aliases_for_symbol(const TSLanguage *self, TSSymbol original_symbol, const TSSymbol **start,
const TSSymbol **end)
{
*start = &self->public_symbol_map[original_symbol];
*end = *start + 1;
t_u32 idx = 0;
for (;;)
{
TSSymbol symbol = self->alias_map[idx++];
if (symbol == 0 || symbol > original_symbol)
break;
t_u16 count = self->alias_map[idx++];
if (symbol == original_symbol)
{
*start = &self->alias_map[idx];
*end = &self->alias_map[idx + count];
break;
}
idx += count;
}
}
#endif // TREE_SITTER_LANGUAGE_H_
#endif // LANGUAGE_H

43
parser/src/length.c Normal file
View file

@ -0,0 +1,43 @@
#include "./length.h"
#include "./point.h"
bool length_is_undefined(Length length)
{
return (length.bytes == 0 && length.extent.column != 0);
}
Length length_min(Length len1, Length len2)
{
return ((len1.bytes < len2.bytes) ? len1 : len2);
}
Length length_add(Length len1, Length len2)
{
Length result;
result.bytes = len1.bytes + len2.bytes;
result.extent = point_add(len1.extent, len2.extent);
return (result);
}
Length length_sub(Length len1, Length len2)
{
Length result;
result.bytes = len1.bytes - len2.bytes;
result.extent = point_sub(len1.extent, len2.extent);
return (result);
}
Length length_zero(void)
{
return ((Length){0, {0, 0}});
}
Length length_saturating_sub(Length len1, Length len2)
{
if (len1.bytes > len2.bytes)
return (length_sub(len1, len2));
else
return (length_zero());
}

61
parser/src/length.h
View file

@ -1,61 +1,26 @@
#ifndef TREE_SITTER_LENGTH_H_
#define TREE_SITTER_LENGTH_H_
#ifndef LENGTH_H
#define LENGTH_H
#include "./api.h"
#include "./point.h"
#include "me/types.h"
typedef struct Length
struct Length
{
t_u32 bytes;
TSPoint extent;
} Length;
};
typedef struct Length Length;
static const Length LENGTH_UNDEFINED = {0, {0, 1}};
static const Length LENGTH_MAX = {UINT32_MAX, {UINT32_MAX, UINT32_MAX}};
static inline bool length_is_undefined(Length length)
{
return length.bytes == 0 && length.extent.column != 0;
}
static inline Length length_min(Length len1, Length len2)
{
return (len1.bytes < len2.bytes) ? len1 : len2;
}
static inline Length length_add(Length len1, Length len2)
{
Length result;
result.bytes = len1.bytes + len2.bytes;
result.extent = point_add(len1.extent, len2.extent);
return result;
}
static inline Length length_sub(Length len1, Length len2)
{
Length result;
result.bytes = len1.bytes - len2.bytes;
result.extent = point_sub(len1.extent, len2.extent);
return result;
}
static inline Length length_zero(void)
{
Length result = {0, {0, 0}};
return result;
}
static inline Length length_saturating_sub(Length len1, Length len2)
{
if (len1.bytes > len2.bytes)
{
return length_sub(len1, len2);
}
else
{
return length_zero();
}
}
Length length_saturating_sub(Length len1, Length len2);
Length length_zero(void);
Length length_sub(Length len1, Length len2);
Length length_add(Length len1, Length len2);
Length length_min(Length len1, Length len2);
Length length_max(Length len1, Length len2);
bool length_is_undefined(Length length);
#endif

2
parser/src/lex.c
View file

@ -4147,7 +4147,7 @@ bool ts_lex(TSLexer *lexer, TSStateId state)
}
}
static bool ts_lex_keywords(TSLexer *lexer, TSStateId state)
bool ts_lex_keywords(TSLexer *lexer, TSStateId state)
{
START_LEXER();
eof = lexer->eof(lexer);

4
parser/src/lexer.c
View file

@ -1,6 +1,6 @@
#include "./lexer.h"
#include "./length.h"
#include "./unicode.h"
#include "./input.h"
#include "me/mem/mem.h"
#include "me/types.h"
#include <string.h>
@ -69,7 +69,7 @@ static void ts_lexer__get_lookahead(Lexer *self)
}
const t_u8 *chunk = (const t_u8 *)self->chunk + position_in_chunk;
UnicodeDecodeFunction decode = self->input.encoding == TSInputEncodingUTF8 ? ts_decode_utf8 : ts_decode_utf16;
UnicodeDecodeFunction decode = ts_decode_ascii;
self->lookahead_size = decode(chunk, size, &self->data.lookahead);

31
parser/src/lexer.h
View file

@ -1,42 +1,41 @@
#ifndef TREE_SITTER_LEXER_H_
#define TREE_SITTER_LEXER_H_
#ifndef LEXER_H
#define LEXER_H
#include "./api.h"
#include "./length.h"
#include "./parser.h"
#include "me/types.h"
typedef struct Lexer
struct Lexer
{
TSLexer data;
Length current_position;
Length token_start_position;
Length token_end_position;
TSRange *included_ranges;
const t_u8 *chunk;
TSInput input;
TSLogger logger;
t_u32 included_range_count;
t_u32 current_included_range_index;
t_u32 chunk_start;
t_u32 chunk_size;
t_u32 lookahead_size;
bool did_get_column;
t_u8 debug_buffer[TREE_SITTER_SERIALIZATION_BUFFER_SIZE];
} Lexer;
};
void ts_lexer_init(Lexer *);
void ts_lexer_delete(Lexer *);
void ts_lexer_set_input(Lexer *, TSInput);
void ts_lexer_reset(Lexer *, Length);
void ts_lexer_start(Lexer *);
void ts_lexer_finish(Lexer *, t_u32 *);
void ts_lexer_advance_to_end(Lexer *);
void ts_lexer_mark_end(Lexer *);
typedef struct Lexer Lexer;
void ts_lexer_init(Lexer *self);
void ts_lexer_delete(Lexer *self);
void ts_lexer_set_input(Lexer *self, TSInput input);
void ts_lexer_reset(Lexer *self, Length length);
void ts_lexer_start(Lexer *self);
void ts_lexer_finish(Lexer *self, t_u32 *lookahead);
void ts_lexer_advance_to_end(Lexer *self);
void ts_lexer_mark_end(Lexer *self);
bool ts_lexer_set_included_ranges(Lexer *self, const TSRange *ranges, t_u32 count);
TSRange *ts_lexer_included_ranges(const Lexer *self, t_u32 *count);
#endif // TREE_SITTER_LEXER_H_
#endif // LEXER_H

11
parser/src/lib.c
View file

@ -1,13 +1,16 @@
#define _POSIX_C_SOURCE 200112L
#include "./create_language.c"
#include "./input.c"
#include "./language.c"
#include "./length.c"
#include "./lex.c"
#include "./lexer.c"
#include "./node.c"
#include "./parser.c"
#include "./point.c"
#include "./reduce_action.c"
#include "./scanner.c"
#include "./stack.c"
#include "./subtree.c"
#include "./tree.c"
#include "./lex.c"
#include "./create_language.c"
#include "./scanner.c"

6
parser/src/parser.h
View file

@ -1,5 +1,5 @@
#ifndef TREE_SITTER_PARSER_H_
#define TREE_SITTER_PARSER_H_
#ifndef PARSER_H
#define PARSER_H
#include "me/types.h"
@ -267,4 +267,4 @@ start:
} \
}
#endif // TREE_SITTER_PARSER_H_
#endif // PARSER_H

63
parser/src/point.c Normal file
View file

@ -0,0 +1,63 @@
#include "./point.h"
TSPoint point__new(t_u32 row, t_u32 column)
{
return ((TSPoint){row, column});
}
TSPoint point_add(TSPoint a, TSPoint b)
{
if (b.row > 0)
return (point__new(a.row + b.row, b.column));
else
return (point__new(a.row, a.column + b.column));
}
TSPoint point_sub(TSPoint a, TSPoint b)
{
if (a.row > b.row)
return (point__new(a.row - b.row, a.column));
else
return (point__new(0, a.column - b.column));
}
bool point_lte(TSPoint a, TSPoint b)
{
return ((a.row < b.row) || (a.row == b.row && a.column <= b.column));
}
bool point_lt(TSPoint a, TSPoint b)
{
return ((a.row < b.row) || (a.row == b.row && a.column < b.column));
}
bool point_gt(TSPoint a, TSPoint b)
{
return ((a.row > b.row) || (a.row == b.row && a.column > b.column));
}
bool point_gte(TSPoint a, TSPoint b)
{
return ((a.row > b.row) || (a.row == b.row && a.column >= b.column));
}
bool point_eq(TSPoint a, TSPoint b)
{
return (a.row == b.row && a.column == b.column);
}
TSPoint point_min(TSPoint a, TSPoint b)
{
if (a.row < b.row || (a.row == b.row && a.column < b.column))
return (a);
else
return (b);
}
TSPoint point_max(TSPoint a, TSPoint b)
{
if (a.row > b.row || (a.row == b.row && a.column > b.column))
return (a);
else
return (b);
}

76
parser/src/point.h
View file

@ -1,5 +1,5 @@
#ifndef TREE_SITTER_POINT_H_
#define TREE_SITTER_POINT_H_
#ifndef POINT_H
#define POINT_H
#include "./api.h"
#include "me/types.h"
@ -7,67 +7,15 @@
#define POINT_ZERO ((TSPoint){0, 0})
#define POINT_MAX ((TSPoint){UINT32_MAX, UINT32_MAX})
static inline TSPoint point__new(t_u32 row, t_u32 column)
{
TSPoint result = {row, column};
return result;
}
static inline TSPoint point_add(TSPoint a, TSPoint b)
{
if (b.row > 0)
return point__new(a.row + b.row, b.column);
else
return point__new(a.row, a.column + b.column);
}
static inline TSPoint point_sub(TSPoint a, TSPoint b)
{
if (a.row > b.row)
return point__new(a.row - b.row, a.column);
else
return point__new(0, a.column - b.column);
}
static inline bool point_lte(TSPoint a, TSPoint b)
{
return (a.row < b.row) || (a.row == b.row && a.column <= b.column);
}
static inline bool point_lt(TSPoint a, TSPoint b)
{
return (a.row < b.row) || (a.row == b.row && a.column < b.column);
}
static inline bool point_gt(TSPoint a, TSPoint b)
{
return (a.row > b.row) || (a.row == b.row && a.column > b.column);
}
static inline bool point_gte(TSPoint a, TSPoint b)
{
return (a.row > b.row) || (a.row == b.row && a.column >= b.column);
}
static inline bool point_eq(TSPoint a, TSPoint b)
{
return a.row == b.row && a.column == b.column;
}
static inline TSPoint point_min(TSPoint a, TSPoint b)
{
if (a.row < b.row || (a.row == b.row && a.column < b.column))
return a;
else
return b;
}
static inline TSPoint point_max(TSPoint a, TSPoint b)
{
if (a.row > b.row || (a.row == b.row && a.column > b.column))
return a;
else
return b;
}
TSPoint point_max(TSPoint a, TSPoint b);
TSPoint point_min(TSPoint a, TSPoint b);
TSPoint point__new(t_u32 row, t_u32 column);
TSPoint point_add(TSPoint a, TSPoint b);
TSPoint point_sub(TSPoint a, TSPoint b);
bool point_lte(TSPoint a, TSPoint b);
bool point_lt(TSPoint a, TSPoint b);
bool point_gte(TSPoint a, TSPoint b);
bool point_gt(TSPoint a, TSPoint b);
bool point_eq(TSPoint a, TSPoint b);
#endif

17
parser/src/reduce_action.c Normal file
View file

@ -0,0 +1,17 @@
#include "./reduce_action.h"
void ts_reduce_action_set_add(ReduceActionSet *self, ReduceAction new_action)
{
ReduceAction action;
t_u32 i;
i = 0;
while (i < self->size)
{
action = self->contents[i];
if (action.symbol == new_action.symbol && action.count == new_action.count)
return;
i++;
}
array_push(self, new_action);
}

22
parser/src/reduce_action.h
View file

@ -1,29 +1,21 @@
#ifndef TREE_SITTER_REDUCE_ACTION_H_
#define TREE_SITTER_REDUCE_ACTION_H_
#ifndef REDUCE_ACTION_H
#define REDUCE_ACTION_H
#include "./api.h"
#include "./array.h"
#include "me/types.h"
typedef struct ReduceAction
struct ReduceAction
{
t_u32 count;
TSSymbol symbol;
int dynamic_precedence;
t_u16 production_id;
} ReduceAction;
};
typedef struct ReduceAction ReduceAction;
typedef Array(ReduceAction) ReduceActionSet;
static inline void ts_reduce_action_set_add(ReduceActionSet *self, ReduceAction new_action)
{
for (t_u32 i = 0; i < self->size; i++)
{
ReduceAction action = self->contents[i];
if (action.symbol == new_action.symbol && action.count == new_action.count)
return;
}
array_push(self, new_action);
}
void ts_reduce_action_set_add(ReduceActionSet *self, ReduceAction new_action);
#endif // TREE_SITTER_REDUCE_ACTION_H_
#endif // REDUCE_ACTION_H

11
parser/src/stack.h
View file

@ -1,14 +1,13 @@
#ifndef TREE_SITTER_PARSE_STACK_H_
#define TREE_SITTER_PARSE_STACK_H_
#ifndef PARSE_STACK_H
#define PARSE_STACK_H
#include "me/types.h"
#include "./array.h"
#include "./subtree.h"
#include "me/types.h"
typedef struct Stack Stack;
typedef t_u32 StackVersion;
#define STACK_VERSION_NONE ((StackVersion)-1)
typedef struct StackSlice
@ -123,4 +122,4 @@ void ts_stack_remove_version(Stack *, StackVersion);
void ts_stack_clear(Stack *);
#endif // TREE_SITTER_PARSE_STACK_H_
#endif // PARSE_STACK_H

198
parser/src/subtree.h
View file

@ -1,12 +1,11 @@
#ifndef TREE_SITTER_SUBTREE_H_
#define TREE_SITTER_SUBTREE_H_
#ifndef SUBTREE_H
#define SUBTREE_H
#include "me/types.h"
#include "./api.h"
#include "./array.h"
#include "./length.h"
#include "./parser.h"
#include "api.h"
#include "me/types.h"
#include <limits.h>
#include <stdbool.h>
#include <stdio.h>
@ -16,6 +15,15 @@
#define TS_TREE_STATE_NONE USHRT_MAX
#define NULL_SUBTREE ((Subtree){.ptr = NULL})
struct ExternalScannerState
{
union {
char *long_data;
char short_data[24];
};
t_u32 length;
};
// The serialized state of an external scanner.
//
// Every time an external token subtree is created after a call to an
@ -26,14 +34,7 @@
//
// Small byte arrays are stored inline, and long ones are allocated
// separately on the heap.
typedef struct ExternalScannerState
{
union {
char *long_data;
char short_data[24];
};
t_u32 length;
} ExternalScannerState;
typedef struct ExternalScannerState ExternalScannerState;
// A compact representation of a subtree.
//
@ -48,63 +49,24 @@ typedef struct ExternalScannerState
// the pointer or the inline struct.
typedef struct SubtreeInlineData SubtreeInlineData;
#define SUBTREE_BITS \
bool visible : 1; \
bool named : 1; \
bool extra : 1; \
bool has_changes : 1; \
bool is_missing : 1; \
struct SubtreeInlineData
{
bool is_inline : 1;
bool visible : 1;
bool named : 1;
bool extra : 1;
bool has_changes : 1;
bool is_missing : 1;
bool is_keyword : 1;
#define SUBTREE_SIZE \
t_u8 padding_columns; \
t_u8 padding_rows : 4; \
t_u8 lookahead_bytes : 4; \
t_u8 padding_bytes; \
t_u8 symbol;
t_u16 parse_state;
t_u8 padding_columns;
t_u8 padding_rows : 4;
t_u8 lookahead_bytes : 4;
t_u8 padding_bytes;
t_u8 size_bytes;
#if TS_BIG_ENDIAN
# if TS_PTR_SIZE == 32
struct SubtreeInlineData
{
t_u16 parse_state;
t_u8 symbol;
SUBTREE_BITS
bool unused : 1;
bool is_inline : 1;
SUBTREE_SIZE
};
# else
struct SubtreeInlineData
{
SUBTREE_SIZE
t_u16 parse_state;
t_u8 symbol;
SUBTREE_BITS
bool unused : 1;
bool is_inline : 1;
};
# endif
#else
struct SubtreeInlineData
{
bool is_inline : 1;
SUBTREE_BITS
t_u8 symbol;
t_u16 parse_state;
SUBTREE_SIZE
};
#endif
#undef SUBTREE_BITS
#undef SUBTREE_SIZE
// A heap-allocated representation of a subtree.
//
// This representation is used for parent nodes, external tokens,
@ -213,47 +175,43 @@ Subtree ts_subtree_last_external_token(Subtree);
const ExternalScannerState *ts_subtree_external_scanner_state(Subtree self);
bool ts_subtree_external_scanner_state_eq(Subtree, Subtree);
#define SUBTREE_GET(self, name) ((self).data.is_inline ? (self).data.name : (self).ptr->name)
static inline TSSymbol ts_subtree_symbol(Subtree self)
{
return SUBTREE_GET(self, symbol);
return ((self).data.is_inline ? (self).data.symbol : (self).ptr->symbol);
}
static inline bool ts_subtree_visible(Subtree self)
{
return SUBTREE_GET(self, visible);
return ((self).data.is_inline ? (self).data.visible : (self).ptr->visible);
}
static inline bool ts_subtree_named(Subtree self)
{
return SUBTREE_GET(self, named);
return ((self).data.is_inline ? (self).data.named : (self).ptr->named);
}
static inline bool ts_subtree_extra(Subtree self)
{
return SUBTREE_GET(self, extra);
return ((self).data.is_inline ? (self).data.extra : (self).ptr->extra);
}
static inline bool ts_subtree_has_changes(Subtree self)
{
return SUBTREE_GET(self, has_changes);
return ((self).data.is_inline ? (self).data.has_changes : (self).ptr->has_changes);
}
static inline bool ts_subtree_missing(Subtree self)
{
return SUBTREE_GET(self, is_missing);
return ((self).data.is_inline ? (self).data.is_missing : (self).ptr->is_missing);
}
static inline bool ts_subtree_is_keyword(Subtree self)
{
return SUBTREE_GET(self, is_keyword);
return ((self).data.is_inline ? (self).data.is_keyword : (self).ptr->is_keyword);
}
static inline TSStateId ts_subtree_parse_state(Subtree self)
{
return SUBTREE_GET(self, parse_state);
return ((self).data.is_inline ? (self).data.parse_state : (self).ptr->parse_state);
}
static inline t_u32 ts_subtree_lookahead_bytes(Subtree self)
{
return SUBTREE_GET(self, lookahead_bytes);
return ((self).data.is_inline ? (self).data.lookahead_bytes : (self).ptr->lookahead_bytes);
}
#undef SUBTREE_GET
// Get the size needed to store a heap-allocated subtree with the given
// number of children.
static inline size_t ts_subtree_alloc_size(t_u32 child_count)
@ -268,14 +226,10 @@ static inline size_t ts_subtree_alloc_size(t_u32 child_count)
static inline void ts_subtree_set_extra(MutableSubtree *self, bool is_extra)
{
if (self->data.is_inline)
{
self->data.extra = is_extra;
}
else
{
self->ptr->extra = is_extra;
}
}
static inline TSSymbol ts_subtree_leaf_symbol(Subtree self)
{
@ -298,156 +252,132 @@ static inline TSStateId ts_subtree_leaf_parse_state(Subtree self)
static inline Length ts_subtree_padding(Subtree self)
{
if (self.data.is_inline)
{
Length result = {self.data.padding_bytes, {self.data.padding_rows, self.data.padding_columns}};
return result;
}
return ((Length){self.data.padding_bytes, {self.data.padding_rows, self.data.padding_columns}});
else
{
return self.ptr->padding;
}
}
static inline Length ts_subtree_size(Subtree self)
{
if (self.data.is_inline)
{
Length result = {self.data.size_bytes, {0, self.data.size_bytes}};
return result;
}
return ((Length){self.data.size_bytes, {0, self.data.size_bytes}});
else
{
return self.ptr->size;
}
}
static inline Length ts_subtree_total_size(Subtree self)
{
return length_add(ts_subtree_padding(self), ts_subtree_size(self));
return (length_add(ts_subtree_padding(self), ts_subtree_size(self)));
}
static inline t_u32 ts_subtree_total_bytes(Subtree self)
{
return ts_subtree_total_size(self).bytes;
return (ts_subtree_total_size(self).bytes);
}
static inline t_u32 ts_subtree_child_count(Subtree self)
{
return self.data.is_inline ? 0 : self.ptr->child_count;
return (self.data.is_inline ? 0 : self.ptr->child_count);
}
static inline t_u32 ts_subtree_repeat_depth(Subtree self)
{
return self.data.is_inline ? 0 : self.ptr->repeat_depth;
return (self.data.is_inline ? 0 : self.ptr->repeat_depth);
}
static inline t_u32 ts_subtree_is_repetition(Subtree self)
{
return self.data.is_inline ? 0 : !self.ptr->named && !self.ptr->visible && self.ptr->child_count != 0;
return (self.data.is_inline ? 0 : !self.ptr->named && !self.ptr->visible && self.ptr->child_count != 0);
}
static inline t_u32 ts_subtree_visible_descendant_count(Subtree self)
{
return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->visible_descendant_count;
return ((self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->visible_descendant_count);
}
static inline t_u32 ts_subtree_visible_child_count(Subtree self)
{
if (ts_subtree_child_count(self) > 0)
{
return self.ptr->visible_child_count;
}
return (self.ptr->visible_child_count);
else
{
return 0;
}
}
static inline t_u32 ts_subtree_error_cost(Subtree self)
{
if (ts_subtree_missing(self))
{
return ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY;
}
return (ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY);
else
{
return self.data.is_inline ? 0 : self.ptr->error_cost;
}
return (self.data.is_inline ? 0 : self.ptr->error_cost);
}
static inline t_i32 ts_subtree_dynamic_precedence(Subtree self)
{
return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->dynamic_precedence;
return ((self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->dynamic_precedence);
}
static inline t_u16 ts_subtree_production_id(Subtree self)
{
if (ts_subtree_child_count(self) > 0)
{
return self.ptr->production_id;
}
return (self.ptr->production_id);
else
{
return 0;
}
return (0);
}
static inline bool ts_subtree_fragile_left(Subtree self)
{
return self.data.is_inline ? false : self.ptr->fragile_left;
return (self.data.is_inline ? false : self.ptr->fragile_left);
}
static inline bool ts_subtree_fragile_right(Subtree self)
{
return self.data.is_inline ? false : self.ptr->fragile_right;
return (self.data.is_inline ? false : self.ptr->fragile_right);
}
static inline bool ts_subtree_has_external_tokens(Subtree self)
{
return self.data.is_inline ? false : self.ptr->has_external_tokens;
return (self.data.is_inline ? false : self.ptr->has_external_tokens);
}
static inline bool ts_subtree_has_external_scanner_state_change(Subtree self)
{
return self.data.is_inline ? false : self.ptr->has_external_scanner_state_change;
return (self.data.is_inline ? false : self.ptr->has_external_scanner_state_change);
}
static inline bool ts_subtree_depends_on_column(Subtree self)
{
return self.data.is_inline ? false : self.ptr->depends_on_column;
return (self.data.is_inline ? false : self.ptr->depends_on_column);
}
static inline bool ts_subtree_is_fragile(Subtree self)
{
return self.data.is_inline ? false : (self.ptr->fragile_left || self.ptr->fragile_right);
return (self.data.is_inline ? false : (self.ptr->fragile_left || self.ptr->fragile_right));
}
static inline bool ts_subtree_is_error(Subtree self)
{
return ts_subtree_symbol(self) == ts_builtin_sym_error;
return (ts_subtree_symbol(self) == ts_builtin_sym_error);
}
static inline bool ts_subtree_is_eof(Subtree self)
{
return ts_subtree_symbol(self) == ts_builtin_sym_end;
return (ts_subtree_symbol(self) == ts_builtin_sym_end);
}
static inline Subtree ts_subtree_from_mut(MutableSubtree self)
{
Subtree result;
result.data = self.data;
return result;
return (result);
}
static inline MutableSubtree ts_subtree_to_mut_unsafe(Subtree self)
{
MutableSubtree result;
result.data = self.data;
return result;
return (result);
}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_SUBTREE_H_
#endif // SUBTREE_H

27
parser/src/tree.h
View file

@ -1,17 +1,12 @@
#ifndef TREE_SITTER_TREE_H_
#define TREE_SITTER_TREE_H_
#ifndef TREE_H
#define TREE_H
#include "me/types.h"
#include "./subtree.h"
#include "me/types.h"
typedef struct ParentCacheEntry
{
const Subtree *child;
const Subtree *parent;
Length position;
TSSymbol alias_symbol;
} ParentCacheEntry;
typedef struct ParentCacheEntry ParentCacheEntry;
struct TSTree
{
@ -21,7 +16,15 @@ struct TSTree
t_u32 included_range_count;
};
struct ParentCacheEntry
{
const Subtree *child;
const Subtree *parent;
Length position;
TSSymbol alias_symbol;
};
TSTree *ts_tree_new(Subtree root, const TSLanguage *language, const TSRange *, t_u32);
TSNode ts_node_new(const TSTree *, const Subtree *, Length, TSSymbol);
#endif // TREE_SITTER_TREE_H_
#endif // TREE_H

36
parser/src/unicode.h
View file

@ -1,36 +0,0 @@
#ifndef TREE_SITTER_UNICODE_H_
#define TREE_SITTER_UNICODE_H_
#include "me/types.h"
static const t_i32 TS_DECODE_ERROR = -1;
// These functions read one unicode code point from the given string,
// returning the number of bytes consumed.
typedef t_u32 (*UnicodeDecodeFunction)(const t_u8 *string, t_u32 length, t_i32 *code_point);
static inline t_u32 ts_decode_ascii(const t_u8 *string, t_u32 length, t_i32 *code_point)
{
(void)(length);
*code_point = 0;
*(t_u8 *)code_point = *string;
return (1);
}
static inline t_u32 ts_decode_utf8(const t_u8 *string, t_u32 length, t_i32 *code_point)
{
return (ts_decode_ascii(string, length, code_point));
// t_u32 i = 0;
// U8_NEXT(string, i, length, *code_point);
// return i;
}
static inline t_u32 ts_decode_utf16(const t_u8 *string, t_u32 length, t_i32 *code_point)
{
return (ts_decode_ascii(string, length, code_point));
// t_u32 i = 0;
// U16_NEXT(((t_u16 *)string), i, length, *code_point);
// return i * 2;
}
#endif // TREE_SITTER_UNICODE_H_