From acb734cda0b23e217749cb2b88eb08047601414d Mon Sep 17 00:00:00 2001 From: Maieul BOYER Date: Fri, 30 Aug 2024 19:57:14 +0200 Subject: [PATCH] Update: WIP trash removal --- parser/include/parser/api.h | 52 +---------- parser/include/parser/parser.h | 1 - parser/src/lexer.c | 158 ++++++++------------------------- 3 files changed, 36 insertions(+), 175 deletions(-) diff --git a/parser/include/parser/api.h b/parser/include/parser/api.h index 2d217cca..1715a093 100644 --- a/parser/include/parser/api.h +++ b/parser/include/parser/api.h @@ -6,7 +6,7 @@ /* By: maiboyer +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/07/22 13:54:54 by maiboyer #+# #+# */ -/* Updated: 2024/08/22 15:22:06 by maiboyer ### ########.fr */ +/* Updated: 2024/08/30 19:46:23 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -317,56 +317,6 @@ TSTree *ts_parser_parse_string_encoding(TSParser *self, t_const_str string, t_u3 */ void ts_parser_reset(TSParser *self); -/** - * Set the maximum duration in microseconds that parsing should be allowed to - * take before halting. - * - * If parsing takes longer than this, it will halt early, returning NULL. - * See [`ts_parser_parse`] for more information. - */ -void ts_parser_set_timeout_micros(TSParser *self, t_u64 timeout_micros); - -/** - * Get the duration in microseconds that parsing is allowed to take. - */ -t_u64 ts_parser_timeout_micros(const TSParser *self); - -/** - * Set the parser's current cancellation flag pointer. - * - * If a non-null pointer is assigned, then the parser will periodically read - * from this pointer during parsing. If it reads a non-zero value, it will - * halt early, returning NULL. See [`ts_parser_parse`] for more information. - */ -void ts_parser_set_cancellation_flag(TSParser *self, const size_t *flag); - -/** - * Get the parser's current cancellation flag pointer. - */ -const size_t *ts_parser_cancellation_flag(const TSParser *self); - -/** - * Set the logger that a parser should use during parsing. - * - * The parser does not take ownership over the logger payload. If a logger was - * previously assigned, the caller is responsible for releasing any memory - * owned by the previous logger. - */ -void ts_parser_set_logger(TSParser *self, TSLogger logger); - -/** - * Get the parser's current logger. - */ -TSLogger ts_parser_logger(const TSParser *self); - -/** - * Set the file descriptor to which the parser should write debugging graphs - * during parsing. The graphs are formatted in the DOT language. You may want - * to pipe these graphs directly to a `dot(1)` process in order to generate - * SVG output. You can turn off this logging by passing a negative number. - */ -void ts_parser_print_dot_graphs(TSParser *self, int fd); - /******************/ /* Section - Tree */ /******************/ diff --git a/parser/include/parser/parser.h b/parser/include/parser/parser.h index 67bf3f55..59fa421e 100644 --- a/parser/include/parser/parser.h +++ b/parser/include/parser/parser.h @@ -43,7 +43,6 @@ struct TSLexer void (*advance)(TSLexer *, bool); void (*mark_end)(TSLexer *); t_u32 (*get_column)(TSLexer *); - bool (*is_at_included_range_start)(const TSLexer *); bool (*eof)(const TSLexer *); }; diff --git a/parser/src/lexer.c b/parser/src/lexer.c index 0df9c19e..df7b0564 100644 --- a/parser/src/lexer.c +++ b/parser/src/lexer.c @@ -3,32 +3,20 @@ #include "me/types.h" #include "parser/input.h" #include "parser/length.h" +#include #include -#define LOG(...) - -static const t_i32 BYTE_ORDER_MARK = 0xFEFF; - -static const TSRange DEFAULT_RANGE = {.start_point = - { - .row = 0, - .column = 0, - }, - .end_point = - { - .row = UINT32_MAX, - .column = UINT32_MAX, - }, - .start_byte = 0, - .end_byte = UINT32_MAX}; +#define BYTE_ORDER_MARK 0xFEFF // Check if the lexer has reached EOF. This state is stored // by setting the lexer's `current_included_range_index` such that // it has consumed all of its available ranges. static bool ts_lexer__eof(const TSLexer *_self) { - Lexer *self = (Lexer *)_self; - return self->current_included_range_index == self->included_range_count; + Lexer *self; + + self = (Lexer *)_self; + return (self->current_included_range_index == self->included_range_count); } // Clear the currently stored chunk of source code, because the lexer's @@ -58,46 +46,35 @@ static void ts_lexer__get_chunk(Lexer *self) // code that spans the current position. static void ts_lexer__get_lookahead(Lexer *self) { - t_u32 position_in_chunk = self->current_position.bytes - self->chunk_start; - t_u32 size = self->chunk_size - position_in_chunk; + t_u32 position_in_chunk; + t_u32 size; + const t_u8 *chunk; + position_in_chunk = self->current_position.bytes - self->chunk_start; + size = self->chunk_size - position_in_chunk; if (size == 0) { self->lookahead_size = 1; self->data.lookahead = '\0'; return; } - - const t_u8 *chunk = (const t_u8 *)self->chunk + position_in_chunk; - UnicodeDecodeFunction decode = ts_decode_ascii; - - self->lookahead_size = decode(chunk, size, &self->data.lookahead); - - // If this chunk ended in the middle of a multi-byte character, - // try again with a fresh chunk. - if (self->data.lookahead == TS_DECODE_ERROR && size < 4) - { - ts_lexer__get_chunk(self); - chunk = (const t_u8 *)self->chunk; - size = self->chunk_size; - self->lookahead_size = decode(chunk, size, &self->data.lookahead); - } - + chunk = (const t_u8 *)self->chunk + position_in_chunk; + self->lookahead_size = ts_decode_ascii(chunk, size, &self->data.lookahead); if (self->data.lookahead == TS_DECODE_ERROR) - { self->lookahead_size = 1; - } } static void ts_lexer_goto(Lexer *self, Length position) { - self->current_position = position; + bool found_included_range; + TSRange *included_range; + TSRange *last_included_range; - // Move to the first valid position at or after the given position. - bool found_included_range = false; + found_included_range = false; + self->current_position = position; for (t_u32 i = 0; i < self->included_range_count; i++) { - TSRange *included_range = &self->included_ranges[i]; + included_range = &self->included_ranges[i]; if (included_range->end_byte > self->current_position.bytes && included_range->end_byte > included_range->start_byte) { if (included_range->start_byte >= self->current_position.bytes) @@ -113,7 +90,6 @@ static void ts_lexer_goto(Lexer *self, Length position) break; } } - if (found_included_range) { // If the current position is outside of the current chunk of text, @@ -127,13 +103,12 @@ static void ts_lexer_goto(Lexer *self, Length position) self->lookahead_size = 0; self->data.lookahead = '\0'; } - // If the given position is beyond any of included ranges, move to the EOF // state - past the end of the included ranges. else { self->current_included_range_index = self->included_range_count; - TSRange *last_included_range = &self->included_ranges[self->included_range_count - 1]; + last_included_range = &self->included_ranges[self->included_range_count - 1]; self->current_position = (Length){ .bytes = last_included_range->end_byte, .extent = last_included_range->end_point, @@ -209,16 +184,6 @@ static void ts_lexer__advance(TSLexer *_self, bool skip) Lexer *self = (Lexer *)_self; if (!self->chunk) return; - - if (skip) - { - LOG("skip", self->data.lookahead) - } - else - { - LOG("consume", self->data.lookahead) - } - ts_lexer__do_advance(self, skip); } @@ -293,19 +258,27 @@ static bool ts_lexer__is_at_included_range_start(const TSLexer *_self) return false; } } +static const TSRange DEFAULT_RANGE = {.start_point = + { + .row = 0, + .column = 0, + }, + .end_point = + { + .row = UINT32_MAX, + .column = UINT32_MAX, + }, + .start_byte = 0, + .end_byte = UINT32_MAX}; void ts_lexer_init(Lexer *self) { *self = (Lexer){ .data = { - // The lexer's methods are stored as struct fields so that generated - // parsers can call them without needing to be linked against this - // library. .advance = ts_lexer__advance, .mark_end = ts_lexer__mark_end, .get_column = ts_lexer__get_column, - .is_at_included_range_start = ts_lexer__is_at_included_range_start, .eof = ts_lexer__eof, .lookahead = 0, .result_symbol = 0, @@ -315,16 +288,10 @@ void ts_lexer_init(Lexer *self) .chunk_start = 0, .current_position = {0, {0, 0}}, .logger = {.payload = NULL, .log = NULL}, - .included_ranges = NULL, - .included_range_count = 0, + .included_ranges = (void *)&DEFAULT_RANGE, + .included_range_count = 1, .current_included_range_index = 0, }; - ts_lexer_set_included_ranges(self, NULL, 0); -} - -void ts_lexer_delete(Lexer *self) -{ - mem_free(self->included_ranges); } void ts_lexer_set_input(Lexer *self, TSInput input) @@ -339,9 +306,7 @@ void ts_lexer_set_input(Lexer *self, TSInput input) void ts_lexer_reset(Lexer *self, Length position) { if (position.bytes != self->current_position.bytes) - { ts_lexer_goto(self, position); - } } void ts_lexer_start(Lexer *self) @@ -364,10 +329,8 @@ void ts_lexer_start(Lexer *self) void ts_lexer_finish(Lexer *self, t_u32 *lookahead_end_byte) { if (length_is_undefined(self->token_end_position)) - { ts_lexer__mark_end(&self->data); - } - + (void)(lookahead_end_byte); // If the token ended at an included range boundary, then its end position // will have been reset to the end of the preceding range. Reset the start // position to match. @@ -375,66 +338,15 @@ void ts_lexer_finish(Lexer *self, t_u32 *lookahead_end_byte) { self->token_start_position = self->token_end_position; } - - t_u32 current_lookahead_end_byte = self->current_position.bytes + 1; - - // In order to determine that a byte sequence is invalid UTF8 or UTF16, - // the character decoding algorithm may have looked at the following byte. - // Therefore, the next byte *after* the current (invalid) character - // affects the interpretation of the current character. - if (self->data.lookahead == TS_DECODE_ERROR) - { - current_lookahead_end_byte += 4; // the maximum number of bytes read to identify an invalid code point - } - - if (current_lookahead_end_byte > *lookahead_end_byte) - { - *lookahead_end_byte = current_lookahead_end_byte; - } } void ts_lexer_advance_to_end(Lexer *self) { while (self->chunk) - { ts_lexer__advance(&self->data, false); - } } void ts_lexer_mark_end(Lexer *self) { ts_lexer__mark_end(&self->data); } - -bool ts_lexer_set_included_ranges(Lexer *self, const TSRange *ranges, t_u32 count) -{ - if (count == 0 || !ranges) - { - ranges = &DEFAULT_RANGE; - count = 1; - } - else - { - t_u32 previous_byte = 0; - for (t_u32 i = 0; i < count; i++) - { - const TSRange *range = &ranges[i]; - if (range->start_byte < previous_byte || range->end_byte < range->start_byte) - return false; - previous_byte = range->end_byte; - } - } - - size_t size = count * sizeof(TSRange); - self->included_ranges = mem_realloc(self->included_ranges, size); - mem_copy(self->included_ranges, ranges, size); - self->included_range_count = count; - ts_lexer_goto(self, self->current_position); - return true; -} - -TSRange *ts_lexer_included_ranges(const Lexer *self, t_u32 *count) -{ - *count = self->included_range_count; - return self->included_ranges; -}