From 4f1a08cbe6f3b700451da26b5078560f4187460f Mon Sep 17 00:00:00 2001 From: maix0 Date: Thu, 3 Oct 2024 21:38:27 +0200 Subject: [PATCH] update: made some small fixes --- parser/Filelist.parser.mk | 2 ++ parser/include/parser/passes.h | 3 ++- parser/include/parser/token.h | 4 ++-- parser/src/passes.c | 6 ++++-- parser/src/passes/template_file.c | 13 ++++++++++++- parser/src/tokenizer.c | 24 ++++++++++++++++++------ sources/_helper_main.c | 3 +-- sources/main.c | 9 ++++++--- 8 files changed, 47 insertions(+), 17 deletions(-) diff --git a/parser/Filelist.parser.mk b/parser/Filelist.parser.mk index d22139eb..a89efddc 100644 --- a/parser/Filelist.parser.mk +++ b/parser/Filelist.parser.mk @@ -1,4 +1,6 @@ SRC_FILES = \ +passes \ +passes/template_file \ token_lifetime \ tokenizer \ diff --git a/parser/include/parser/passes.h b/parser/include/parser/passes.h index b13b81c6..cf40b6f1 100644 --- a/parser/include/parser/passes.h +++ b/parser/include/parser/passes.h @@ -6,7 +6,7 @@ /* By: maiboyer +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/10/02 18:43:41 by maiboyer #+# #+# */ -/* Updated: 2024/10/02 19:13:19 by maiboyer ### ########.fr */ +/* Updated: 2024/10/03 21:32:32 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -25,6 +25,7 @@ struct s_ts_pass_def t_const_str name; }; +t_error ts_apply_passes(t_vec_token ts, t_vec_token *out); // list passes function here diff --git a/parser/include/parser/token.h b/parser/include/parser/token.h index f073c71d..b49b186a 100644 --- a/parser/include/parser/token.h +++ b/parser/include/parser/token.h @@ -6,7 +6,7 @@ /* By: maiboyer +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/09/26 17:59:23 by maiboyer #+# #+# */ -/* Updated: 2024/10/02 19:12:26 by maiboyer ### ########.fr */ +/* Updated: 2024/10/03 21:33:14 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -58,6 +58,6 @@ bool token_is_meta(t_token tok); t_token token_clone(t_token *tok); /* PARSING */ -t_error tokeniser(t_const_str raw); +t_error tokenize(t_const_str s, t_vec_token *out); #endif diff --git a/parser/src/passes.c b/parser/src/passes.c index 057eac95..550ebf18 100644 --- a/parser/src/passes.c +++ b/parser/src/passes.c @@ -6,7 +6,7 @@ /* By: maiboyer +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/10/02 18:41:16 by maiboyer #+# #+# */ -/* Updated: 2024/10/02 19:22:37 by maiboyer ### ########.fr */ +/* Updated: 2024/10/03 21:32:14 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -46,7 +46,7 @@ static const struct s_ts_pass_def g_ts_passes[] = { {do_fuck_all, "does nothing lol"}, }; -t_error apply_all_passes(t_vec_token ts, t_vec_token *out) +t_error ts_apply_passes(t_vec_token ts, t_vec_token *out) { t_usize i; t_vec_token next; @@ -58,6 +58,8 @@ t_error apply_all_passes(t_vec_token ts, t_vec_token *out) return (vec_token_free(ts), ERROR); if ((g_ts_passes[i].fn)(ts, &next)) return (me_eprintf("failed on %s token pass\n", g_ts_passes[i].name), ERROR); + else + me_printf("Applied '%s' pass\n", g_ts_passes[i].name); ts = next; i++; } diff --git a/parser/src/passes/template_file.c b/parser/src/passes/template_file.c index f45f5313..430366b5 100644 --- a/parser/src/passes/template_file.c +++ b/parser/src/passes/template_file.c @@ -6,7 +6,7 @@ /* By: maiboyer +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/10/02 19:04:32 by maiboyer #+# #+# */ -/* Updated: 2024/10/02 19:13:31 by maiboyer ### ########.fr */ +/* Updated: 2024/10/03 21:37:04 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -15,11 +15,22 @@ #include "me/vec/vec_token.h" #include "parser/token.h" +/// This is a sample pass +/// +/// There is a few rules the rest of the tokenizer machinery assumes +/// theses function follows: +/// - the input vec WILL be freed when the function return, even in +/// case of error +/// - the output vector isn't populated if the function returns an error, +/// thus it shouldn't be freed in case of error +/// - the output tokens may not be direct copy of the input tokens, +/// but need to be cloned (different allocations for stuff) t_error do_fuck_all(t_vec_token input, t_vec_token *output) { t_vec_token out; t_usize i; + i = 0; out = vec_token_new(input.len, token_free); while (i < input.len) { diff --git a/parser/src/tokenizer.c b/parser/src/tokenizer.c index c8d81331..2356f679 100644 --- a/parser/src/tokenizer.c +++ b/parser/src/tokenizer.c @@ -6,7 +6,7 @@ /* By: maiboyer +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/09/30 19:39:39 by maiboyer #+# #+# */ -/* Updated: 2024/10/02 18:02:59 by maiboyer ### ########.fr */ +/* Updated: 2024/10/03 21:31:57 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -28,8 +28,16 @@ static void push_token_and_create_new(\ string_push(&tmp.string, s); vec_token_push(tokens, tmp); } +static void push_token_and_set_new(\ + t_vec_token *tokens, t_token *tok, enum e_token ttype, t_const_str s) +{ + if (tok->type != TOK_NONE) + vec_token_push(tokens, *tok); + *tok = token_new(ttype); + string_push(&tok->string, s); +} -void handle_quote(t_vec_token *ret, char chr, t_token *tok, char *quote) +static void handle_quote(t_vec_token *ret, char chr, t_token *tok, char *quote) { if (chr == *quote) { @@ -42,7 +50,7 @@ void handle_quote(t_vec_token *ret, char chr, t_token *tok, char *quote) string_push_char(&tok->string, chr); } -void handle_noquote(t_vec_token *ret, char chr, t_token *tok, char *quote) +static void handle_noquote(t_vec_token *ret, char chr, t_token *tok, char *quote) { *quote = '\0'; if (chr == '$') @@ -71,15 +79,15 @@ void handle_noquote(t_vec_token *ret, char chr, t_token *tok, char *quote) } } -void tokenize_inner(t_vec_token *ret, char chr, t_token *tok, char *quote) +static void tokenize_inner(t_vec_token *ret, char chr, t_token *tok, char *quote) { if (*quote == '\0') { *quote = chr; if (chr == '\"') - push_token_and_create_new(ret, tok, TOK_DQUOTE, ""); + push_token_and_set_new(ret, tok, TOK_DQUOTE, ""); else if (chr == '\'') - push_token_and_create_new(ret, tok, TOK_SQUOTE, ""); + push_token_and_set_new(ret, tok, TOK_SQUOTE, ""); else handle_noquote(ret, chr, tok, quote); } @@ -89,6 +97,10 @@ void tokenize_inner(t_vec_token *ret, char chr, t_token *tok, char *quote) me_abort("invalid quote type"); } +// This should even be wrapped with the passes function so the consumer only +// see the last version of the tokenstream +// +// currently it is "Public" API though t_error tokenize(t_const_str s, t_vec_token *out) { t_usize i; diff --git a/sources/_helper_main.c b/sources/_helper_main.c index c5192216..2d005d85 100644 --- a/sources/_helper_main.c +++ b/sources/_helper_main.c @@ -6,7 +6,7 @@ /* By: rparodi +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/09/06 16:31:41 by rparodi #+# #+# */ -/* Updated: 2024/09/30 20:06:27 by maiboyer ### ########.fr */ +/* Updated: 2024/10/03 21:09:35 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -47,7 +47,6 @@ t_error get_user_input(t_state *state) } } line_edit_stop(&lstate); - printf("state->str_input = %s\n", state->str_input); return (NO_ERROR); } diff --git a/sources/main.c b/sources/main.c index 2d488cf5..1a025d79 100644 --- a/sources/main.c +++ b/sources/main.c @@ -6,7 +6,7 @@ /* By: rparodi +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/03/28 14:40:38 by rparodi #+# #+# */ -/* Updated: 2024/09/30 20:11:12 by maiboyer ### ########.fr */ +/* Updated: 2024/10/03 21:32:41 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -24,6 +24,8 @@ #include "me/types.h" #include "me/vec/vec_str.h" #include "me/vec/vec_token.h" +#include "parser/passes.h" +#include "parser/token.h" #include #include @@ -104,15 +106,16 @@ void func(t_usize i, t_token *token, void *state) { (void)(state); (void)(i); - printf("%s => %s\n", token_name(token), token->string.buf); + printf("["COL_GREEN"%10s"RESET"] '"COL_YELLOW"%s"RESET"'\n", token_name(token), token->string.buf); } -t_error tokenize(t_const_str s, t_vec_token *out); void parse_str(t_state *state) { t_vec_token tokens; if (tokenize(state->str_input, &tokens)) return ; + if (ts_apply_passes(tokens, &tokens)) + return ; vec_token_iter(&tokens, func, NULL); }