From 1d317c03886b466fe39b0239b315b3c8fc5157e3 Mon Sep 17 00:00:00 2001 From: maix0 Date: Wed, 2 Oct 2024 19:19:35 +0200 Subject: [PATCH] update: added passses mechanisme to tokenstream processing --- parser/include/parser/passes.h | 34 +++++++++++++++++ parser/include/parser/token.h | 8 +--- parser/src/passes.c | 63 +++++++++++++++++++++++++++++++ parser/src/passes/template_file.c | 31 +++++++++++++++ parser/src/token_lifetime.c | 31 ++++++++++++--- 5 files changed, 156 insertions(+), 11 deletions(-) create mode 100644 parser/include/parser/passes.h create mode 100644 parser/src/passes.c create mode 100644 parser/src/passes/template_file.c diff --git a/parser/include/parser/passes.h b/parser/include/parser/passes.h new file mode 100644 index 00000000..b13b81c6 --- /dev/null +++ b/parser/include/parser/passes.h @@ -0,0 +1,34 @@ +/* ************************************************************************** */ +/* */ +/* ::: :::::::: */ +/* passes.h :+: :+: :+: */ +/* +:+ +:+ +:+ */ +/* By: maiboyer +#+ +:+ +#+ */ +/* +#+#+#+#+#+ +#+ */ +/* Created: 2024/10/02 18:43:41 by maiboyer #+# #+# */ +/* Updated: 2024/10/02 19:13:19 by maiboyer ### ########.fr */ +/* */ +/* ************************************************************************** */ + +#ifndef PASSES_H +#define PASSES_H + +#include "me/types.h" +#include "me/vec/vec_token.h" +#include "parser/token.h" + +typedef t_error (*t_ts_pass)(t_vec_token input, t_vec_token *output); + +struct s_ts_pass_def +{ + t_ts_pass fn; + t_const_str name; +}; + + +// list passes function here + +// this is a example one, does absolutly nothing lol +t_error do_fuck_all(t_vec_token input, t_vec_token *output); + +#endif /* PASSES_H */ diff --git a/parser/include/parser/token.h b/parser/include/parser/token.h index dbaf637d..f073c71d 100644 --- a/parser/include/parser/token.h +++ b/parser/include/parser/token.h @@ -6,7 +6,7 @@ /* By: maiboyer +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/09/26 17:59:23 by maiboyer #+# #+# */ -/* Updated: 2024/09/30 19:47:53 by maiboyer ### ########.fr */ +/* Updated: 2024/10/02 19:12:26 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -55,13 +55,9 @@ t_token token_new(enum e_token type); t_token token_new_none(void); void token_free(t_token tok); bool token_is_meta(t_token tok); +t_token token_clone(t_token *tok); /* PARSING */ -bool is_dollar(char c); -bool is_quote(char c); -bool is_space(char c); -t_error find_end_string(t_const_str raw, t_usize *start, t_token *output); -t_error start_analyse(t_const_str raw, t_vec_token *output); t_error tokeniser(t_const_str raw); #endif diff --git a/parser/src/passes.c b/parser/src/passes.c new file mode 100644 index 00000000..49e6505a --- /dev/null +++ b/parser/src/passes.c @@ -0,0 +1,63 @@ +/* ************************************************************************** */ +/* */ +/* ::: :::::::: */ +/* passes.c :+: :+: :+: */ +/* +:+ +:+ +:+ */ +/* By: maiboyer +#+ +:+ +#+ */ +/* +#+#+#+#+#+ +#+ */ +/* Created: 2024/10/02 18:41:16 by maiboyer #+# #+# */ +/* Updated: 2024/10/02 19:15:38 by maiboyer ### ########.fr */ +/* */ +/* ************************************************************************** */ + +#include "parser/passes.h" +#include "me/printf/printf.h" +#include "me/types.h" +#include "me/vec/vec_token.h" +#include "parser/token.h" + +/// This is a list what of kind of passes we need to make on the tokenstream +/// they'll all have the same function signature, basically taking a token +/// stream in and returning a tokenstream out +/// +/// so `t_error pass_name(t_vec_token input, t_vec_token *ouput)` +/// or t_ts_pass +/// +/// there is a few stuff we want to do, for example: +/// - combine any whitespace token that are following eachother into a +/// single one, basically discarding duplicates +/// - combine any *QUOTE token that are next to eachothers +/// into a single metatoken WORD +/// - combine multiple tokens into a single if they can +/// (double PIPE into OR, double AMP into AND, +/// double LCARRET into DLCARET, ...) +/// - create EXPENSION token when DOLLAR and NQUOTE follow eachother, +/// maybe leaving some stuff after + +// here is the signature easily accessible: +// +// t_error pass_name(t_vec_token input, t_vec_token *output); + +// bellow is some function to apply all the different passes ! + +static const struct s_ts_pass_def g_ts_passes[] = { + {do_fuck_all, "does nothing lol"}, +}; + +t_error apply_all_passes(t_vec_token ts, t_vec_token *out) +{ + t_usize i; + t_vec_token next; + + i = 0; + while (i < sizeof(g_ts_passes) / sizeof(g_ts_passes[0])) + { + if (g_ts_passes[i].fn == NULL) + return (vec_token_free(ts), ERROR); + if ((g_ts_passes[i].fn)(ts, &next)) + return (me_eprintf("failed on %s token pass\n", g_ts_passes[i].name), ERROR); + ts = next; + i++; + } + return (*out = ts, NO_ERROR); +} diff --git a/parser/src/passes/template_file.c b/parser/src/passes/template_file.c new file mode 100644 index 00000000..f45f5313 --- /dev/null +++ b/parser/src/passes/template_file.c @@ -0,0 +1,31 @@ +/* ************************************************************************** */ +/* */ +/* ::: :::::::: */ +/* template_file.c :+: :+: :+: */ +/* +:+ +:+ +:+ */ +/* By: maiboyer +#+ +:+ +#+ */ +/* +#+#+#+#+#+ +#+ */ +/* Created: 2024/10/02 19:04:32 by maiboyer #+# #+# */ +/* Updated: 2024/10/02 19:13:31 by maiboyer ### ########.fr */ +/* */ +/* ************************************************************************** */ + +#include "parser/passes.h" +#include "me/types.h" +#include "me/vec/vec_token.h" +#include "parser/token.h" + +t_error do_fuck_all(t_vec_token input, t_vec_token *output) +{ + t_vec_token out; + t_usize i; + + out = vec_token_new(input.len, token_free); + while (i < input.len) + { + vec_token_push(&out, token_clone(&input.buffer[i])); + i++; + } + vec_token_free(input); + return (*output = out, NO_ERROR); +} diff --git a/parser/src/token_lifetime.c b/parser/src/token_lifetime.c index 4ff16b4b..c2bb5a1d 100644 --- a/parser/src/token_lifetime.c +++ b/parser/src/token_lifetime.c @@ -6,10 +6,11 @@ /* By: maiboyer +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/09/28 14:37:13 by maiboyer #+# #+# */ -/* Updated: 2024/09/30 20:15:05 by maiboyer ### ########.fr */ +/* Updated: 2024/10/02 19:11:25 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ +#include "me/string/string.h" #include "me/vec/vec_token.h" #include "parser/token.h" @@ -33,14 +34,34 @@ t_token token_new_meta(enum e_token type) bool token_is_meta(t_token tok) { - if (tok.type == TOK_WORD) - return (true); - return (false); + return (tok.subtokens.buffer != NULL); } t_token token_new_none(void) { - return ((t_token){.type = TOK_NONE, .string = {NULL, 0, 0}, .subtokens = vec_token_new(16, token_free)}); + return ((t_token){.type = TOK_NONE, .string = {NULL, 0, 0}, .subtokens = {NULL, 0, 0, NULL}}); +} + +t_token token_clone(t_token *tok) +{ + t_token out; + t_usize i; + + out = token_new_none(); + out.type = tok->type; + if (tok->string.buf != NULL) + { + out.string = string_new(tok->string.capacity); + string_push(&out.string, tok->string.buf); + } + if (tok->subtokens.buffer != NULL) + { + out.subtokens = vec_token_new(tok->subtokens.capacity, token_free); + i = 0; + while (i < tok->subtokens.len) + vec_token_push(&out.subtokens, token_clone(&tok->subtokens.buffer[i++])); + } + return (out); } // TO REMOVE