diff --git a/allocator/include/aq/internal_vg_funcs.h b/allocator/include/aq/internal_vg_funcs.h index 94652b0f..c0430a99 100644 --- a/allocator/include/aq/internal_vg_funcs.h +++ b/allocator/include/aq/internal_vg_funcs.h @@ -6,7 +6,7 @@ /* By: maiboyer +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/05/12 22:20:30 by maiboyer #+# #+# */ -/* Updated: 2024/09/13 15:32:52 by maiboyer ### ########.fr */ +/* Updated: 2024/10/03 22:46:09 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -16,13 +16,13 @@ # include "me/types.h" # ifndef NVALGRIND -# if _INCLUDE_VALGRIND +# ifndef _INCLUDE_VALGRIND # define _INCLUDE_VALGRIND # endif # endif # ifdef VGHEADER -# if _INCLUDE_VALGRIND +# ifndef _INCLUDE_VALGRIND # define _INCLUDE_VALGRIND # endif # endif diff --git a/parser/Filelist.parser.mk b/parser/Filelist.parser.mk index a89efddc..70ac6062 100644 --- a/parser/Filelist.parser.mk +++ b/parser/Filelist.parser.mk @@ -1,8 +1,11 @@ SRC_FILES = \ passes \ +passes/double_quote_parsing \ +passes/fold_expansion \ passes/template_file \ token_lifetime \ tokenizer \ +tokenizer_utils \ GEN_FILES = \ \ diff --git a/parser/include/parser/passes.h b/parser/include/parser/passes.h index cf40b6f1..05eacc9f 100644 --- a/parser/include/parser/passes.h +++ b/parser/include/parser/passes.h @@ -6,7 +6,7 @@ /* By: maiboyer +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/10/02 18:43:41 by maiboyer #+# #+# */ -/* Updated: 2024/10/03 21:32:32 by maiboyer ### ########.fr */ +/* Updated: 2024/10/03 22:31:21 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -17,19 +17,26 @@ #include "me/vec/vec_token.h" #include "parser/token.h" -typedef t_error (*t_ts_pass)(t_vec_token input, t_vec_token *output); +typedef t_error (*t_ts_pass)(t_vec_token input, t_vec_token *output); struct s_ts_pass_def { t_ts_pass fn; - t_const_str name; + t_const_str name; }; t_error ts_apply_passes(t_vec_token ts, t_vec_token *out); +// This function is to apply stuff on doublequote meta token +// You shouldn't have to use it ! +// It *may* disapear/change/whatever +t_error ts_dq_apply_passes(t_vec_token ts, t_vec_token *out); + // list passes function here // this is a example one, does absolutly nothing lol -t_error do_fuck_all(t_vec_token input, t_vec_token *output); +t_error ts_do_fuck_all(t_vec_token input, t_vec_token *output); +t_error ts_fold_expension(t_vec_token input, t_vec_token *output); +t_error ts_double_string_pass(t_vec_token input, t_vec_token *output); #endif /* PASSES_H */ diff --git a/parser/include/parser/token.h b/parser/include/parser/token.h index b49b186a..95868033 100644 --- a/parser/include/parser/token.h +++ b/parser/include/parser/token.h @@ -6,7 +6,7 @@ /* By: maiboyer +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/09/26 17:59:23 by maiboyer #+# #+# */ -/* Updated: 2024/10/03 21:33:14 by maiboyer ### ########.fr */ +/* Updated: 2024/10/03 22:30:42 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -26,7 +26,7 @@ enum e_token TOK_DOLLAR, // dollar == $ TOK_DQUOTE, // double quote string TOK_DRCARRET, // double right carret == >> - TOK_EXPENSION, // an expension == $ + TOK_EXPENSION, // an expension == $; the $ is not in .string TOK_LCARRET, // left carret == < TOK_LPAREN, // left parenthesis == ( TOK_NQUOTE, // no quote string diff --git a/parser/src/passes.c b/parser/src/passes.c index 550ebf18..8275d907 100644 --- a/parser/src/passes.c +++ b/parser/src/passes.c @@ -6,7 +6,7 @@ /* By: maiboyer +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/10/02 18:41:16 by maiboyer #+# #+# */ -/* Updated: 2024/10/03 21:32:14 by maiboyer ### ########.fr */ +/* Updated: 2024/10/03 22:41:08 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -33,7 +33,7 @@ /// double LCARRET into DLCARET, ...) /// - create EXPENSION token when DOLLAR and NQUOTE follow eachother, /// maybe leaving some stuff after -/// - parse double quote string to see expansion in them, creating a meta +/// - parse double quote string to see expansion in them, creating a meta /// token consisting of the pieces // here is the signature easily accessible: @@ -43,7 +43,8 @@ // bellow is some function to apply all the different passes ! static const struct s_ts_pass_def g_ts_passes[] = { - {do_fuck_all, "does nothing lol"}, + {ts_double_string_pass, "double string parser"}, + {ts_fold_expension, "fold expansion"}, }; t_error ts_apply_passes(t_vec_token ts, t_vec_token *out) @@ -65,3 +66,28 @@ t_error ts_apply_passes(t_vec_token ts, t_vec_token *out) } return (*out = ts, NO_ERROR); } + +static const struct s_ts_pass_def g_ts_dq_passes[] = { + {ts_do_fuck_all, "does nothing lol"}, + {ts_fold_expension, "fold expansion"}, +}; + +t_error ts_dq_apply_passes(t_vec_token ts, t_vec_token *out) +{ + t_usize i; + t_vec_token next; + + i = 0; + while (i < sizeof(g_ts_dq_passes) / sizeof(g_ts_dq_passes[0])) + { + if (g_ts_dq_passes[i].fn == NULL) + return (vec_token_free(ts), ERROR); + if ((g_ts_dq_passes[i].fn)(ts, &next)) + return (me_eprintf("failed on '%s' dq token pass\n", g_ts_dq_passes[i].name), ERROR); + else + me_printf("Applied '%s' dq_pass\n", g_ts_dq_passes[i].name); + ts = next; + i++; + } + return (*out = ts, NO_ERROR); +} diff --git a/parser/src/passes/double_quote_parsing.c b/parser/src/passes/double_quote_parsing.c new file mode 100644 index 00000000..17a15999 --- /dev/null +++ b/parser/src/passes/double_quote_parsing.c @@ -0,0 +1,98 @@ +/* ************************************************************************** */ +/* */ +/* ::: :::::::: */ +/* double_quote_parsing.c :+: :+: :+: */ +/* +:+ +:+ +:+ */ +/* By: maiboyer +#+ +:+ +#+ */ +/* +#+#+#+#+#+ +#+ */ +/* Created: 2024/10/02 19:04:32 by maiboyer #+# #+# */ +/* Updated: 2024/10/03 22:43:27 by maiboyer ### ########.fr */ +/* */ +/* ************************************************************************** */ + +#include "me/string/string.h" +#include "parser/passes.h" +#include "me/types.h" +#include "me/vec/vec_token.h" +#include "parser/token.h" +#include "me/char/char.h" + +void push_token_and_create_new(\ + t_vec_token *tokens, t_token *tok, enum e_token ttype, t_const_str s); +void push_token_and_set_new(\ + t_vec_token *tokens, t_token *tok, enum e_token ttype, t_const_str s); + +t_error _parse_dquote_inner(t_token dquote, t_vec_token *append) +{ + t_token ctok; + t_token out; + t_usize i; + char c; + + out = token_new_meta(TOK_DQUOTE); + i = 0; + ctok = token_new_none(); + while (dquote.string.buf[i] != '\0') + { + c = dquote.string.buf[i++]; + if (me_isspace(c)) + { + if (ctok.type == TOK_NONE) + ctok = token_new(TOK_WHITESPACE); + if (ctok.type != TOK_WHITESPACE) + { + vec_token_push(&out.subtokens, ctok); + ctok = token_new(TOK_WHITESPACE); + } + string_push_char(&ctok.string, c); + } + else if (c == '$') + push_token_and_create_new(&out.subtokens, &ctok, TOK_DOLLAR, "$"); + else + { + if (ctok.type == TOK_NONE) + ctok = token_new(TOK_NQUOTE); + if (ctok.type != TOK_NQUOTE) + { + vec_token_push(&out.subtokens, ctok); + ctok = token_new(TOK_NQUOTE); + } + string_push_char(&ctok.string, c); + } + }; + if (ctok.type != TOK_NONE) + vec_token_push(&out.subtokens, ctok); + if (ts_dq_apply_passes(out.subtokens, &out.subtokens)) + return (ERROR); + return (vec_token_push(append, out), NO_ERROR); +} + +/// There is a few rules the rest of the tokenizer machinery assumes +/// theses function follows: +/// - the input vec WILL be freed when the function return, even in +/// case of error +/// - the output vector isn't populated if the function returns an error, +/// thus it shouldn't be freed in case of error +/// - the output tokens may not be direct copy of the input tokens, +/// but need to be cloned (different allocations for stuff) +t_error ts_double_string_pass(t_vec_token input, t_vec_token *output) +{ + t_vec_token out; + t_usize i; + + i = 0; + out = vec_token_new(input.len, token_free); + while (i < input.len) + { + if (input.buffer[i].type == TOK_DQUOTE) + { + if (_parse_dquote_inner(input.buffer[i], &out)) + return (vec_token_free(input), ERROR); + } + else + vec_token_push(&out, token_clone(&input.buffer[i])); + i++; + } + vec_token_free(input); + return (*output = out, NO_ERROR); +} diff --git a/parser/src/passes/fold_expansion.c b/parser/src/passes/fold_expansion.c new file mode 100644 index 00000000..9f17cbe4 --- /dev/null +++ b/parser/src/passes/fold_expansion.c @@ -0,0 +1,54 @@ +/* ************************************************************************** */ +/* */ +/* ::: :::::::: */ +/* fold_expansion.c :+: :+: :+: */ +/* +:+ +:+ +:+ */ +/* By: maiboyer +#+ +:+ +#+ */ +/* +#+#+#+#+#+ +#+ */ +/* Created: 2024/10/02 19:04:32 by maiboyer #+# #+# */ +/* Updated: 2024/10/03 22:50:40 by maiboyer ### ########.fr */ +/* */ +/* ************************************************************************** */ + +#include "me/types.h" +#include "me/vec/vec_token.h" +#include "parser/passes.h" +#include "parser/token.h" + +/// This is a sample pass +/// +/// There is a few rules the rest of the tokenizer machinery assumes +/// theses function follows: +/// - the input vec WILL be freed when the function return, even in +/// case of error +/// - the output vector isn't populated if the function returns an error, +/// thus it shouldn't be freed in case of error +/// - the output tokens may not be direct copy of the input tokens, +/// but need to be cloned (different allocations for stuff) +t_error ts_fold_expension(t_vec_token input, t_vec_token *output) +{ + t_vec_token out; + t_usize i; + t_token tmp; + + i = 0; + out = vec_token_new(input.len, token_free); + while (i < input.len) + { + if (i + 1 >= input.len) + vec_token_push(&out, token_clone(&input.buffer[i])); + else if (input.buffer[i].type == TOK_DOLLAR \ + && (input.buffer[i + 1].type == TOK_DOLLAR \ + || input.buffer[i + 1].type == TOK_NQUOTE)) + { + tmp = token_clone(&input.buffer[++i]); + tmp.type= TOK_EXPENSION; + vec_token_push(&out, tmp); + } + else + vec_token_push(&out, token_clone(&input.buffer[i])); + i++; + } + vec_token_free(input); + return (*output = out, NO_ERROR); +} diff --git a/parser/src/passes/template_file.c b/parser/src/passes/template_file.c index 430366b5..f225b477 100644 --- a/parser/src/passes/template_file.c +++ b/parser/src/passes/template_file.c @@ -6,7 +6,7 @@ /* By: maiboyer +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/10/02 19:04:32 by maiboyer #+# #+# */ -/* Updated: 2024/10/03 21:37:04 by maiboyer ### ########.fr */ +/* Updated: 2024/10/03 22:23:40 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -25,7 +25,7 @@ /// thus it shouldn't be freed in case of error /// - the output tokens may not be direct copy of the input tokens, /// but need to be cloned (different allocations for stuff) -t_error do_fuck_all(t_vec_token input, t_vec_token *output) +t_error ts_do_fuck_all(t_vec_token input, t_vec_token *output) { t_vec_token out; t_usize i; diff --git a/parser/src/tokenizer.c b/parser/src/tokenizer.c index 2356f679..3d1dc635 100644 --- a/parser/src/tokenizer.c +++ b/parser/src/tokenizer.c @@ -6,7 +6,7 @@ /* By: maiboyer +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/09/30 19:39:39 by maiboyer #+# #+# */ -/* Updated: 2024/10/03 21:31:57 by maiboyer ### ########.fr */ +/* Updated: 2024/10/03 22:44:57 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -16,26 +16,10 @@ #include "me/vec/vec_token.h" #include "parser/token.h" -static void push_token_and_create_new(\ - t_vec_token *tokens, t_token *tok, enum e_token ttype, t_const_str s) -{ - t_token tmp; - - if (tok->type != TOK_NONE) - vec_token_push(tokens, *tok); - *tok = token_new_none(); - tmp = token_new(ttype); - string_push(&tmp.string, s); - vec_token_push(tokens, tmp); -} -static void push_token_and_set_new(\ - t_vec_token *tokens, t_token *tok, enum e_token ttype, t_const_str s) -{ - if (tok->type != TOK_NONE) - vec_token_push(tokens, *tok); - *tok = token_new(ttype); - string_push(&tok->string, s); -} +void push_token_and_create_new(\ + t_vec_token *tokens, t_token *tok, enum e_token ttype, t_const_str s); +void push_token_and_set_new(\ + t_vec_token *tokens, t_token *tok, enum e_token ttype, t_const_str s); static void handle_quote(t_vec_token *ret, char chr, t_token *tok, char *quote) { @@ -68,7 +52,7 @@ static void handle_noquote(t_vec_token *ret, char chr, t_token *tok, char *quote else if (chr == ')') push_token_and_create_new(ret, tok, TOK_RPAREN, ")"); else if (chr == ';') - push_token_and_create_new(ret, tok, TOK_RPAREN, ";"); + push_token_and_create_new(ret, tok, TOK_SEMICOLON, ";"); else if (me_isspace(chr)) push_token_and_create_new(ret, tok, TOK_WHITESPACE, " "); else diff --git a/parser/src/tokenizer_utils.c b/parser/src/tokenizer_utils.c new file mode 100644 index 00000000..f0a9a17c --- /dev/null +++ b/parser/src/tokenizer_utils.c @@ -0,0 +1,46 @@ +/* ************************************************************************** */ +/* */ +/* ::: :::::::: */ +/* tokenizer_utils.c :+: :+: :+: */ +/* +:+ +:+ +:+ */ +/* By: maiboyer +#+ +:+ +#+ */ +/* +#+#+#+#+#+ +#+ */ +/* Created: 2024/10/03 22:07:25 by maiboyer #+# #+# */ +/* Updated: 2024/10/03 22:08:14 by maiboyer ### ########.fr */ +/* */ +/* ************************************************************************** */ + +#include "me/char/char.h" +#include "me/string/string.h" +#include "me/types.h" +#include "me/vec/vec_token.h" +#include "parser/token.h" + +// These *should* be stable-ish, just copy the definition into the files to +// use them +void push_token_and_create_new(\ + t_vec_token *tokens, t_token *tok, enum e_token ttype, t_const_str s); +void push_token_and_set_new(\ + t_vec_token *tokens, t_token *tok, enum e_token ttype, t_const_str s); + +void push_token_and_create_new(\ + t_vec_token *tokens, t_token *tok, enum e_token ttype, t_const_str s) +{ + t_token tmp; + + if (tok->type != TOK_NONE) + vec_token_push(tokens, *tok); + *tok = token_new_none(); + tmp = token_new(ttype); + string_push(&tmp.string, s); + vec_token_push(tokens, tmp); +} + +void push_token_and_set_new(\ + t_vec_token *tokens, t_token *tok, enum e_token ttype, t_const_str s) +{ + if (tok->type != TOK_NONE) + vec_token_push(tokens, *tok); + *tok = token_new(ttype); + string_push(&tok->string, s); +} diff --git a/sources/main.c b/sources/main.c index 1a025d79..e3445378 100644 --- a/sources/main.c +++ b/sources/main.c @@ -6,7 +6,7 @@ /* By: rparodi +#+ +:+ +#+ */ /* +#+#+#+#+#+ +#+ */ /* Created: 2024/03/28 14:40:38 by rparodi #+# #+# */ -/* Updated: 2024/10/03 21:32:41 by maiboyer ### ########.fr */ +/* Updated: 2024/10/03 22:47:05 by maiboyer ### ########.fr */ /* */ /* ************************************************************************** */ @@ -102,11 +102,29 @@ void print_node_data(t_node *t, t_usize depth) */ t_str token_name(t_token *out); -void func(t_usize i, t_token *token, void *state) +void func(t_usize i, t_token *token, void *vdepth) { - (void)(state); - (void)(i); - printf("["COL_GREEN"%10s"RESET"] '"COL_YELLOW"%s"RESET"'\n", token_name(token), token->string.buf); + t_usize depth; + t_string sdepth; + + depth = 0; + if (vdepth != NULL) + depth = *(t_usize *)vdepth; + sdepth = string_new(16); + i = 0; + while (i++ < depth) + string_push_char(&sdepth, '\t'); + + if (token->subtokens.buffer != NULL) + { + depth++; + printf("%s[" COL_GREEN "%10s"RESET"]\n", sdepth.buf ,token_name(token)); + vec_token_iter(&token->subtokens, func, &depth); + } + else + printf("%s[" COL_GREEN "%10s"RESET"] '"COL_YELLOW"%s"RESET"'\n",\ + sdepth.buf ,token_name(token), token->string.buf); + string_free(sdepth); } void parse_str(t_state *state) @@ -117,6 +135,7 @@ void parse_str(t_state *state) if (ts_apply_passes(tokens, &tokens)) return ; vec_token_iter(&tokens, func, NULL); + vec_token_free(tokens); } t_i32 main(t_i32 argc, t_str argv[], t_str envp[])