update: added passses mechanisme to tokenstream processing

parser/src/passes.c

@@ -0,0 +1,63 @@
+/* ************************************************************************** */
+/*                                                                            */
+/*                                                        :::      ::::::::   */
+/*   passes.c                                           :+:      :+:    :+:   */
+/*                                                    +:+ +:+         +:+     */
+/*   By: maiboyer <maiboyer@student.42.fr>          +#+  +:+       +#+        */
+/*                                                +#+#+#+#+#+   +#+           */
+/*   Created: 2024/10/02 18:41:16 by maiboyer          #+#    #+#             */
+/*   Updated: 2024/10/02 19:15:38 by maiboyer         ###   ########.fr       */
+/*                                                                            */
+/* ************************************************************************** */
+
+#include "parser/passes.h"
+#include "me/printf/printf.h"
+#include "me/types.h"
+#include "me/vec/vec_token.h"
+#include "parser/token.h"
+
+/// This is a list what of kind of passes we need to make on the tokenstream
+/// they'll all have the same function signature, basically taking a token
+/// stream in and returning a tokenstream out
+///
+/// so `t_error	pass_name(t_vec_token input, t_vec_token *ouput)`
+/// or t_ts_pass
+///
+/// there is a few stuff we want to do, for example:
+/// 	- combine any whitespace token that are following eachother into a
+/// 		single one, basically discarding duplicates
+/// 	- combine any *QUOTE token that are next to eachothers
+/// 		into a single metatoken WORD
+/// 	- combine multiple tokens into a single if they can
+/// 		(double PIPE into OR, double AMP into AND,
+/// 			double LCARRET into DLCARET, ...)
+/// 	- create EXPENSION token when DOLLAR and NQUOTE follow eachother,
+/// 			maybe leaving some stuff after
+
+// here is the signature easily accessible:
+//
+// t_error	pass_name(t_vec_token input, t_vec_token *output);
+
+// bellow is some function to apply all the different passes !
+
+static const struct s_ts_pass_def g_ts_passes[] = {
+	{do_fuck_all, "does nothing lol"},
+};
+
+t_error apply_all_passes(t_vec_token ts, t_vec_token *out)
+{
+	t_usize		i;
+	t_vec_token next;
+
+	i = 0;
+	while (i < sizeof(g_ts_passes) / sizeof(g_ts_passes[0]))
+	{
+		if (g_ts_passes[i].fn == NULL)
+			return (vec_token_free(ts), ERROR);
+		if ((g_ts_passes[i].fn)(ts, &next))
+			return (me_eprintf("failed on %s token pass\n", g_ts_passes[i].name), ERROR);
+		ts = next;
+		i++;
+	}
+	return (*out = ts, NO_ERROR);
+}

parser/src/passes/template_file.c

@@ -0,0 +1,31 @@
+/* ************************************************************************** */
+/*                                                                            */
+/*                                                        :::      ::::::::   */
+/*   template_file.c                                    :+:      :+:    :+:   */
+/*                                                    +:+ +:+         +:+     */
+/*   By: maiboyer <maiboyer@student.42.fr>          +#+  +:+       +#+        */
+/*                                                +#+#+#+#+#+   +#+           */
+/*   Created: 2024/10/02 19:04:32 by maiboyer          #+#    #+#             */
+/*   Updated: 2024/10/02 19:13:31 by maiboyer         ###   ########.fr       */
+/*                                                                            */
+/* ************************************************************************** */
+
+#include "parser/passes.h"
+#include "me/types.h"
+#include "me/vec/vec_token.h"
+#include "parser/token.h"
+
+t_error	do_fuck_all(t_vec_token input, t_vec_token *output)
+{
+	t_vec_token	out;
+	t_usize		i;
+
+	out = vec_token_new(input.len, token_free);
+	while (i < input.len)
+	{
+		vec_token_push(&out, token_clone(&input.buffer[i]));
+		i++;
+	}
+	vec_token_free(input);
+	return (*output = out, NO_ERROR);
+}

parser/src/token_lifetime.c

@@ -6,10 +6,11 @@
 /*   By: maiboyer <maiboyer@student.42.fr>          +#+  +:+       +#+        */
 /*                                                +#+#+#+#+#+   +#+           */
 /*   Created: 2024/09/28 14:37:13 by maiboyer          #+#    #+#             */
-/*   Updated: 2024/09/30 20:15:05 by maiboyer         ###   ########.fr       */
+/*   Updated: 2024/10/02 19:11:25 by maiboyer         ###   ########.fr       */
 /*                                                                            */
 /* ************************************************************************** */
 
+#include "me/string/string.h"
 #include "me/vec/vec_token.h"
 #include "parser/token.h"
 
@@ -33,14 +34,34 @@ t_token token_new_meta(enum e_token type)
 
 bool token_is_meta(t_token tok)
 {
-	if (tok.type == TOK_WORD)
-		return (true);
-	return (false);
+	return (tok.subtokens.buffer != NULL);
 }
 
 t_token token_new_none(void)
 {
-	return ((t_token){.type = TOK_NONE, .string = {NULL, 0, 0}, .subtokens = vec_token_new(16, token_free)});
+	return ((t_token){.type = TOK_NONE, .string = {NULL, 0, 0}, .subtokens = {NULL, 0, 0, NULL}});
+}
+
+t_token token_clone(t_token *tok)
+{
+	t_token out;
+	t_usize i;
+
+	out = token_new_none();
+	out.type = tok->type;
+	if (tok->string.buf != NULL)
+	{
+		out.string = string_new(tok->string.capacity);
+		string_push(&out.string, tok->string.buf);
+	}
+	if (tok->subtokens.buffer != NULL)
+	{
+		out.subtokens = vec_token_new(tok->subtokens.capacity, token_free);
+		i = 0;
+		while (i < tok->subtokens.len)
+			vec_token_push(&out.subtokens, token_clone(&tok->subtokens.buffer[i++]));
+	}
+	return (out);
 }
 
 // TO REMOVE