update: added python version of the tokenizer

parser/token.py/prettier.py

@@ -0,0 +1,92 @@
+from enum import Enum
+from dataclasses import dataclass
+import make_token as mt
+
+
+TokenType = Enum(
+    "TokenType",
+    [
+        "AND",
+        "DOLLAR",
+        "DQUOTE",
+        "EXPENSION",
+        "LCARRET",
+        "LCARRET_DOUBLE",
+        "LPAREN",
+        "NQUOTE",
+        "OR",
+        "PIPE",
+        "RCARRET",
+        "RCARRET_DOUBLE",
+        "RPAREN",
+        "SEMICOLON",
+        "SQUOTE",
+        "WHITESPACE",
+        "WORD",
+    ],
+)
+
+
+@dataclass
+class Token:
+    raw: str
+    raw_list: list
+    ty: TokenType
+
+    def is_list(self):
+        return self.ty == TokenType.WORD or self.ty == TokenType.EXPENSION
+
+
+def is_word_mt(tok: mt.Token):
+    return (
+        tok.ty == mt.TokenType.SQUOTE
+        or tok.ty == mt.TokenType.DQUOTE
+        or tok.ty == mt.TokenType.NQUOTE
+        or tok.ty == mt.TokenType.DOLLAR
+    )
+
+
+def pass1(tokens: list[mt.Token]):
+    i = 0
+    out = []
+    while i < len(tokens):
+        tok = tokens[i]
+        if is_word_mt(tok):
+            concat = Token("", [], TokenType.WORD)
+            concat.raw_list.append(tok)
+            j = 1
+            while i + j < len(tokens) and is_word_mt(tokens[i + j]):
+                concat.raw_list.append(tokens[i + j])
+                j += 1
+            i += j
+        else:
+            out.append(tok)
+            i += 1
+    return out
+
+
+def print_tokenlist(tokens: list[Token]):
+    print("\n")
+    for tok in tokens:
+        col = "0"
+        if tok.ty == TokenType.SQUOTE:
+            col = "33"
+        if tok.ty == TokenType.DQUOTE:
+            col = "32"
+        if tok.ty == TokenType.WHITESPACE:
+            col = "31;4"
+        if tok.ty == TokenType.DOLLAR:
+            col = "31"
+        if tok.ty == TokenType.LPAREN:
+            col = "35"
+        if tok.ty == TokenType.RPAREN:
+            col = "35"
+        if tok.ty == TokenType.PIPE:
+            col = "35"
+        if tok.ty == TokenType.SEMICOLON:
+            col = "35"
+        if not Token.is_list(tok):
+            print(f"\x1b[{col}m{tok.raw}\x1b[0m", end="")
+        else:
+            print("NOT_PRINT_YET_LOL", end="")
+    print("\n")