update: commented the python code so it is more readable

parser/token.py/app.py

@@ -1,5 +1,5 @@
 import str_to_token
-import prettier
+import concat 
 import ttoken
 
 s = input("> ")

parser/token.py/concat.py

@@ -1,20 +1,25 @@
 from ttoken import *
 
+
+# This function will make a "big" token that will represent a word in the shell sense
 def concat(tokens: list[Token]):
     i = 0
     out = []
     while i < len(tokens):
         tok = tokens[i]
+        # if the token is a token that can be inside a word, then we start createing a WORD "metaToken"
         if tok.is_word():
             word = Token(TokenType.WORD, subtokens=[])
             word.subtokens.append(tok)
             j = 1
+            # then we get every token after the first that is also a word and we push them
             while i + j < len(tokens) and (tokens[i + j]).is_word():
                 word.subtokens.append(tokens[i + j])
                 j += 1
             i += j
             out.append(word)
         else:
+            # otherwise we just push the token alone
             out.append(tok)
             i += 1
     return out

parser/token.py/str_to_token.py

@@ -7,6 +7,7 @@ def is_quote(c: chr):
     return c == "'" or c == '"'
 
 
+# This function takes the string and seperate them into different tokens depending on the quotes
 def str_to_token(s: str):
     tokens = []
     current_token = None
@@ -15,15 +16,20 @@ def str_to_token(s: str):
     while i < len(s):
         c = s[i]
         if quote == 0:
+            # if we have a quote, juste push the current token if any, then switch the the correct quote token 
             if is_quote(c):
                 if current_token != None:
                     tokens.append(current_token)
                 quote = c
                 current_token = Token(TT.DQUOTE if c == '"' else TT.SQUOTE, string="")
             else:
+                # here we have no quote, so we first create a token if none exist, then handle special stuff 
+                # like whitespace for example, or any character we want to spit in a single token of their own (; $ | &)
                 if current_token == None:
                     current_token = Token(TT.NQUOTE, string="")
                 if c.isspace():
+                    # we have a whitespace, then create a whitespace token, and push the current token 
+                    # if it isn't empty and not whitesace
                     if (
                         len(current_token.string) != 0
                         and current_token.ty != TT.WHITESPACE
@@ -31,6 +37,7 @@ def str_to_token(s: str):
                         tokens.append(current_token)
                     current_token = Token(TT.WHITESPACE, string="")
                 else:
+                    # we DON'T have a whitespace, then if the current token is a whitespace, just push it and set the new token to raw_string
                     if current_token.ty == TT.WHITESPACE:
                         tokens.append(current_token)
                         current_token = Token(TT.NQUOTE, string="")
@@ -65,6 +72,7 @@ def str_to_token(s: str):
                 else:
                     current_token.append_char(c)
         elif quote == "'":
+            # we are in a single quotem basically we push until we have another single quote
             if c == "'":
                 tokens.append(current_token)
                 current_token = None
@@ -74,6 +82,7 @@ def str_to_token(s: str):
                     current_token = Token(TT.SQUOTE, string="")
                 current_token.append_char(c)
         elif quote == '"':
+            # we are in a double quotem basically we push until we have another double quote
             if c == '"':
                 tokens.append(current_token)
                 current_token = None
@@ -85,6 +94,7 @@ def str_to_token(s: str):
         else:
             print("you fucked up you quote thingy")
         i += 1
+    # if the current token is not none and the current token is "no quote" then we push it
     if current_token != None and current_token.ty == TT.NQUOTE:
         tokens.append(current_token)
     return tokens