diff --git a/parser/token.py/app.py b/parser/token.py/app.py index faa03e1c..b6b174b4 100644 --- a/parser/token.py/app.py +++ b/parser/token.py/app.py @@ -1,5 +1,5 @@ import str_to_token -import prettier +import concat import ttoken s = input("> ") diff --git a/parser/token.py/concat.py b/parser/token.py/concat.py index ae284f8e..eb5f0cd2 100644 --- a/parser/token.py/concat.py +++ b/parser/token.py/concat.py @@ -1,20 +1,25 @@ from ttoken import * + +# This function will make a "big" token that will represent a word in the shell sense def concat(tokens: list[Token]): i = 0 out = [] while i < len(tokens): tok = tokens[i] + # if the token is a token that can be inside a word, then we start createing a WORD "metaToken" if tok.is_word(): word = Token(TokenType.WORD, subtokens=[]) word.subtokens.append(tok) j = 1 + # then we get every token after the first that is also a word and we push them while i + j < len(tokens) and (tokens[i + j]).is_word(): word.subtokens.append(tokens[i + j]) j += 1 i += j out.append(word) else: + # otherwise we just push the token alone out.append(tok) i += 1 return out diff --git a/parser/token.py/str_to_token.py b/parser/token.py/str_to_token.py index 2abed189..cc1c16fb 100644 --- a/parser/token.py/str_to_token.py +++ b/parser/token.py/str_to_token.py @@ -7,6 +7,7 @@ def is_quote(c: chr): return c == "'" or c == '"' +# This function takes the string and seperate them into different tokens depending on the quotes def str_to_token(s: str): tokens = [] current_token = None @@ -15,15 +16,20 @@ def str_to_token(s: str): while i < len(s): c = s[i] if quote == 0: + # if we have a quote, juste push the current token if any, then switch the the correct quote token if is_quote(c): if current_token != None: tokens.append(current_token) quote = c current_token = Token(TT.DQUOTE if c == '"' else TT.SQUOTE, string="") else: + # here we have no quote, so we first create a token if none exist, then handle special stuff + # like whitespace for example, or any character we want to spit in a single token of their own (; $ | &) if current_token == None: current_token = Token(TT.NQUOTE, string="") if c.isspace(): + # we have a whitespace, then create a whitespace token, and push the current token + # if it isn't empty and not whitesace if ( len(current_token.string) != 0 and current_token.ty != TT.WHITESPACE @@ -31,6 +37,7 @@ def str_to_token(s: str): tokens.append(current_token) current_token = Token(TT.WHITESPACE, string="") else: + # we DON'T have a whitespace, then if the current token is a whitespace, just push it and set the new token to raw_string if current_token.ty == TT.WHITESPACE: tokens.append(current_token) current_token = Token(TT.NQUOTE, string="") @@ -65,6 +72,7 @@ def str_to_token(s: str): else: current_token.append_char(c) elif quote == "'": + # we are in a single quotem basically we push until we have another single quote if c == "'": tokens.append(current_token) current_token = None @@ -74,6 +82,7 @@ def str_to_token(s: str): current_token = Token(TT.SQUOTE, string="") current_token.append_char(c) elif quote == '"': + # we are in a double quotem basically we push until we have another double quote if c == '"': tokens.append(current_token) current_token = None @@ -85,6 +94,7 @@ def str_to_token(s: str): else: print("you fucked up you quote thingy") i += 1 + # if the current token is not none and the current token is "no quote" then we push it if current_token != None and current_token.ty == TT.NQUOTE: tokens.append(current_token) return tokens