90 lines
3.3 KiB
Python
90 lines
3.3 KiB
Python
from ttoken import *
|
|
|
|
TT = TokenType
|
|
|
|
|
|
def is_quote(c: chr):
|
|
return c == "'" or c == '"'
|
|
|
|
|
|
def str_to_token(s: str):
|
|
tokens = []
|
|
current_token = None
|
|
quote = 0
|
|
i = 0
|
|
while i < len(s):
|
|
c = s[i]
|
|
if quote == 0:
|
|
if is_quote(c):
|
|
if current_token != None:
|
|
tokens.append(current_token)
|
|
quote = c
|
|
current_token = Token(TT.DQUOTE if c == '"' else TT.SQUOTE, string="")
|
|
else:
|
|
if current_token == None:
|
|
current_token = Token(TT.NQUOTE, string="")
|
|
if c.isspace():
|
|
if (
|
|
len(current_token.string) != 0
|
|
and current_token.ty != TT.WHITESPACE
|
|
):
|
|
tokens.append(current_token)
|
|
current_token = Token(TT.WHITESPACE, string="")
|
|
else:
|
|
if current_token.ty == TT.WHITESPACE:
|
|
tokens.append(current_token)
|
|
current_token = Token(TT.NQUOTE, string="")
|
|
if c == "$":
|
|
tokens.append(current_token)
|
|
current_token = None
|
|
tokens.append(Token(TT.DOLLAR, string="$"))
|
|
elif c == "(":
|
|
tokens.append(current_token)
|
|
current_token = None
|
|
tokens.append(Token(TT.LPAREN, string="("))
|
|
elif c == ")":
|
|
tokens.append(current_token)
|
|
current_token = None
|
|
tokens.append(Token(TT.RPAREN, string=")"))
|
|
elif c == "|":
|
|
tokens.append(current_token)
|
|
current_token = None
|
|
tokens.append(Token(TT.PIPE, string="|"))
|
|
elif c == "&":
|
|
tokens.append(current_token)
|
|
current_token = None
|
|
tokens.append(Token(TT.AMP, string="&"))
|
|
elif c == ";":
|
|
tokens.append(current_token)
|
|
current_token = None
|
|
tokens.append(Token(TT.CARRET, string=";"))
|
|
elif c == ">" or c == "<":
|
|
tokens.append(current_token)
|
|
current_token = None
|
|
tokens.append(Token(TT.CARRET, string=c))
|
|
else:
|
|
current_token.append_char(c)
|
|
elif quote == "'":
|
|
if c == "'":
|
|
tokens.append(current_token)
|
|
current_token = None
|
|
quote = 0
|
|
else:
|
|
if current_token == None:
|
|
current_token = Token(TT.SQUOTE, string="")
|
|
current_token.append_char(c)
|
|
elif quote == '"':
|
|
if c == '"':
|
|
tokens.append(current_token)
|
|
current_token = None
|
|
quote = 0
|
|
else:
|
|
if current_token == None:
|
|
current_token = Token(TT.DQUOTE, string="")
|
|
current_token.append_char(c)
|
|
else:
|
|
print("you fucked up you quote thingy")
|
|
i += 1
|
|
if current_token != None and current_token.ty == TT.NQUOTE:
|
|
tokens.append(current_token)
|
|
return tokens
|