83 lines
2 KiB
Python
83 lines
2 KiB
Python
from enum import Enum
|
|
from dataclasses import dataclass
|
|
|
|
TokenType = Enum(
|
|
"TokenType",
|
|
[
|
|
"AMP",
|
|
"AND",
|
|
"CARRET",
|
|
"DOLLAR",
|
|
"DQUOTE",
|
|
"EXPENSION",
|
|
"LCARRET",
|
|
"LCARRET_DOUBLE",
|
|
"LPAREN",
|
|
"NQUOTE",
|
|
"OR",
|
|
"PIPE",
|
|
"RCARRET",
|
|
"RCARRET_DOUBLE",
|
|
"RPAREN",
|
|
"SEMICOLON",
|
|
"SQUOTE",
|
|
"WHITESPACE",
|
|
"WORD",
|
|
],
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class Token:
|
|
ty: TokenType
|
|
string: str = None
|
|
subtokens: list = None
|
|
|
|
def is_subtoken(self) -> bool:
|
|
return self.subtokens != None
|
|
|
|
def append_char(self, c: str):
|
|
if self.string is None:
|
|
raise Exception(f"Tried to push a char on a token that contains subtokens, TT={self.ty}")
|
|
self.string += c
|
|
|
|
def is_word(self):
|
|
return (
|
|
self.ty == TokenType.SQUOTE
|
|
or self.ty == TokenType.DQUOTE
|
|
or self.ty == TokenType.NQUOTE
|
|
or self.ty == TokenType.DOLLAR
|
|
)
|
|
|
|
|
|
def print_tokenlist(tokens: list[Token], *, between="", end="\n"):
|
|
for tok in tokens:
|
|
col = "0"
|
|
if tok.ty == TokenType.SQUOTE:
|
|
col = "33"
|
|
if tok.ty == TokenType.DQUOTE:
|
|
col = "32"
|
|
if tok.ty == TokenType.WHITESPACE:
|
|
col = "31;4"
|
|
if tok.ty == TokenType.DOLLAR:
|
|
col = "31"
|
|
if tok.ty == TokenType.LPAREN:
|
|
col = "35"
|
|
if tok.ty == TokenType.RPAREN:
|
|
col = "35"
|
|
if tok.ty == TokenType.AMP:
|
|
col = "35"
|
|
if tok.ty == TokenType.PIPE:
|
|
col = "35"
|
|
if tok.ty == TokenType.SEMICOLON:
|
|
col = "35"
|
|
if tok.ty == TokenType.CARRET:
|
|
col = "35"
|
|
if tok.is_subtoken():
|
|
print_tokenlist(tok.subtokens, between="\x1b[100m", end="")
|
|
else:
|
|
print(f"\x1b[{col}m{between}{tok.string}\x1b[0m", end="")
|
|
#print(end)
|
|
|
|
|
|
__all__ = ["TokenType", "Token", "print_tokenlist"]
|