update: make something that collapses tokens into one if it can

This commit is contained in:
maix0 2024-09-26 22:20:24 +02:00
parent 2e811bcec2
commit 774f374965
5 changed files with 81 additions and 54 deletions

View file

@ -1,10 +1,12 @@
import collapse
import concat
import str_to_token
import concat
import ttoken
s = input("> ")
print(s)
tokens = str_to_token.str_to_token(s)
concated_tokens = concat.concat(tokens)
collapsed_tokens = collapse.collapse(concated_tokens)
ttoken.print_tokenlist(concated_tokens)
ttoken.print_tokenlist(collapsed_tokens)