diff options
Diffstat (limited to 'src/tokenizer.rs')
| -rw-r--r-- | src/tokenizer.rs | 10 |
1 files changed, 7 insertions, 3 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs index f1f0658..efea700 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1,5 +1,5 @@ use std::fs; -use logos::Logos; +use logos::{Logos, Span}; #[derive(Logos, Debug, PartialEq)] pub enum Token { @@ -26,6 +26,10 @@ pub enum Token { #[token("loop")] Loop, + // TODO(ed): Remove + #[token("print")] + Print, + #[token("+")] Plus, #[token("++")] @@ -94,8 +98,8 @@ pub enum Token { Error, } -pub fn file_to_tokens(filename: &str) -> Vec<Token> { +pub fn file_to_tokens(filename: &str) -> Vec<(Token, Span)> { let content = fs::read_to_string(filename).unwrap(); let lexer = Token::lexer(&content); - lexer.collect() + lexer.spanned().collect() } |
