aboutsummaryrefslogtreecommitdiffstats
path: root/src/tokenizer.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/tokenizer.rs')
-rw-r--r--src/tokenizer.rs10
1 files changed, 7 insertions, 3 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index f1f0658..efea700 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -1,5 +1,5 @@
use std::fs;
-use logos::Logos;
+use logos::{Logos, Span};
#[derive(Logos, Debug, PartialEq)]
pub enum Token {
@@ -26,6 +26,10 @@ pub enum Token {
#[token("loop")]
Loop,
+ // TODO(ed): Remove
+ #[token("print")]
+ Print,
+
#[token("+")]
Plus,
#[token("++")]
@@ -94,8 +98,8 @@ pub enum Token {
Error,
}
-pub fn file_to_tokens(filename: &str) -> Vec<Token> {
+pub fn file_to_tokens(filename: &str) -> Vec<(Token, Span)> {
let content = fs::read_to_string(filename).unwrap();
let lexer = Token::lexer(&content);
- lexer.collect()
+ lexer.spanned().collect()
}