1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
|
use std::fs;
use logos::{Logos, Span};
#[derive(Logos, Debug, PartialEq, Clone)]
pub enum Token {
#[regex(r"[[:alpha:]][[:alnum:]]*", |lex| lex.slice().to_string())]
Identifier(String),
#[regex(r#""[^"]*""#, |lex| lex.slice().to_string())]
String(String),
#[regex(r"[\d]+\.[\d]*|[\d]*\.[\d]+", |lex| lex.slice().parse(), priority=2)]
Float(f64),
#[regex(r"[\d]+", |lex| lex.slice().parse())]
Int(i64),
#[regex(r"true|false", |lex| lex.slice().parse(), priority=2)]
Bool(bool),
#[token("if")]
If,
#[token("for")]
For,
#[token("in")]
In,
#[token("loop")]
Loop,
// TODO(ed): Remove
#[token("print")]
Print,
#[token("+")]
Plus,
#[token("++")]
PlusPlus,
#[token("-")]
Minus,
#[token("--")]
MinusMinus,
#[token("*")]
Star,
#[token("/")]
Slash,
#[token("+=")]
PlusEqual,
#[token("-=")]
MinusEqual,
#[token("*=")]
StarEqual,
#[token("/=")]
SlashEqual,
#[token(":")]
Colon,
#[token("::")]
ColonColon,
#[token("=")]
Equal,
#[token("==")]
EqualEqual,
#[token("(")]
LeftParen,
#[token(")")]
RightParen,
#[token("[")]
LeftBracket,
#[token("]")]
RightBracket,
#[token("{")]
LeftBrace,
#[token("}")]
RightBrace,
#[token(">")]
Greater,
#[token(">=")]
GreaterEqual,
#[token("<")]
Less,
#[token("<=")]
LessEqual,
#[token(".")]
Dot,
#[token("->")]
Arrow,
#[token("\n")]
Newline,
#[regex(r"[ \t\r]", logos::skip)]
Whitespace,
EOF,
#[error]
Error,
}
pub type PlacedToken = (Token, Span);
pub type TokenStream = Vec<PlacedToken>;
pub fn file_to_tokens(filename: &str) -> TokenStream {
let content = fs::read_to_string(filename).unwrap();
let lexer = Token::lexer(&content);
lexer.spanned().collect()
}
|
