diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/compiler.rs | 788 | ||||
| -rw-r--r-- | src/lib.rs | 618 | ||||
| -rw-r--r-- | src/sectionizer.rs | 105 | ||||
| -rw-r--r-- | src/tokenizer.rs | 3 | ||||
| -rw-r--r-- | src/vm.rs | 135 |
5 files changed, 807 insertions, 842 deletions
diff --git a/src/compiler.rs b/src/compiler.rs index 124606e..dc3bd96 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -1,11 +1,12 @@ -use std::{borrow::Cow, path::{Path, PathBuf}}; +use std::path::{Path, PathBuf}; use std::cell::RefCell; use std::collections::{HashMap, hash_map::Entry}; use std::rc::Rc; use crate::{Blob, Block, Op, Prog, RustFunction, Type, Value}; use crate::error::{Error, ErrorKind}; -use crate::tokenizer::{Token, TokenStream}; +use crate::sectionizer::Section; +use crate::tokenizer::Token; macro_rules! nextable_enum { ( $name:ident { $( $thing:ident ),* $( , )? } ) => { @@ -39,34 +40,37 @@ macro_rules! expect { }; } +// NOTE(ed): This can cause some strange bugs. It would be nice if we could +// remove this function, but to do that we need to rewrite some code. Like +// tuples and fields. Might be worth it tough. macro_rules! parse_branch { ($compiler:expr, $block:expr, [ $( $call:expr ),* ]) => { { let block_length = $block.ops.len(); - let token_length = $compiler.curr; + let token_length = $compiler.current_token; let num_errors = $compiler.errors.len(); let mut stored_errors = Vec::new(); // Closures for early return on success. - let success = (|| { + let success = loop { // We risk getting a lot of errors if we are in an invalid state // when we start the parse. if $compiler.panic { - return false; + break false; } $( $call; if !$compiler.panic { - return true; + break true; } $compiler.panic = false; - $compiler.curr = token_length; + $compiler.current_token = token_length; let thrown_errors = $compiler.errors.len() - num_errors - 1; stored_errors.extend($compiler.errors.split_off(thrown_errors)); $block.ops.truncate(block_length); )* - false - })(); + break false; + }; if !success { $compiler.errors.extend(stored_errors); @@ -79,31 +83,102 @@ macro_rules! parse_branch { ($compiler:expr, $block:expr, $call:expr) => { { let block_length = $block.ops.len(); - let token_length = $compiler.curr; + let token_length = $compiler.current_token; let num_errors = $compiler.errors.len(); let mut stored_errors = Vec::new(); // Closures for early return on success. - (|| { + loop { // We risk getting a lot of errors if we are in an invalid state // when we start the parse. if $compiler.panic { - return false; + break false; } $call; if !$compiler.panic { - return true; + break true; } $compiler.panic = false; - $compiler.curr = token_length; + $compiler.current_token = token_length; let thrown_errors = $compiler.errors.len() - num_errors - 1; stored_errors.extend($compiler.errors.split_off(thrown_errors)); $block.ops.truncate(block_length); - false - })() + break false; + } + } + }; +} + +macro_rules! push_frame { + ($compiler:expr, $block:expr, $code:tt) => { + { + $compiler.frames_mut().push(Frame::new()); + + // Return value stored as a variable + let var = Variable::new("", true, Type::Unknown); + $compiler.define(var).unwrap(); + + $code + + let frame = $compiler.frames_mut().pop().unwrap(); + // 0-th slot is the function itself. + for var in frame.stack.iter().skip(1) { + if !(var.read || var.upvalue) { + let e = ErrorKind::SyntaxError( + var.line, + Token::Identifier(var.name.clone() + )); + $compiler.error_on_line( + e, + var.line, + Some(format!("Unused value '{}'.", var.name)) + ); + } + $compiler.panic = false; + } + // The 0th slot is the return value, which is passed out + // from functions, and should not be popped. + 0 } }; } +macro_rules! push_scope { + ($compiler:expr, $block:expr, $code:tt) => { + let ss = $compiler.stack().len(); + $compiler.frame_mut().scope += 1; + + $code; + + $compiler.frame_mut().scope -= 1; + + let mut errors = Vec::new(); + for var in $compiler.frame().stack.iter().skip(ss).rev() { + if !(var.read || var.upvalue) { + let e = ErrorKind::SyntaxError( + var.line, + Token::Identifier(var.name.clone() + )); + errors.push(( + e, + var.line, + format!("Usage of undefined value: '{}'.", var.name),) + ); + } + if var.captured { + add_op($compiler, $block, Op::PopUpvalue); + } else { + add_op($compiler, $block, Op::Pop); + } + } + + for (e, l, m) in errors.iter() { + $compiler.error_on_line(e.clone(), *l, Some(m.clone())); + $compiler.panic = false; + } + $compiler.stack_mut().truncate(ss); + }; +} + nextable_enum!(Prec { No, Assert, @@ -152,11 +227,13 @@ impl Variable { } } +#[derive(Debug)] enum LoopOp { Continue, Break, } +#[derive(Debug)] struct Frame { loops: Vec<Vec<(usize, usize, LoopOp)>>, stack: Vec<Variable>, @@ -208,6 +285,17 @@ impl Frame { } } + fn find_outer(&self, name: &str) -> Option<Variable> { + // Only really makes sense in the outermost frame + // where declaration order doesn't matter + for var in self.stack.iter().rev() { + if var.name == name { + return Some(var.clone()); + } + } + None + } + fn find_local(&self, name: &str) -> Option<Variable> { for var in self.stack.iter().rev() { if var.name == name && var.active { @@ -240,94 +328,48 @@ impl Frame { } } -pub(crate) struct Compiler { - curr: usize, - tokens: TokenStream, - current_file: PathBuf, +type Namespace = HashMap<String, Name>; +struct CompilerContext { frames: Vec<Frame>, - - panic: bool, - errors: Vec<Error>, - - blocks: Vec<Rc<RefCell<Block>>>, - blob_id: usize, - unknown: HashMap<String, (usize, usize)>, - - functions: HashMap<String, (usize, RustFunction)>, - constants: Vec<Value>, - strings: Vec<String>, + namespace: Namespace, } -macro_rules! push_frame { - ($compiler:expr, $block:expr, $code:tt) => { - { - $compiler.frames.push(Frame::new()); +impl CompilerContext { + fn new() -> Self { + Self { + frames: vec![Frame::new()], + namespace: Namespace::new(), + } + } +} - // Return value stored as a variable - let var = Variable::new("", true, Type::Unknown); - $compiler.define(var).unwrap(); +#[derive(Debug, Clone)] +enum Name { + Slot(usize, usize), + Unknown(usize, usize), + Namespace(PathBuf), +} - $code +pub(crate) struct Compiler { + current_token: usize, + current_section: usize, + sections: Vec<Section>, - let frame = $compiler.frames.pop().unwrap(); - // 0-th slot is the function itself. - for var in frame.stack.iter().skip(1) { - if !(var.read || var.upvalue) { - let e = ErrorKind::SyntaxError( - var.line, - Token::Identifier(var.name.clone() - )); - $compiler.error_on_line( - e, - var.line, - Some(format!("Unused value '{}'.", var.name)) - ); - } - $compiler.panic = false; - } - // The 0th slot is the return value, which is passed out - // from functions, and should not be popped. - 0 - } - }; -} + contextes: HashMap<PathBuf, CompilerContext>, -macro_rules! push_scope { - ($compiler:expr, $block:expr, $code:tt) => { - let ss = $compiler.stack().len(); - $compiler.frame_mut().scope += 1; + panic: bool, + errors: Vec<Error>, - $code; + blocks: Vec<Rc<RefCell<Block>>>, + blob_id: usize, - $compiler.frame_mut().scope -= 1; + functions: HashMap<String, (usize, RustFunction)>, - let mut errors = Vec::new(); - for var in $compiler.frame().stack.iter().skip(ss).rev() { - if !(var.read || var.upvalue) { - let e = ErrorKind::SyntaxError( - var.line, - Token::Identifier(var.name.clone() - )); - errors.push(( - e, - var.line, - format!("Usage of undefined value: '{}'.", var.name),) - ); - } - if var.captured { - add_op($compiler, $block, Op::PopUpvalue); - } else { - add_op($compiler, $block, Op::Pop); - } - } + strings: Vec<String>, - for (e, l, m) in errors.iter() { - $compiler.error_on_line(e.clone(), *l, Some(m.clone())); - $compiler.panic = false; - } - $compiler.stack_mut().truncate(ss); - }; + constants: Vec<Value>, + values: HashMap<Value, usize>, } /// Helper function for adding operations to the given block. @@ -336,36 +378,32 @@ fn add_op(compiler: &Compiler, block: &mut Block, op: Op) -> usize { } impl Compiler { - pub(crate) fn new(current_file: &Path, tokens: TokenStream) -> Self { + pub(crate) fn new(sections: Vec<Section>) -> Self { + let contextes = sections + .iter() + .map(|section| (section.path.to_path_buf(), CompilerContext::new())) + .collect(); + Self { - curr: 0, - tokens, - current_file: PathBuf::from(current_file), + current_token: 0, + current_section: 0, + sections, - frames: vec![Frame::new()], + contextes, panic: false, errors: vec![], blocks: Vec::new(), blob_id: 0, - unknown: HashMap::new(), functions: HashMap::new(), - constants: vec![Value::Nil], strings: Vec::new(), - } - } - fn nil_value(&self) -> usize { - self.constants.iter() - .enumerate() - .find_map(|(i, x)| - match x { - Value::Nil => Some(i), - _ => None, - }).unwrap() + constants: vec![], + values: HashMap::new(), + } } fn new_blob_id(&mut self) -> usize { @@ -374,9 +412,39 @@ impl Compiler { id } + fn add_namespace(&mut self, name: String) { + let path = Path::new(&format!("{}.sy", name)).to_path_buf(); + match self.names_mut().entry(name.clone()) { + Entry::Vacant(v) => { + v.insert(Name::Namespace(path)); + } + Entry::Occupied(_) => { + error!(self, format!("Namespace {} already present.", name)); + } + } + } + fn add_constant(&mut self, value: Value) -> usize { - self.constants.push(value); - self.constants.len() - 1 + if matches!(value, Value::Float(_) + | Value::Int(_) + | Value::Bool(_) + | Value::String(_) + | Value::Tuple(_) + | Value::Nil) + { + let entry = self.values.entry(value.clone()); + if let Entry::Occupied(entry) = entry { + *entry.get() + } else { + let slot = self.constants.len(); + self.constants.push(value); + entry.or_insert(slot); + slot + } + } else { + self.constants.push(value); + self.constants.len() - 1 + } } fn intern_string(&mut self, string: String) -> usize { @@ -384,14 +452,45 @@ impl Compiler { self.strings.len() - 1 } + fn section(&self) -> &Section { + &self.sections[self.current_section] + } + + fn current_file(&self) -> &Path { + &self.section().path + } + + fn current_context(&self) -> &CompilerContext { + self.contextes.get(self.current_file()).unwrap() + } + + fn current_context_mut(&mut self) -> &mut CompilerContext { + let file = self.current_file().to_path_buf(); + self.contextes.get_mut(&file).unwrap() + } + fn frame(&self) -> &Frame { - let last = self.frames.len() - 1; - &self.frames[last] + self.current_context().frames.last().unwrap() } fn frame_mut(&mut self) -> &mut Frame { - let last = self.frames.len() - 1; - &mut self.frames[last] + self.current_context_mut().frames.last_mut().unwrap() + } + + fn frames(&self) -> &[Frame] { + &self.current_context().frames + } + + fn frames_mut(&mut self) -> &mut Vec<Frame> { + &mut self.current_context_mut().frames + } + + fn names(&self) -> &Namespace { + &self.current_context().namespace + } + + fn names_mut(&mut self) -> &mut Namespace { + &mut self.current_context_mut().namespace } /// Marks a variable as read. Also marks upvalues. @@ -401,16 +500,16 @@ impl Compiler { return; } - - if if let Some(up) = self.frames[frame_id].upvalues.get(var.slot) { - up.name == var.name - } else { false } { - let mut inner_var = self.frames[frame_id].upvalues[var.slot].clone(); - inner_var.slot = inner_var.outer_slot; - self.mark_read(frame_id - 1, &inner_var); - self.frames[frame_id].upvalues[var.slot].read = true; - } else { - self.frames[frame_id].stack[var.slot].read = true; + match self.frames()[frame_id].upvalues.get(var.slot) { + Some(up) if up.name == var.name => { + let mut inner_var = self.frames()[frame_id].upvalues[var.slot].clone(); + inner_var.slot = inner_var.outer_slot; + self.mark_read(frame_id - 1, &inner_var); + self.frames_mut()[frame_id].upvalues[var.slot].read = true; + } + _ => { + self.frames_mut()[frame_id].stack[var.slot].read = true; + } } } @@ -446,22 +545,26 @@ impl Compiler { self.panic = true; self.errors.push(Error { kind, - file: self.current_file.clone(), + file: self.current_file().to_path_buf(), line, message, }); } + fn init_section(&mut self, section: usize) { + self.current_token = 0; + self.current_section = section; + } fn peek(&self) -> Token { self.peek_at(0) } fn peek_at(&self, at: usize) -> Token { - if self.tokens.len() <= self.curr + at { + if self.section().tokens.len() <= self.current_token + at { crate::tokenizer::Token::EOF } else { - self.tokens[self.curr + at].0.clone() + self.section().tokens[self.current_token + at].0.clone() } } @@ -472,12 +575,12 @@ impl Compiler { fn eat(&mut self) -> Token { let t = self.peek(); - self.curr += 1; + self.current_token += 1; match t { Token::GitConflictBegin => { - self.curr -= 1; + self.current_token -= 1; let start = self.line(); - self.curr += 1; + self.current_token += 1; while !matches!(self.eat(), Token::GitConflictEnd) {} self.panic = false; self.error_on_line(ErrorKind::GitConflictError(start, self.line()), start, None); @@ -490,10 +593,11 @@ impl Compiler { /// The line of the current token. fn line(&self) -> usize { - if self.curr < self.tokens.len() { - self.tokens[self.curr].1 + if self.section().tokens.len() == 0 { + // unreachable!("An error occured without a section."); + 666666 } else { - self.tokens[self.tokens.len() - 1].1 + self.section().tokens[std::cmp::min(self.current_token, self.section().tokens.len() - 1)].1 } } @@ -679,8 +783,15 @@ impl Compiler { } } - fn find_and_capture_variable<'a, I>(name: &str, mut iterator: I) -> Option<Variable> - where I: Iterator<Item = &'a mut Frame> { + fn find_namespace(&self, name: &str) -> Option<&Namespace> { + match self.names().get(name) { + Some(Name::Namespace(path)) => Some(&self.contextes.get(path).unwrap().namespace), + _ => None, + } + } + + fn find_and_capture_variable<'i, I>(name: &str, mut iterator: I) -> Option<Variable> + where I: Iterator<Item = &'i mut Frame> { if let Some(frame) = iterator.next() { if let Some(res) = frame.find_local(name) { frame.stack[res.slot].captured = true; @@ -710,22 +821,73 @@ impl Compiler { return Some(res); } - Self::find_and_capture_variable(name, self.frames.iter_mut().rev()) + Self::find_and_capture_variable(name, self.frames_mut().iter_mut().rev()) } fn find_constant(&mut self, name: &str) -> usize { - let res = self.constants.iter().enumerate().find_map(|(i, x)| match x { - Value::Blob(b) if b.name == name => Some(i), - Value::Function(_, b) if b.borrow().name == name => Some(i), - _ => None, - }); - if let Some(res) = res { - return res; + match self.names_mut().entry(name.to_string()) { + Entry::Occupied(entry) => { + match entry.get() { + Name::Slot(i, _) => { return *i; }, + Name::Unknown(i, _) => { return *i; }, + _ => { + error!(self, format!("Tried to find constant '{}' but it was a namespace.", name)); + return 0; + } + } + }, + Entry::Vacant(_) => {}, + }; + + let slot = self.add_constant(Value::Unknown); + let line = self.line(); + self.names_mut().insert(name.to_string(), Name::Unknown(slot, line)); + slot + } + + fn named_constant(&mut self, name: String, value: Value) -> usize { + let line = self.line(); + match self.names_mut().entry(name.clone()) { + Entry::Occupied(mut entry) => { + let slot = if let Name::Unknown(slot, _) = entry.get() { + *slot + } else { + error!(self, format!("Constant named \"{}\" already has a value.", name)); + return 0; + }; + entry.insert(Name::Slot(slot, line)); + self.constants[slot] = value; + return slot; + }, + Entry::Vacant(_) => {}, } - let constant = self.add_constant(Value::Nil); + let slot = self.add_constant(value); + self.names_mut().insert(name, Name::Slot(slot, line)); + slot + } + + fn forward_constant(&mut self, name: String) -> usize { let line = self.line(); - let entry = self.unknown.entry(name.to_string()); - entry.or_insert((constant, line)).0 + let slot = self.add_constant(Value::Unknown); + match self.names_mut().entry(name.clone()) { + Entry::Occupied(_) => { + error!(self, format!("Constant named \"{}\" already has a value.", name)); + 0 + }, + Entry::Vacant(entry) => { + entry.insert(Name::Unknown(slot, line)); + slot + }, + } + } + + fn call_maybe(&mut self, block: &mut Block) -> bool { + if matches!(self.peek(), Token::Bang | Token::LeftParen) { + self.call(block); + true + } else { + false + } } fn call(&mut self, block: &mut Block) { @@ -793,25 +955,21 @@ impl Compiler { } // TODO(ed): de-complexify - fn function(&mut self, block: &mut Block, name: Option<&str>) { + fn function(&mut self, block: &mut Block, in_name: Option<&str>) { expect!(self, Token::Fn, "Expected 'fn' at start of function."); - let top = self.stack().len() - 1; - let name = if let Some(name) = name { - Cow::Owned(String::from(name)) - } else if !self.stack()[top].active { - self.stack_mut()[top].active = true; - Cow::Borrowed(&self.stack()[top].name) + let name = if let Some(name) = in_name { + String::from(name) } else { - Cow::Owned(format!("λ {}@{:03}", self.current_file.display(), self.line())) + format!("λ {}@{:03}", self.current_file().display(), self.line()) }; let mut args = Vec::new(); let mut return_type = Type::Void; - let mut function_block = Block::new(&name, &self.current_file); + let mut function_block = Block::new(&name, self.current_file()); let block_id = self.blocks.len(); - let temp_block = Block::new(&name, &self.current_file); + let temp_block = Block::new(&name, self.current_file()); self.blocks.push(Rc::new(RefCell::new(temp_block))); let _ret = push_frame!(self, function_block, { @@ -822,7 +980,8 @@ impl Compiler { expect!(self, Token::Colon, "Expected ':' after parameter name."); if let Ok(typ) = self.parse_type() { args.push(typ.clone()); - let var = Variable::new(&name, true, typ); + let mut var = Variable::new(&name, true, typ); + var.read = true; if let Ok(slot) = self.define(var) { self.stack_mut()[slot].active = true; } @@ -859,12 +1018,13 @@ impl Compiler { } }); + let nil = self.add_constant(Value::Nil); for op in function_block.ops.iter().rev() { match op { Op::Pop | Op::PopUpvalue => {} Op::Return => { break; } , _ => { - add_op(self, &mut function_block, Op::Constant(self.nil_value())); + add_op(self, &mut function_block, Op::Constant(nil)); add_op(self, &mut function_block, Op::Return); break; } @@ -872,28 +1032,65 @@ impl Compiler { } if function_block.ops.is_empty() { - add_op(self, &mut function_block, Op::Constant(self.nil_value())); + add_op(self, &mut function_block, Op::Constant(nil)); add_op(self, &mut function_block, Op::Return); } function_block.ty = Type::Function(args, Box::new(return_type)); let function_block = Rc::new(RefCell::new(function_block)); - // Note(ed): We deliberately add the constant as late as possible. // This behaviour is used in `constant_statement`. let function = Value::Function(Vec::new(), Rc::clone(&function_block)); self.blocks[block_id] = function_block; - let constant = self.add_constant(function); + let constant = if in_name.is_some() { + self.named_constant(name, function) + } else { + self.add_constant(function) + }; add_op(self, block, Op::Constant(constant)); } fn variable_expression(&mut self, block: &mut Block) { - let name = match self.eat() { + let name = match self.peek() { Token::Identifier(name) => name, _ => unreachable!(), }; + if let Some(_) = self.find_namespace(&name) { + self.eat(); + // TODO(ed): This is a clone I would love to get rid of... + let mut namespace = self.find_namespace(&name).unwrap().clone(); + loop { + if self.eat() != Token::Dot { + error!(self, "Expect '.' after namespace."); + return; + } + if let Token::Identifier(field) = self.eat() { + match namespace.get(&field) { + Some(Name::Slot(slot, _)) | Some(Name::Unknown(slot, _)) => { + add_op(self, block, Op::Constant(*slot)); + self.call_maybe(block); + return; + } + Some(Name::Namespace(inner_name)) => { + namespace = self.contextes + .get(inner_name) + .unwrap().namespace + .clone(); + } + _ => { + error!(self, "Invalid namespace field."); + } + } + } else { + error!(self, "Expected fieldname after '.'."); + } + } + } + + self.eat(); + // Global functions take precedence if let Some(slot) = self.find_extern_function(&name) { let string = self.add_constant(Value::ExternFunction(slot)); @@ -904,7 +1101,7 @@ impl Compiler { // Variables if let Some(var) = self.find_variable(&name) { - self.mark_read(self.frames.len() - 1, &var); + self.mark_read(self.frames().len() - 1, &var); if var.upvalue { add_op(self, block, Op::ReadUpvalue(var.slot)); } else { @@ -923,7 +1120,7 @@ impl Compiler { } } _ => { - if !parse_branch!(self, block, self.call(block)) { + if !self.call_maybe(block) { return; } } @@ -934,68 +1131,85 @@ impl Compiler { // Blobs - Always returns a blob since it's filled in if it isn't used. let con = self.find_constant(&name); add_op(self, block, Op::Constant(con)); - parse_branch!(self, block, self.call(block)); + self.call_maybe(block); } fn define(&mut self, mut var: Variable) -> Result<usize, ()> { - if let Some(var) = self.find_variable(&var.name) { - if var.scope == self.frame().scope { + let frame = self.frame(); + + if let Some(res) = frame.find_local(&var.name).or(frame.find_upvalue(&var.name)) { + if res.scope == frame.scope { error!(self, format!("Multiple definitions of '{}' in this block.", - var.name)); + res.name)); return Err(()); } } let slot = self.stack().len(); var.slot = slot; - var.scope = self.frame().scope; + var.scope = frame.scope; var.line = self.line(); self.stack_mut().push(var); Ok(slot) } fn definition_statement(&mut self, name: &str, typ: Type, block: &mut Block) { - let var = Variable::new(name, true, typ.clone()); - let slot = self.define(var); - self.expression(block); - let constant = self.add_constant(Value::Ty(typ)); - add_op(self, block, Op::Define(constant)); + if self.frames().len() <= 1 { + // Global + let var = self.frame().find_outer(name) + .expect(&format!("Couldn't find variable '{}' during prepass.", name)); + assert!(var.mutable); + + self.expression(block); + self.stack_mut()[var.slot].active = true; + } else { + // Local + let var = Variable::new(name, true, typ.clone()); + let slot = self.define(var); + self.expression(block); + let constant = self.add_constant(Value::Ty(typ)); + add_op(self, block, Op::Define(constant)); - if let Ok(slot) = slot { - self.stack_mut()[slot].active = true; + if let Ok(slot) = slot { + self.stack_mut()[slot].active = true; + } } } fn constant_statement(&mut self, name: &str, typ: Type, block: &mut Block) { // Magical global constants - if self.frames.len() <= 1 { - if parse_branch!(self, block, self.function(block, Some(name))) { - // Remove the function, since it's a constant and we already - // added it. - block.ops.pop().unwrap(); - let slot = if let Entry::Occupied(entry) = self.unknown.entry(String::from(name)) { - let (_, (slot, _)) = entry.remove_entry(); - self.constants[slot] = self.constants.pop().unwrap(); - slot - } else { - self.constants.len() - 1 - }; - add_op(self, block, Op::Link(slot)); - if let Value::Function(_, block) = &self.constants[slot] { - block.borrow_mut().mark_constant(); - } else { - unreachable!(); - } - return; + if self.frames().len() <= 1 && self.peek() == Token::Fn { + self.function(block, Some(name)); + // Remove the function, since it's a constant and we already + // added it. + block.ops.pop().unwrap(); + let slot = self.find_constant(name); + add_op(self, block, Op::Link(slot)); + if let Value::Function(_, block) = &self.constants[slot] { + block.borrow_mut().mark_constant(); + } else { + unreachable!(); } + return; } - let var = Variable::new(name, false, typ); - let slot = self.define(var); - self.expression(block); + if self.frames().len() <= 1 { + // Global + let var = self.frame().find_outer(name) + .expect(&format!("Couldn't find constant '{}' during prepass.", name)); + assert!(!var.mutable); - if let Ok(slot) = slot { - self.stack_mut()[slot].active = true; + self.expression(block); + self.stack_mut()[var.slot].active = true; + } else { + // Local + let var = Variable::new(name, false, typ); + let slot = self.define(var); + self.expression(block); + + if let Ok(slot) = slot { + self.stack_mut()[slot].active = true; + } } } @@ -1254,21 +1468,30 @@ impl Compiler { expect!(self, Token::RightBrace, "Expected '}' after 'blob' body. AKA '}'."); let blob = Value::Blob(Rc::new(blob)); - if let Entry::Occupied(entry) = self.unknown.entry(name) { - let (_, (slot, _)) = entry.remove_entry(); - self.constants[slot] = blob; + self.named_constant(name, blob); + } + + fn access_dotted(&mut self, block: &mut Block) { + let name = match self.peek() { + Token::Identifier(name) => name, + _ => unreachable!(), + }; + if let Some(_) = self.find_namespace(&name) { + self.expression(block); } else { - self.constants.push(blob); + parse_branch!(self, block, [self.blob_field(block), self.expression(block)]); } } + //TODO rename fn blob_field(&mut self, block: &mut Block) { let name = match self.eat() { Token::Identifier(name) => name, _ => unreachable!(), }; + if let Some(var) = self.find_variable(&name) { - self.mark_read(self.frames.len() - 1, &var); + self.mark_read(self.frames().len() - 1, &var); if var.upvalue { add_op(self, block, Op::ReadUpvalue(var.slot)); } else { @@ -1316,7 +1539,7 @@ impl Compiler { return; } _ => { - if !parse_branch!(self, block, self.call(block)) { + if !self.call_maybe(block) { error!(self, "Unexpected token when parsing blob-field."); return; } @@ -1329,6 +1552,44 @@ impl Compiler { } } + fn outer_statement(&mut self, block: &mut Block) { + self.clear_panic(); + match self.peek_four() { + (Token::Identifier(name), Token::ColonEqual, ..) => { + self.eat(); + self.eat(); + self.definition_statement(&name, Type::Unknown, block); + }, + + (Token::Identifier(name), Token::ColonColon, ..) => { + self.eat(); + self.eat(); + self.constant_statement(&name, Type::Unknown, block); + }, + + (Token::Blob, Token::Identifier(_), ..) => { + self.blob_statement(block); + }, + + (Token::Identifier(name), Token::Colon, ..) => { + self.eat(); + self.eat(); + if let Ok(typ) = self.parse_type() { + expect!(self, Token::Equal, "Expected assignment."); + self.definition_statement(&name, typ, block); + } else { + error!(self, format!("Expected type found '{:?}'.", self.peek())); + } + } + + (Token::Newline, ..) => {} + + (a, b, c, d) => { + error!(self, format!("Unknown outer token sequence: {:?} {:?} {:?} {:?}.", a, b, c, d)) + } + } + } + fn statement(&mut self, block: &mut Block) { self.clear_panic(); @@ -1350,7 +1611,7 @@ impl Compiler { } (Token::Identifier(_), Token::Dot, ..) => { - parse_branch!(self, block, [self.blob_field(block), self.expression(block)]); + self.access_dotted(block); } (Token::Identifier(name), Token::Colon, ..) => { @@ -1433,42 +1694,117 @@ impl Compiler { add_op(self, block, Op::Pop); } } - } pub(crate) fn compile(&mut self, name: &str, file: &Path, functions: &[(String, RustFunction)]) -> Result<Prog, Vec<Error>> { + let main = Variable::new("/main/", false, Type::Void); + let slot = self.define(main).unwrap(); + self.frame_mut().stack[slot].read = true; + + for section in 0..self.sections.len() { + self.init_section(section); + let section = &mut self.sections[section]; + match (section.tokens.get(0), section.tokens.get(1), section.tokens.get(2)) { + (Some((Token::Use, _)), + Some((Token::Identifier(name), _)), ..) => { + let name = name.to_string(); + self.add_namespace(name); + } + + (Some((Token::Identifier(name), _)), + Some((Token::ColonColon, _)), + Some((Token::Fn, _))) => { + let name = name.to_string(); + self.forward_constant(name); + } + + (Some((Token::Blob, _)), + Some((Token::Identifier(name), _)), ..) => { + let name = name.to_string(); + self.forward_constant(name); + } + + (Some((Token::Identifier(name), _)), + Some((Token::Colon, _)), ..) => { + let name = name.to_string(); + self.eat(); + self.eat(); + if let Ok(ty) = self.parse_type() { + let is_mut = self.peek() == Token::Equal; + let var = Variable::new(&name, is_mut, ty); + let _ = self.define(var).unwrap(); + } else { + error!(self, format!("Failed to parse type global '{}'.", name)); + } + } + + (Some((Token::Identifier(name), _)), + Some((Token::ColonColon, _)), ..) => { + let var = Variable::new(name, false, Type::Unknown); + let _ = self.define(var).unwrap(); + } + + (Some((Token::Identifier(name), _)), + Some((Token::ColonEqual, _)), ..) => { + let var = Variable::new(name, true, Type::Unknown); + let _ = self.define(var).unwrap(); + } + + + (None, ..) => {} + + (a, b, c) => { + section.faulty = true; + let msg = format!("Unknown outer token sequence: {:?} {:?} {:?}. Expected 'use', function, blob or variable.", a, b, c); + error!(self, msg); + } + } + } + self.functions = functions .to_vec() .into_iter() .enumerate() .map(|(i, (s, f))| (s, (i, f))) .collect(); - let main = Variable::new("/main/", false, Type::Void); - let _ = self.define(main); - let mut block = Block::new(name, file); - while self.peek() != Token::EOF { - self.statement(&mut block); - expect!(self, Token::Newline | Token::EOF, - "Expect newline or EOF after expression."); + for section in 0..self.sections.len() { + self.init_section(section); + if self.sections[section].faulty { + continue; + } + while !matches!(self.peek(), Token::EOF | Token::Use) { + self.outer_statement(&mut block); + expect!(self, Token::Newline | Token::EOF, + "Expect newline or EOF after expression."); + } } - add_op(self, &mut block, Op::Constant(self.nil_value())); - add_op(self, &mut block, Op::Return); block.ty = Type::Function(Vec::new(), Box::new(Type::Void)); - if self.unknown.len() != 0 { - let errors: Vec<_> = self.unknown.iter().map(|(name, (_, line))| - (ErrorKind::SyntaxError(*line, Token::Identifier(name.clone())), - *line, - format!("Usage of undefined value: '{}'.", name,) - )) - .collect(); + if self.names().len() != 0 { + let errors: Vec<_> = self.names().iter().filter_map(|(name, kind)| + if let Name::Unknown(_, line) = kind { + Some((ErrorKind::SyntaxError(*line, Token::Identifier(name.clone())), + *line, + format!("Usage of undefined value: '{}'.", name,))) + } else { + None + }) .collect(); for (e, l, m) in errors.iter() { self.error_on_line(e.clone(), *l, Some(m.clone())); } } - for var in self.frames.pop().unwrap().stack.iter().skip(1) { + self.init_section(0); + let constant = self.find_constant("start"); + add_op(self, &mut block, Op::Constant(constant)); + add_op(self, &mut block, Op::Call(0)); + + let tmp = self.add_constant(Value::Unknown); + add_op(self, &mut block, Op::Constant(tmp)); + add_op(self, &mut block, Op::Return); + + for var in self.frames_mut().pop().unwrap().stack.iter().skip(1) { if !(var.read || var.upvalue) { let e = ErrorKind::SyntaxError(var.line, Token::Identifier(var.name.clone())); let m = format!("Unused value '{}'.", var.name); @@ -4,11 +4,11 @@ use std::collections::hash_map::Entry; use std::fmt::Debug; use std::path::{Path, PathBuf}; use std::rc::Rc; +use std::hash::{Hash, Hasher}; use owo_colors::OwoColorize; use error::Error; -use tokenizer::TokenStream; use crate::error::ErrorKind; @@ -16,16 +16,18 @@ pub mod error; pub mod vm; mod compiler; +mod sectionizer; mod tokenizer; /// Compiles a file and links the supplied functions as callable external /// functions. Use this if you want your programs to be able to yield. -pub fn compile_file(path: &Path, - print: bool, - functions: Vec<(String, RustFunction)> - ) -> Result<vm::VM, Vec<Error>> { - let tokens = tokenizer::file_to_tokens(path); - match compiler::Compiler::new(path, tokens).compile("main", path, &functions) { +pub fn compile_file( + path: &Path, + print: bool, + functions: Vec<(String, RustFunction)> +) -> Result<vm::VM, Vec<Error>> { + let sections = sectionizer::sectionize(path); + match compiler::Compiler::new(sections).compile("main", path, &functions) { Ok(prog) => { let mut vm = vm::VM::new(); vm.print_blocks = print; @@ -42,18 +44,19 @@ pub fn compile_file(path: &Path, /// external functions. If you want your program to be able to yield, use /// [compile_file]. pub fn run_file(path: &Path, print: bool, functions: Vec<(String, RustFunction)>) -> Result<(), Vec<Error>> { - run(tokenizer::file_to_tokens(path), path, print, functions) + run(path, print, functions) } -/// Compile and run a string containing source code. The supplied functions are -/// linked as callable external functions. This is useful for short test -/// programs. -pub fn run_string(s: &str, print: bool, functions: Vec<(String, RustFunction)>) -> Result<(), Vec<Error>> { - run(tokenizer::string_to_tokens(s), Path::new("builtin"), print, functions) +pub fn run_string(source: &str, print: bool, functions: Vec<(String, RustFunction)>) -> Result<(), Vec<Error>> { + let mut path = std::env::temp_dir(); + path.push(format!("test_{}.sy", rand::random::<u32>())); + std::fs::write(path.clone(), source).expect("Failed to write source to temporary file"); + run(&path, print, functions) } -fn run(tokens: TokenStream, path: &Path, print: bool, functions: Vec<(String, RustFunction)>) -> Result<(), Vec<Error>> { - match compiler::Compiler::new(path, tokens).compile("main", path, &functions) { +fn run(path: &Path, print: bool, functions: Vec<(String, RustFunction)>) -> Result<(), Vec<Error>> { + let sections = sectionizer::sectionize(path); + match compiler::Compiler::new(sections).compile("main", path, &functions) { Ok(prog) => { let mut vm = vm::VM::new(); vm.print_blocks = print; @@ -198,6 +201,43 @@ impl Debug for Value { } } +impl PartialEq<Value> for Value { + fn eq(&self, other: &Value) -> bool { + match (self, other) { + (Value::Float(a), Value::Float(b)) => a == b, + (Value::Int(a), Value::Int(b)) => a == b, + (Value::Bool(a), Value::Bool(b)) => a == b, + (Value::String(a), Value::String(b)) => a == b, + (Value::Tuple(a), Value::Tuple(b)) => { + a.len() == b.len() && a.iter().zip(b.iter()).all(|(a, b)| a == b) + } + (Value::Nil, Value::Nil) => true, + _ => false, + } + } +} + +impl Eq for Value {} + +impl Hash for Value { + fn hash<H: Hasher>(&self, state: &mut H) { + match self { + Value::Float(a) => { + // We have to limit the values, because + // floats are wierd. + assert!(a.is_finite()); + a.to_bits().hash(state); + }, + Value::Int(a) => a.hash(state), + Value::Bool(a) => a.hash(state), + Value::String(a) => a.hash(state), + Value::Tuple(a) => a.hash(state), + Value::Nil => state.write_i8(0), + _ => {}, + }; + } +} + impl Value { fn identity(self) -> Self { match self { @@ -762,12 +802,6 @@ pub struct Prog { #[cfg(test)] mod tests { - use std::path::Path; - - use crate::error::ErrorKind; - - use super::{run_file, run_string}; - #[macro_export] macro_rules! assert_errs { ($result:expr, [ $( $kind:pat ),* ]) => { @@ -808,6 +842,7 @@ mod tests { use owo_colors::OwoColorize; // Shamelessly stolen from https://github.com/rust-lang/rfcs/issues/2798 + #[allow(dead_code)] pub fn panic_after<T, F>(d: Duration, f: F) -> T where T: Send + 'static, @@ -837,544 +872,27 @@ mod tests { } #[macro_export] - macro_rules! test_string { - ($fn:ident, $prog:literal) => { + macro_rules! test_file { + ($fn:ident, $path:literal, $print:expr) => { #[test] fn $fn() { - crate::tests::panic_after(std::time::Duration::from_millis(500), || { - match $crate::run_string($prog, true, Vec::new()) { - Ok(()) => {}, - Err(errs) => { - for e in errs.iter() { - eprintln!("{}", e); - } - eprintln!(" {} - failed\n", stringify!($fn)); - unreachable!(); - } - } - }); + let file = std::path::Path::new($path); + crate::run_file(&file, $print, Vec::new()).unwrap(); } }; - ($fn:ident, $prog:literal, $errs:tt) => { + ($fn:ident, $path:literal, $print:expr, $errs:tt) => { #[test] fn $fn() { - crate::tests::panic_after(std::time::Duration::from_millis(500), || { - $crate::assert_errs!($crate::run_string($prog, true, Vec::new()), $errs); - }) - } - } - } + use crate::error::ErrorKind; + #[allow(unused_imports)] + use crate::Type; - #[macro_export] - macro_rules! test_file { - ($fn:ident, $path:literal) => { - #[test] - fn $fn() { - let file = Path::new($path); - run_file(&file, true, Vec::new()).unwrap(); + let file = std::path::Path::new($path); + let res = crate::run_file(&file, $print, Vec::new()); + $crate::assert_errs!(res, $errs); } }; } - #[test] - fn unreachable_token() { - assert_errs!(run_string("<!>\n", true, Vec::new()), [ErrorKind::Unreachable]); - } - - #[test] - fn assign_to_constant() { - assert_errs!(run_string("a :: 2\na = 2", true, Vec::new()), [ErrorKind::SyntaxError(_, _)]); - } - - #[test] - fn assign_to_constant_upvalue() { - assert_errs!(run_string("a :: 2\nq :: fn { a = 2 }\nq()\na", true, Vec::new()), [ErrorKind::SyntaxError(_, _)]); - } - - #[test] - fn undefined_blob() { - assert_errs!(run_string("a :: B()\n", true, Vec::new()), [ErrorKind::SyntaxError(_, _)]); - } - - #[test] - fn call_before_link() { - let prog = " -a := 1 -f() -c := 5 - -f :: fn { - c <=> 5 -} -a - "; - assert_errs!(run_string(prog, true, Vec::new()), [ErrorKind::InvalidProgram, ErrorKind::TypeError(_, _)]); - } - - #[test] - fn unused_variable() { - assert_errs!(run_string("a := 1", true, Vec::new()), [ErrorKind::SyntaxError(1, _)]); - } - - #[test] - fn unused_upvalue() { - assert_errs!(run_string("a := 1\nf :: fn { a = 2 }\nf()", true, Vec::new()), [ErrorKind::SyntaxError(1, _)]); - } - - #[test] - fn unused_function() { - assert_errs!(run_string("a := 1\nf := fn { a }\n", true, Vec::new()), [ErrorKind::SyntaxError(2, _)]); - } - - macro_rules! test_multiple { - ($mod:ident, $( $fn:ident : $prog:literal ),+ $( , )? ) => { - mod $mod { - $( test_string!($fn, $prog); )+ - } - } - } - - test_multiple!( - order_of_operations, - terms_and_factors: "1 + 1 * 2 <=> 3 - 1 * 2 + 3 <=> 5", - in_rhs: "5 <=> 1 * 2 + 3", - parenthesis: "(1 + 2) * 3 <=> 9", - negation: "-1 <=> 0 - 1 - -1 + 2 <=> 1 - -(1 + 2) <=> -3 - 1 + -1 <=> 0 - 2 * -1 <=> -2", - ); - - test_multiple!( - variables, - single_variable: "a := 1 - a <=> 1", - two_variables: "a := 1 - b := 2 - a <=> 1 - b <=> 2", - stack_ordering: "a := 1 - b := 2 - b <=> 2 - a <=> 1", - assignment: "a := 1 - b := 2 - a = b - a <=> 2 - b <=> 2", - ); - - test_multiple!( - if_, - compare_constants_equality: "if 1 == 2 { - <!> - }", - compare_constants_unequality: "if 1 != 1 { - <!> - }", - compare_variable: "a := 1 - if a == 0 { - <!> - } - if a != 1 { - <!> - }", - else_: "a := 1 - res := 0 - if a == 0 { - <!> - } else { - res = 1 - } - res <=> 1", - else_if: "a := 1 - res := 0 - if a == 0 { - <!> - } else if a == 1 { - res = 1 - } else { - <!> - } - res <=> 1", - ); - - test_multiple!( - fun, - simplest: "f := fn {} - f()", - param_1: "f := fn a: int {} - f(1)", - return_1: "f := fn -> int { - ret 1 - } - f() <=> 1", - param_and_return: "f := fn a: int -> int { - ret a * 2 - } - f(1) <=> 2 - f(5) <=> 10", - param_2: "add := fn a: int, b: int -> int { - ret a + b - } - add(1, 1) <=> 2 - add(10, 20) <=> 30", - calls_inside_calls: "one := fn -> int { - ret 1 - } - add := fn a: int, b: int -> int { - ret a + b - } - add(one(), one()) <=> 2 - add(add(one(), one()), one()) <=> 3 - add(one(), add(one(), one())) <=> 3", - passing_functions: "g := fn -> int { - ret 1 - } - f := fn inner: fn -> int -> int { - ret inner() - } - f(g) <=> 1", - passing_functions_mixed: "g := fn a: int -> int { - ret a * 2 - } - f := fn inner: fn int -> int, a: int -> int { - ret inner(a) - } - f(g, 2) <=> 4", - multiple_returns: "f := fn a: int -> int { - if a == 1 { - ret 2 - } else { - ret 3 - } - } - f(0) <=> 3 - f(1) <=> 2 - f(2) <=> 3", - precedence: "f := fn a: int, b: int -> int { - ret a + b - } - 1 + f(2, 3) <=> 6 - 2 * f(2, 3) <=> 10 - f(2, 3) - (2 + 3) <=> 0", - factorial: "factorial : fn int -> int = fn n: int -> int { - if n <= 1 { - ret 1 - } - ret n * factorial(n - 1) - } - factorial(5) <=> 120 - factorial(6) <=> 720 - factorial(12) <=> 479001600", - - returning_closures: " -f : fn -> fn -> int = fn -> fn -> int { - x : int = 0 - f := fn -> int { - x = x + 1 - ret x - } - f() <=> 1 - ret f -} - -a := f() -b := f() - -a() <=> 2 -a() <=> 3 - -b() <=> 2 -b() <=> 3 - -a() <=> 4 -", - ); - - test_multiple!( - blob, - simple: "blob A {}", - instantiate: "blob A {} - a := A() - a", - field: "blob A { a: int }", - field_assign: "blob A { a: int } - a := A() - a.a = 2", - field_get: "blob A { a: int } - a := A() - a.a = 2 - a.a <=> 2 - 2 <=> a.a", - multiple_fields: "blob A { - a: int - b: int - } - a := A() - a.a = 2 - a.b = 3 - a.a + a.b <=> 5 - 5 <=> a.a + a.b", - blob_infer: " -blob A { } -a : A = A() -a -", - ); - - test_multiple!(tuples, - add: "(1, 2, 3, 4) + (4, 3, 2, 1) <=> (5, 5, 5, 5)", - sub: "(1, -2, 3, -4) - (4, 3, -2, -1) <=> (-3, 1, 1, -5)", - mul: "(0, 1, 2) * (2, 3, 4) <=> (0, 3, 8)", - types: "a: (int, float, int) = (1, 1., 1)\na", - more_types: "a: (str, bool, int) = (\"abc\", true, 1)\na", - ); - - test_file!(scoping, "progs/tests/scoping.sy"); - test_file!(for_, "progs/tests/for.sy"); - - test_multiple!( - op_assign, - add: "a := 1\na += 1\na <=> 2", - sub: "a := 2\na -= 1\na <=> 1", - mul: "a := 2\na *= 2\na <=> 4", - div: "a := 2\na /= 2\na <=> 1", - cluster: " -blob A { a: int } -a := A() -a.a = 0 -a.a += 1 -a.a <=> 1 -a.a *= 2 -a.a <=> 2 -a.a /= 2 -a.a <=> 1 -a.a -= 1 -a.a <=> 0" - ); - - test_multiple!( - fancy_call, - not: "f := fn {}\n f!\n", - one_arg: "f := fn a:int { a <=> 1 }\n f! 1\n", - two_arg: "f := fn a:int, b:int { b <=> 3 }\n f! 1, 1 + 2\n", - three_arg: "f := fn a:int, b:int, c:int { c <=> 13 }\n f! 1, 1 + 2, 1 + 4 * 3\n", - ); - - test_multiple!( - newline_regression, - simple: "a := 1 // blargh \na += 1 // blargh \n a <=> 2 // HARGH", - expressions: "1 + 1 // blargh \n 2 // blargh \n // HARGH \n", - ); - - test_multiple!( - break_and_continue, - simple_break: " -a := 0 -for i := 0, i < 10, i += 1 { - a = a + 1 - if i == 2 { - break - } -} -a <=> 3 -", - - simple_continue: " -a := 0 -for i := 0, i < 4, i += 1 { - if i == 2 { - continue - } - a = a + 1 -} -a <=> 3 -", - - advanced_break: " -a := 0 -for i := 0, i < 10, i += 1 { - q := 0 - qq := 0 - qqq := 0 - qqqq := 0 - - a = a + 1 - if i == 2 { - break - } -} -a <=> 3 -", - - advanced_continue: " -a := 0 -for i := 0, i < 4, i += 1 { - q := 0 - qq := 0 - qqq := 0 - qqqq := 0 - - if i == 2 { - continue - } - a = a + 1 -} -a <=> 3 -", - ); - - test_multiple!( - read_constants, - simple: " -a :: 1 -a <=> 1 -b := 2 -{ - a <=> 1 - b <=> 2 -}", - ); - - test_multiple!( - assignment_op_regression, - simple_add: " -a := 0 -b := 99999 -a += 1 -a <=> 1 -b <=> 99999 -", - - simple_sub: " -a := 0 -b := 99999 -a -= 1 -a <=> -1 -b <=> 99999 -", - - strange: " -a := 0 -{ - b := 99999 - { - a := 99999 - } - a -= 1 -} -a <=> -1 -", - ); - - test_multiple!( - declaration_order, - blob_simple: " -a := A() -a - -blob A { - a: int -} -", - - blob_complex: " -a := A() -b := B() -c := C() -b2 := B() - -a -b -c -b2 - -blob A { - c: C -} -blob C { } -blob B { } -", - - blob_infer: " -blob A { } - -a : A = A() -a -", - - - constant_function: " -a() -a :: fn {} -", - - constant_function_complex: " -h :: fn -> int { - ret 3 -} - -a() <=> 3 - -k :: fn -> int { - ret h() -} - -a :: fn -> int { - ret q() -} - -q :: fn -> int { - ret k() -} -", - - constant_function_closure: " -q := 1 - -f :: fn -> int { - q += 1 - ret q -} - -f() <=> 2 -f() <=> 3 -f() <=> 4 -f() <=> 5 -", - - constants_in_inner_functions: " -q : int = 0 - -f :: fn -> fn -> { - g :: fn { - q += 1 - } - ret g -} - -g := f() -g() -q <=> 1 -g() -q <=> 2 -g() -q <=> 3 -", - - ); - - test_string!(conflict_markers, " -<<<<<<< HEAD -print extern_test(4.0) -======= -print extern_test(5.0) ->>>>>>> 2 -", - [ErrorKind::SyntaxError(_, _), ErrorKind::GitConflictError(2, 6)] - ); - + sylt_macro::find_tests!(); } diff --git a/src/sectionizer.rs b/src/sectionizer.rs new file mode 100644 index 0000000..8c5e238 --- /dev/null +++ b/src/sectionizer.rs @@ -0,0 +1,105 @@ +use crate::tokenizer::{PlacedToken, Token, file_to_tokens}; + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +pub struct Section { + pub tokens: Vec<PlacedToken>, + pub path: PathBuf, + pub faulty: bool, +} + +impl Section { + fn new(path: PathBuf, tokens: &[PlacedToken]) -> Self { + Self { + tokens: Vec::from(tokens), + path, + faulty: false, + } + } +} + +pub fn sectionize(path: &Path) -> Vec<Section> { + let mut read_files = HashSet::new(); + read_files.insert(path.to_path_buf()); + let tokens = file_to_tokens(path); + let mut all_tokens = vec![(path.to_path_buf(), tokens)]; + let mut sections = Vec::new(); + + let mut i = 0; + while i < all_tokens.len() { + let (path, tokens) = all_tokens[i].clone(); + i += 1; + let mut last = 0; + let mut curr = 0; + while curr < tokens.len() { + if match (tokens.get(curr + 0), tokens.get(curr + 1), tokens.get(curr + 2)) { + (Some((Token::Newline, _)), ..) + => { + if curr == last { + last += 1; + } + false + }, + + (Some((Token::Use, _)), + Some((Token::Identifier(use_file), _)), + Some((Token::Newline, _))) => { + let use_file: PathBuf = format!("{}.sy", use_file).into(); + if !read_files.contains(&use_file) { + let use_file_tokens = file_to_tokens(&use_file); + read_files.insert(use_file.clone()); + all_tokens.push((use_file, use_file_tokens)) + } + true + }, + + (Some((Token::LeftBrace, _)), ..) + => { + let mut blocks = 1; + loop { + curr += 1; + match tokens.get(curr) { + Some((Token::LeftBrace, _)) => { + blocks += 1; + } + + Some((Token::RightBrace, _)) => { + curr += 1; + blocks -= 1; + if blocks <= 0 { + break; + } + } + + None => { + break; + } + + _ => {} + } + } + false + }, + + (Some((Token::Identifier(_), _)), + Some((Token::ColonColon, _)), + Some(_)) + => true, + + (Some((Token::Identifier(_), _)), + Some((Token::ColonEqual, _)), + Some(_)) + => true, + + _ => false, + } { + sections.push(Section::new(path.clone(), &tokens[last..curr])); + last = curr; + } + curr += 1; + } + sections.push(Section::new(path.clone(), &tokens[last..curr])); + } + sections +} diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 2c8e5e8..3b61e5f 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -127,6 +127,9 @@ pub enum Token { #[token("\n")] Newline, + #[token("use")] + Use, + #[token("<<<<<<<")] GitConflictBegin, #[token(">>>>>>>")] @@ -6,7 +6,7 @@ use std::rc::Rc; use owo_colors::OwoColorize; -use crate::{Block, Op, Prog, UpValue, Value, op}; +use crate::{Block, BlockLinkState, Op, Prog, UpValue, Value, op}; use crate::error::{Error, ErrorKind}; use crate::RustFunction; use crate::Type; @@ -236,22 +236,26 @@ impl VM { let offset = self.frame().stack_offset; let constant = self.constant(value).clone(); let value = match constant { - Value::Function(_, block) => { - let mut ups = Vec::new(); - for (slot, is_up, _) in block.borrow().upvalues.iter() { - let up = if *is_up { - if let Value::Function(local_ups, _) = &self.stack[offset] { - Rc::clone(&local_ups[*slot]) + Value::Function(ups, block) => { + if matches!(block.borrow().linking, BlockLinkState::Linked) { + Value::Function(ups.clone(), block) + } else { + let mut ups = Vec::new(); + for (slot, is_up, _) in block.borrow().upvalues.iter() { + let up = if *is_up { + if let Value::Function(local_ups, _) = &self.stack[offset] { + Rc::clone(&local_ups[*slot]) + } else { + unreachable!() + } } else { - unreachable!() - } - } else { - let slot = self.frame().stack_offset + slot; - Rc::clone(self.find_upvalue(slot)) - }; - ups.push(up); + let slot = self.frame().stack_offset + slot; + Rc::clone(self.find_upvalue(slot)) + }; + ups.push(up); + } + Value::Function(ups, block) } - Value::Function(ups, block) }, value => value, }; @@ -553,36 +557,37 @@ impl VM { match self.constant(value).clone() { Value::Function(_, block) => { self.push(Value::Function(Vec::new(), block.clone())); + if !matches!(block.borrow().linking, BlockLinkState::Linked) { + if block.borrow().needs_linking() { + error!(self, + ErrorKind::InvalidProgram, + format!("Calling function '{}' before all captured variables are declared.", + block.borrow().name)); + } - if block.borrow().needs_linking() { - error!(self, - ErrorKind::InvalidProgram, - format!("Calling function '{}' before all captured variables are declared.", - block.borrow().name)); - } - - let mut types = Vec::new(); - for (slot, is_up, ty) in block.borrow().upvalues.iter() { - if *is_up { - types.push(ty.clone()); - } else { - types.push(Type::from(&self.stack[*slot])); + let mut types = Vec::new(); + for (slot, is_up, ty) in block.borrow().upvalues.iter() { + if *is_up { + types.push(ty.clone()); + } else { + types.push(Type::from(&self.stack[*slot])); + } } - } - let mut block_mut = block.borrow_mut(); - for (i, (_, is_up, ty)) in block_mut.upvalues.iter_mut().enumerate() { - if *is_up { continue; } + let mut block_mut = block.borrow_mut(); + for (i, (_, is_up, ty)) in block_mut.upvalues.iter_mut().enumerate() { + if *is_up { continue; } - let suggestion = &types[i]; - if matches!(ty, Type::Unknown) { - *ty = suggestion.clone(); - } else { - if ty != suggestion { - error!(self, ErrorKind::CannotInfer(ty.clone(), suggestion.clone())); + let suggestion = &types[i]; + if matches!(ty, Type::Unknown) { + *ty = suggestion.clone(); + } else { + if ty != suggestion { + error!(self, ErrorKind::CannotInfer(ty.clone(), suggestion.clone())); + } } - } - }; + }; + } }, value => { self.push(value.clone()); @@ -652,6 +657,7 @@ impl VM { let inner = self.frame().block.borrow(); let ret = inner.ret(); if Type::from(&a) != *ret { + error!(self, ErrorKind::TypeMismatch(a.into(), ret.clone()), "Value does not match return type."); } @@ -679,6 +685,29 @@ impl VM { match self.constant(slot).clone() { Value::Function(_, block) => { block.borrow_mut().link(); + + let mut types = Vec::new(); + for (slot, is_up, ty) in block.borrow().upvalues.iter() { + if *is_up { + types.push(ty.clone()); + } else { + types.push(Type::from(&self.stack[*slot])); + } + } + + let mut block_mut = block.borrow_mut(); + for (i, (_, is_up, ty)) in block_mut.upvalues.iter_mut().enumerate() { + if *is_up { continue; } + + let suggestion = &types[i]; + if matches!(ty, Type::Unknown) { + *ty = suggestion.clone(); + } else { + if ty != suggestion { + error!(self, ErrorKind::CannotInfer(ty.clone(), suggestion.clone())); + } + } + } } value => { error!(self, @@ -824,29 +853,3 @@ impl VM { } } } - -#[cfg(test)] -mod tests { - mod typing { - use crate::error::ErrorKind; - use crate::{test_string, Type}; - - test_string!(uncallable_type, " - f := fn i: int { - i() - } - f", - [ErrorKind::InvalidProgram]); - - test_string!(invalid_assign, "a := 1\na = 0.1\na", - [ErrorKind::TypeMismatch(Type::Int, Type::Float)]); - - test_string!(wrong_params, " - f : fn -> int = fn a: int -> int {}\nf", - [ErrorKind::TypeMismatch(_, _), ErrorKind::TypeMismatch(Type::Void, Type::Int)]); - - test_string!(wrong_ret, " - f : fn -> int = fn {}\nf", - [ErrorKind::TypeMismatch(_, _)]); - } -} |