diff options
| -rw-r--r-- | src/compiler.rs | 112 | ||||
| -rw-r--r-- | src/lib.rs | 51 | ||||
| -rw-r--r-- | src/sectionizer.rs | 109 | ||||
| -rw-r--r-- | src/tokenizer.rs | 3 |
4 files changed, 150 insertions, 125 deletions
diff --git a/src/compiler.rs b/src/compiler.rs index fc28b7d..90bc51e 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -1,11 +1,12 @@ -use std::{path::{Path, PathBuf}}; +use std::path::{Path, PathBuf}; use std::cell::RefCell; use std::collections::{HashMap, hash_map::Entry}; use std::rc::Rc; use crate::{Blob, Block, Op, Prog, RustFunction, Type, Value}; use crate::error::{Error, ErrorKind}; -use crate::tokenizer::{Token, PlacedToken, TokenStream}; +use crate::sectionizer::Section; +use crate::tokenizer::Token; macro_rules! nextable_enum { ( $name:ident { $( $thing:ident ),* $( , )? } ) => { @@ -330,31 +331,17 @@ impl<'a> CompilerContext<'a> { } } -struct Section<'a> { - path: PathBuf, - tokens: &'a [PlacedToken], -} - -impl<'a> Section<'a> { - fn new(path: PathBuf, tokens: &'a [PlacedToken]) -> Self { - Section { - path, - tokens, - } - } -} - #[derive(Debug)] enum Name<'a> { Slot(usize, usize), Unknown(usize, usize), - Namespace(&'a Namespace<'a>), + _Namespace(&'a Namespace<'a>), } pub(crate) struct Compiler<'a> { current_token: usize, current_section: usize, - sections: Vec<Section<'a>>, + sections: Vec<Section>, contextes: HashMap<PathBuf, CompilerContext<'a>>, @@ -377,82 +364,13 @@ fn add_op(compiler: &Compiler, block: &mut Block, op: Op) -> usize { block.add(op, compiler.line()) } -fn split_sections<'a>(file_name: PathBuf, tokens: &'a TokenStream) -> Vec<Section> { - let mut sections = Vec::new(); - - let mut last = 0; - let mut curr = 0; - while curr < tokens.len() { - if match (tokens.get(curr + 0), tokens.get(curr + 1), tokens.get(curr + 2)) { - (Some((Token::Newline, _)), ..) - => { - if curr == last { - last += 1; - } - false - }, - - (Some((Token::LeftBrace, _)), ..) - => { - let mut blocks = 0; - loop { - curr += 1; - match tokens.get(curr) { - Some((Token::LeftBrace, _)) => { - blocks += 1; - } - - Some((Token::RightBrace, _)) => { - curr += 1; - blocks -= 1; - if blocks <= 0 { - break; - } - } - - None => { - break; - } - - _ => {} - } - } - false - }, - - (Some((Token::Identifier(_), _)), - Some((Token::ColonColon, _)), - Some((Token::Fn, _))) - => true, - - (Some((Token::Identifier(_), _)), - Some((Token::ColonColon, _)), - Some(_)) - => true, - - (Some((Token::Identifier(_), _)), - Some((Token::ColonEqual, _)), - Some(_)) - => true, - - _ => false, - } { - sections.push(Section::new(file_name.clone(), &tokens[last..curr])); - last = curr; - } - curr += 1; - } - sections.push(Section::new(file_name, &tokens[last..curr])); - sections -} - impl<'a, 'b> Compiler<'a> { - pub(crate) fn new(current_file: &Path, tokens: &'a TokenStream) -> Self { - let current_file = current_file.to_path_buf(); - let sections = split_sections(current_file.clone(), tokens); + pub(crate) fn new(sections: Vec<Section>) -> Self { + let contextes = sections + .iter() + .map(|section| (section.path.to_path_buf(), CompilerContext::new())) + .collect(); - let mut contextes = HashMap::new(); - contextes.insert(current_file, CompilerContext::new()); Self { current_token: 0, current_section: 0, @@ -1701,7 +1619,6 @@ impl<'a, 'b> Compiler<'a> { } pub(crate) fn compile(&'b mut self, name: &str, file: &Path, functions: &[(String, RustFunction)]) -> Result<Prog, Vec<Error>> { - for section in 0..self.sections.len() { self.init_section(section); let section = &self.sections[section]; @@ -1709,21 +1626,24 @@ impl<'a, 'b> Compiler<'a> { (Some((Token::Identifier(name), _)), Some((Token::ColonColon, _)), Some((Token::Fn, _))) => { - self.forward_constant(name.to_string()); + let name = name.to_string(); + self.forward_constant(name); }, (Some((Token::Blob, _)), Some((Token::Identifier(name), _)), ..) => { - self.forward_constant(name.to_string()); + let name = name.to_string(); + self.forward_constant(name); }, (Some((Token::Identifier(name), _)), Some((Token::Colon, _)), ..) => { + let name = name.to_string(); self.eat(); self.eat(); if let Ok(ty) = self.parse_type() { let is_mut = self.peek() == Token::Equal; - let var = Variable::new(name, is_mut, ty); + let var = Variable::new(&name, is_mut, ty); let _ = self.define(var); } else { error!(self, format!("Failed to parse type global '{}'.", name)); @@ -9,7 +9,6 @@ use std::hash::{Hash, Hasher}; use owo_colors::OwoColorize; use error::Error; -use tokenizer::TokenStream; use crate::error::ErrorKind; @@ -17,45 +16,39 @@ pub mod error; pub mod vm; mod compiler; +mod sectionizer; mod tokenizer; /// Compiles a file and links the supplied functions as callable external /// functions. Use this if you want your programs to be able to yield. -pub fn compile_file( - path: &Path, - print: bool, - functions: Vec<(String, RustFunction)> -) -> Result<vm::VM, Vec<Error>> { - let tokens = tokenizer::file_to_tokens(path); - match compiler::Compiler::new(path, &tokens).compile("main", path, &functions) { - Ok(prog) => { - let mut vm = vm::VM::new(); - vm.print_blocks = print; - vm.print_ops = print; - vm.typecheck(&prog)?; - vm.init(&prog); - Ok(vm) - } - Err(errors) => Err(errors), - } -} +//pub fn compile_file( +// path: &Path, +// print: bool, +// functions: Vec<(String, RustFunction)> +//) -> Result<vm::VM, Vec<Error>> { +// match compiler::Compiler::new(path).compile("main", path, &functions) { +// Ok(prog) => { +// let mut vm = vm::VM::new(); +// vm.print_blocks = print; +// vm.print_ops = print; +// vm.typecheck(&prog)?; +// vm.init(&prog); +// Ok(vm) +// } +// Err(errors) => Err(errors), +// } +//} /// Compiles, links and runs the given file. Supplied functions are callable /// external functions. If you want your program to be able to yield, use /// [compile_file]. pub fn run_file(path: &Path, print: bool, functions: Vec<(String, RustFunction)>) -> Result<(), Vec<Error>> { - run(tokenizer::file_to_tokens(path), path, print, functions) -} - -/// Compile and run a string containing source code. The supplied functions are -/// linked as callable external functions. This is useful for short test -/// programs. -pub fn run_string(s: &str, print: bool, functions: Vec<(String, RustFunction)>) -> Result<(), Vec<Error>> { - run(tokenizer::string_to_tokens(s), Path::new("builtin"), print, functions) + run(path, print, functions) } -fn run(tokens: TokenStream, path: &Path, print: bool, functions: Vec<(String, RustFunction)>) -> Result<(), Vec<Error>> { - match compiler::Compiler::new(path, &tokens).compile("main", path, &functions) { +fn run(path: &Path, print: bool, functions: Vec<(String, RustFunction)>) -> Result<(), Vec<Error>> { + let sections = sectionizer::sectionize(path); + match compiler::Compiler::new(sections).compile("main", path, &functions) { Ok(prog) => { let mut vm = vm::VM::new(); vm.print_blocks = print; diff --git a/src/sectionizer.rs b/src/sectionizer.rs new file mode 100644 index 0000000..b7e083f --- /dev/null +++ b/src/sectionizer.rs @@ -0,0 +1,109 @@ +use crate::tokenizer::{PlacedToken, Token, file_to_tokens}; + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +pub struct Section { + pub tokens: Vec<PlacedToken>, + pub path: PathBuf, +} + +impl Section { + fn new(path: PathBuf, tokens: &[PlacedToken]) -> Self { + Self { + tokens: Vec::from(tokens), + path, + } + } +} + +pub fn sectionize(path: &Path) -> Vec<Section> { + let mut read_files = HashSet::new(); + read_files.insert(path.to_path_buf()); + let tokens = file_to_tokens(path); + let mut all_tokens = vec![(path.to_path_buf(), tokens)]; + let mut sections = Vec::new(); + + let mut i = 0; + while i < all_tokens.len() { + let (path, tokens) = all_tokens[i].clone(); + i += 1; + let mut last = 0; + let mut curr = 0; + while curr < tokens.len() { + if match (tokens.get(curr + 0), tokens.get(curr + 1), tokens.get(curr + 2)) { + (Some((Token::Newline, _)), ..) + => { + if curr == last { + last += 1; + } + false + }, + + (Some((Token::Use, _)), + Some((Token::Identifier(use_file), _)), + Some((Token::Newline, _))) => { + curr += 3; + let use_file: PathBuf = format!("{}.sy", use_file).into(); + if !read_files.contains(&use_file) { + let use_file_tokens = file_to_tokens(&use_file); + read_files.insert(use_file.clone()); + all_tokens.push((use_file, use_file_tokens)) + } + true + }, + + (Some((Token::LeftBrace, _)), ..) + => { + let mut blocks = 0; + loop { + curr += 1; + match tokens.get(curr) { + Some((Token::LeftBrace, _)) => { + blocks += 1; + } + + Some((Token::RightBrace, _)) => { + curr += 1; + blocks -= 1; + if blocks <= 0 { + break; + } + } + + None => { + break; + } + + _ => {} + } + } + false + }, + + (Some((Token::Identifier(_), _)), + Some((Token::ColonColon, _)), + Some((Token::Fn, _))) + => true, + + (Some((Token::Identifier(_), _)), + Some((Token::ColonColon, _)), + Some(_)) + => true, + + (Some((Token::Identifier(_), _)), + Some((Token::ColonEqual, _)), + Some(_)) + => true, + + _ => false, + } { + sections.push(Section::new(path.clone(), &tokens[last..curr])); + last = curr; + } + curr += 1; + } + sections.push(Section::new(path.clone(), &tokens[last..curr])); + } + sections +} diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 2c8e5e8..3b61e5f 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -127,6 +127,9 @@ pub enum Token { #[token("\n")] Newline, + #[token("use")] + Use, + #[token("<<<<<<<")] GitConflictBegin, #[token(">>>>>>>")] |