move sectionizer, sections own tokens

4 files changed, 150 insertions, 125 deletions

diff --git a/src/compiler.rs b/src/compiler.rs
index fc28b7d..90bc51e 100644
--- a/src/compiler.rs
+++ b/src/compiler.rs
@@ -1,11 +1,12 @@
-use std::{path::{Path, PathBuf}};
+use std::path::{Path, PathBuf};
 use std::cell::RefCell;
 use std::collections::{HashMap, hash_map::Entry};
 use std::rc::Rc;
 
 use crate::{Blob, Block, Op, Prog, RustFunction, Type, Value};
 use crate::error::{Error, ErrorKind};
-use crate::tokenizer::{Token, PlacedToken, TokenStream};
+use crate::sectionizer::Section;
+use crate::tokenizer::Token;
 
 macro_rules! nextable_enum {
     ( $name:ident { $( $thing:ident ),* $( , )? } ) => {
@@ -330,31 +331,17 @@ impl<'a> CompilerContext<'a> {
     }
 }
 
-struct Section<'a> {
-    path: PathBuf,
-    tokens: &'a [PlacedToken],
-}
-
-impl<'a> Section<'a> {
-    fn new(path: PathBuf, tokens: &'a [PlacedToken]) -> Self {
-        Section {
-            path,
-            tokens,
-        }
-    }
-}
-
 #[derive(Debug)]
 enum Name<'a> {
     Slot(usize, usize),
     Unknown(usize, usize),
-    Namespace(&'a Namespace<'a>),
+    _Namespace(&'a Namespace<'a>),
 }
 
 pub(crate) struct Compiler<'a> {
     current_token: usize,
     current_section: usize,
-    sections: Vec<Section<'a>>,
+    sections: Vec<Section>,
 
     contextes: HashMap<PathBuf, CompilerContext<'a>>,
 
@@ -377,82 +364,13 @@ fn add_op(compiler: &Compiler, block: &mut Block, op: Op) -> usize {
     block.add(op, compiler.line())
 }
 
-fn split_sections<'a>(file_name: PathBuf, tokens: &'a TokenStream) -> Vec<Section> {
-    let mut sections = Vec::new();
-
-    let mut last = 0;
-    let mut curr = 0;
-    while curr < tokens.len() {
-        if match (tokens.get(curr + 0), tokens.get(curr + 1), tokens.get(curr + 2)) {
-            (Some((Token::Newline, _)), ..)
-                => {
-                if curr == last {
-                    last += 1;
-                }
-                false
-            },
-
-            (Some((Token::LeftBrace, _)), ..)
-                => {
-                let mut blocks = 0;
-                loop {
-                    curr += 1;
-                    match tokens.get(curr) {
-                        Some((Token::LeftBrace, _)) => {
-                            blocks += 1;
-                        }
-
-                        Some((Token::RightBrace, _)) => {
-                            curr += 1;
-                            blocks -= 1;
-                            if blocks <= 0 {
-                                break;
-                            }
-                        }
-
-                        None => {
-                            break;
-                        }
-
-                        _ => {}
-                    }
-                }
-                false
-            },
-
-            (Some((Token::Identifier(_), _)),
-             Some((Token::ColonColon, _)),
-             Some((Token::Fn, _)))
-                => true,
-
-            (Some((Token::Identifier(_), _)),
-             Some((Token::ColonColon, _)),
-             Some(_))
-                => true,
-
-            (Some((Token::Identifier(_), _)),
-             Some((Token::ColonEqual, _)),
-             Some(_))
-                => true,
-
-            _ => false,
-        } {
-            sections.push(Section::new(file_name.clone(), &tokens[last..curr]));
-            last = curr;
-        }
-        curr += 1;
-    }
-    sections.push(Section::new(file_name, &tokens[last..curr]));
-    sections
-}
-
 impl<'a, 'b> Compiler<'a> {
-    pub(crate) fn new(current_file: &Path, tokens: &'a TokenStream) -> Self {
-        let current_file = current_file.to_path_buf();
-        let sections = split_sections(current_file.clone(), tokens);
+    pub(crate) fn new(sections: Vec<Section>) -> Self {
+        let contextes = sections
+            .iter()
+            .map(|section| (section.path.to_path_buf(), CompilerContext::new()))
+            .collect();
 
-        let mut contextes = HashMap::new();
-        contextes.insert(current_file, CompilerContext::new());
         Self {
             current_token: 0,
             current_section: 0,
@@ -1701,7 +1619,6 @@ impl<'a, 'b> Compiler<'a> {
     }
 
     pub(crate) fn compile(&'b mut self, name: &str, file: &Path, functions: &[(String, RustFunction)]) -> Result<Prog, Vec<Error>> {
-
         for section in 0..self.sections.len() {
             self.init_section(section);
             let section = &self.sections[section];
@@ -1709,21 +1626,24 @@ impl<'a, 'b> Compiler<'a> {
                 (Some((Token::Identifier(name), _)),
                  Some((Token::ColonColon, _)),
                  Some((Token::Fn, _))) => {
-                    self.forward_constant(name.to_string());
+                    let name = name.to_string();
+                    self.forward_constant(name);
                 },
 
                 (Some((Token::Blob, _)),
                  Some((Token::Identifier(name), _)), ..) => {
-                    self.forward_constant(name.to_string());
+                    let name = name.to_string();
+                    self.forward_constant(name);
                 },
 
                 (Some((Token::Identifier(name), _)),
                  Some((Token::Colon, _)), ..) => {
+                    let name = name.to_string();
                     self.eat();
                     self.eat();
                     if let Ok(ty) = self.parse_type() {
                         let is_mut = self.peek() == Token::Equal;
-                        let var = Variable::new(name, is_mut, ty);
+                        let var = Variable::new(&name, is_mut, ty);
                         let _ = self.define(var);
                     } else {
                         error!(self, format!("Failed to parse type global '{}'.", name));
diff --git a/src/lib.rs b/src/lib.rs
index 8a023b6..5542d17 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -9,7 +9,6 @@ use std::hash::{Hash, Hasher};
 use owo_colors::OwoColorize;
 
 use error::Error;
-use tokenizer::TokenStream;
 
 use crate::error::ErrorKind;
 
@@ -17,45 +16,39 @@ pub mod error;
 pub mod vm;
 
 mod compiler;
+mod sectionizer;
 mod tokenizer;
 
 /// Compiles a file and links the supplied functions as callable external
 /// functions. Use this if you want your programs to be able to yield.
-pub fn compile_file(
-    path: &Path,
-    print: bool,
-    functions: Vec<(String, RustFunction)>
-) -> Result<vm::VM, Vec<Error>> {
-    let tokens = tokenizer::file_to_tokens(path);
-    match compiler::Compiler::new(path, &tokens).compile("main", path, &functions) {
-        Ok(prog) => {
-            let mut vm = vm::VM::new();
-            vm.print_blocks = print;
-            vm.print_ops = print;
-            vm.typecheck(&prog)?;
-            vm.init(&prog);
-            Ok(vm)
-        }
-        Err(errors) => Err(errors),
-    }
-}
+//pub fn compile_file(
+//    path: &Path,
+//    print: bool,
+//    functions: Vec<(String, RustFunction)>
+//) -> Result<vm::VM, Vec<Error>> {
+//    match compiler::Compiler::new(path).compile("main", path, &functions) {
+//        Ok(prog) => {
+//            let mut vm = vm::VM::new();
+//            vm.print_blocks = print;
+//            vm.print_ops = print;
+//            vm.typecheck(&prog)?;
+//            vm.init(&prog);
+//            Ok(vm)
+//        }
+//        Err(errors) => Err(errors),
+//    }
+//}
 
 /// Compiles, links and runs the given file. Supplied functions are callable
 /// external functions. If you want your program to be able to yield, use
 /// [compile_file].
 pub fn run_file(path: &Path, print: bool, functions: Vec<(String, RustFunction)>) -> Result<(), Vec<Error>> {
-    run(tokenizer::file_to_tokens(path), path, print, functions)
-}
-
-/// Compile and run a string containing source code. The supplied functions are
-/// linked as callable external functions. This is useful for short test
-/// programs.
-pub fn run_string(s: &str, print: bool, functions: Vec<(String, RustFunction)>) -> Result<(), Vec<Error>> {
-    run(tokenizer::string_to_tokens(s), Path::new("builtin"), print, functions)
+    run(path, print, functions)
 }
 
-fn run(tokens: TokenStream, path: &Path, print: bool, functions: Vec<(String, RustFunction)>) -> Result<(), Vec<Error>> {
-    match compiler::Compiler::new(path, &tokens).compile("main", path, &functions) {
+fn run(path: &Path, print: bool, functions: Vec<(String, RustFunction)>) -> Result<(), Vec<Error>> {
+    let sections = sectionizer::sectionize(path);
+    match compiler::Compiler::new(sections).compile("main", path, &functions) {
         Ok(prog) => {
             let mut vm = vm::VM::new();
             vm.print_blocks = print;
diff --git a/src/sectionizer.rs b/src/sectionizer.rs
new file mode 100644
index 0000000..b7e083f
--- /dev/null
+++ b/src/sectionizer.rs
@@ -0,0 +1,109 @@
+use crate::tokenizer::{PlacedToken, Token, file_to_tokens};
+
+use std::collections::HashSet;
+use std::path::{Path, PathBuf};
+
+pub struct Section {
+    pub tokens: Vec<PlacedToken>,
+    pub path: PathBuf,
+}
+
+impl Section {
+    fn new(path: PathBuf, tokens: &[PlacedToken]) -> Self {
+        Self {
+            tokens: Vec::from(tokens),
+            path,
+        }
+    }
+}
+
+pub fn sectionize(path: &Path) -> Vec<Section> {
+    let mut read_files = HashSet::new();
+    read_files.insert(path.to_path_buf());
+    let tokens = file_to_tokens(path);
+    let mut all_tokens = vec![(path.to_path_buf(), tokens)];
+    let mut sections = Vec::new();
+
+    let mut i = 0;
+    while i < all_tokens.len() {
+        let (path, tokens) = all_tokens[i].clone();
+        i += 1;
+        let mut last = 0;
+        let mut curr = 0;
+        while curr < tokens.len() {
+            if match (tokens.get(curr + 0), tokens.get(curr + 1), tokens.get(curr + 2)) {
+                (Some((Token::Newline, _)), ..)
+                    => {
+                        if curr == last {
+                            last += 1;
+                        }
+                        false
+                    },
+
+                (Some((Token::Use, _)),
+                 Some((Token::Identifier(use_file), _)),
+                 Some((Token::Newline, _))) => {
+                    curr += 3;
+                    let use_file: PathBuf = format!("{}.sy", use_file).into();
+                    if !read_files.contains(&use_file) {
+                        let use_file_tokens = file_to_tokens(&use_file);
+                        read_files.insert(use_file.clone());
+                        all_tokens.push((use_file, use_file_tokens))
+                    }
+                    true
+                },
+
+                (Some((Token::LeftBrace, _)), ..)
+                    => {
+                        let mut blocks = 0;
+                        loop {
+                            curr += 1;
+                            match tokens.get(curr) {
+                                Some((Token::LeftBrace, _)) => {
+                                    blocks += 1;
+                                }
+
+                                Some((Token::RightBrace, _)) => {
+                                    curr += 1;
+                                    blocks -= 1;
+                                    if blocks <= 0 {
+                                        break;
+                                    }
+                                }
+
+                                None => {
+                                    break;
+                                }
+
+                                _ => {}
+                            }
+                        }
+                        false
+                    },
+
+                (Some((Token::Identifier(_), _)),
+                 Some((Token::ColonColon, _)),
+                 Some((Token::Fn, _)))
+                    => true,
+
+                (Some((Token::Identifier(_), _)),
+                 Some((Token::ColonColon, _)),
+                 Some(_))
+                    => true,
+
+                (Some((Token::Identifier(_), _)),
+                 Some((Token::ColonEqual, _)),
+                 Some(_))
+                    => true,
+
+                _ => false,
+            } {
+                sections.push(Section::new(path.clone(), &tokens[last..curr]));
+                last = curr;
+            }
+            curr += 1;
+        }
+        sections.push(Section::new(path.clone(), &tokens[last..curr]));
+    }
+    sections
+}
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 2c8e5e8..3b61e5f 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -127,6 +127,9 @@ pub enum Token {
     #[token("\n")]
     Newline,
 
+    #[token("use")]
+    Use,
+
     #[token("<<<<<<<")]
     GitConflictBegin,
     #[token(">>>>>>>")]