From c3e1e3bcbb177a5cbdb972389626e8c7347cbfc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gustav=20S=C3=B6rn=C3=A4s?= Date: Sun, 28 Feb 2021 17:23:09 +0100 Subject: move sectionizer, sections own tokens --- src/sectionizer.rs | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 src/sectionizer.rs (limited to 'src/sectionizer.rs') diff --git a/src/sectionizer.rs b/src/sectionizer.rs new file mode 100644 index 0000000..b7e083f --- /dev/null +++ b/src/sectionizer.rs @@ -0,0 +1,109 @@ +use crate::tokenizer::{PlacedToken, Token, file_to_tokens}; + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +pub struct Section { + pub tokens: Vec, + pub path: PathBuf, +} + +impl Section { + fn new(path: PathBuf, tokens: &[PlacedToken]) -> Self { + Self { + tokens: Vec::from(tokens), + path, + } + } +} + +pub fn sectionize(path: &Path) -> Vec
{ + let mut read_files = HashSet::new(); + read_files.insert(path.to_path_buf()); + let tokens = file_to_tokens(path); + let mut all_tokens = vec![(path.to_path_buf(), tokens)]; + let mut sections = Vec::new(); + + let mut i = 0; + while i < all_tokens.len() { + let (path, tokens) = all_tokens[i].clone(); + i += 1; + let mut last = 0; + let mut curr = 0; + while curr < tokens.len() { + if match (tokens.get(curr + 0), tokens.get(curr + 1), tokens.get(curr + 2)) { + (Some((Token::Newline, _)), ..) + => { + if curr == last { + last += 1; + } + false + }, + + (Some((Token::Use, _)), + Some((Token::Identifier(use_file), _)), + Some((Token::Newline, _))) => { + curr += 3; + let use_file: PathBuf = format!("{}.sy", use_file).into(); + if !read_files.contains(&use_file) { + let use_file_tokens = file_to_tokens(&use_file); + read_files.insert(use_file.clone()); + all_tokens.push((use_file, use_file_tokens)) + } + true + }, + + (Some((Token::LeftBrace, _)), ..) + => { + let mut blocks = 0; + loop { + curr += 1; + match tokens.get(curr) { + Some((Token::LeftBrace, _)) => { + blocks += 1; + } + + Some((Token::RightBrace, _)) => { + curr += 1; + blocks -= 1; + if blocks <= 0 { + break; + } + } + + None => { + break; + } + + _ => {} + } + } + false + }, + + (Some((Token::Identifier(_), _)), + Some((Token::ColonColon, _)), + Some((Token::Fn, _))) + => true, + + (Some((Token::Identifier(_), _)), + Some((Token::ColonColon, _)), + Some(_)) + => true, + + (Some((Token::Identifier(_), _)), + Some((Token::ColonEqual, _)), + Some(_)) + => true, + + _ => false, + } { + sections.push(Section::new(path.clone(), &tokens[last..curr])); + last = curr; + } + curr += 1; + } + sections.push(Section::new(path.clone(), &tokens[last..curr])); + } + sections +} -- cgit v1.2.1 From e2eefa106a8f8b1677ba6275f5ea5879cac20974 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gustav=20S=C3=B6rn=C3=A4s?= Date: Sun, 28 Feb 2021 18:35:09 +0100 Subject: more wip namespaces --- src/sectionizer.rs | 1 - 1 file changed, 1 deletion(-) (limited to 'src/sectionizer.rs') diff --git a/src/sectionizer.rs b/src/sectionizer.rs index b7e083f..9441663 100644 --- a/src/sectionizer.rs +++ b/src/sectionizer.rs @@ -43,7 +43,6 @@ pub fn sectionize(path: &Path) -> Vec
{ (Some((Token::Use, _)), Some((Token::Identifier(use_file), _)), Some((Token::Newline, _))) => { - curr += 3; let use_file: PathBuf = format!("{}.sy", use_file).into(); if !read_files.contains(&use_file) { let use_file_tokens = file_to_tokens(&use_file); -- cgit v1.2.1 From 2090888dce496f893638268b0aef981c96b68ac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edvard=20Th=C3=B6rnros?= Date: Sun, 28 Feb 2021 19:10:27 +0100 Subject: imports almost work --- src/sectionizer.rs | 5 ----- 1 file changed, 5 deletions(-) (limited to 'src/sectionizer.rs') diff --git a/src/sectionizer.rs b/src/sectionizer.rs index 9441663..55f231e 100644 --- a/src/sectionizer.rs +++ b/src/sectionizer.rs @@ -80,11 +80,6 @@ pub fn sectionize(path: &Path) -> Vec
{ false }, - (Some((Token::Identifier(_), _)), - Some((Token::ColonColon, _)), - Some((Token::Fn, _))) - => true, - (Some((Token::Identifier(_), _)), Some((Token::ColonColon, _)), Some(_)) -- cgit v1.2.1 From 9415c7db1cd00b530f82f673efc32a459a4ae789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gustav=20S=C3=B6rn=C3=A4s?= Date: Mon, 1 Mar 2021 19:29:11 +0100 Subject: bugfix sectionizer --- src/sectionizer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/sectionizer.rs') diff --git a/src/sectionizer.rs b/src/sectionizer.rs index 55f231e..c9cfcd5 100644 --- a/src/sectionizer.rs +++ b/src/sectionizer.rs @@ -54,7 +54,7 @@ pub fn sectionize(path: &Path) -> Vec
{ (Some((Token::LeftBrace, _)), ..) => { - let mut blocks = 0; + let mut blocks = 1; loop { curr += 1; match tokens.get(curr) { -- cgit v1.2.1 From dd38069bc1f8b696e8f0c50f43e0e7954729acf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gustav=20S=C3=B6rn=C3=A4s?= Date: Fri, 5 Mar 2021 20:12:39 +0100 Subject: better errors on invalid outer tokens --- src/sectionizer.rs | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/sectionizer.rs') diff --git a/src/sectionizer.rs b/src/sectionizer.rs index c9cfcd5..8c5e238 100644 --- a/src/sectionizer.rs +++ b/src/sectionizer.rs @@ -6,6 +6,7 @@ use std::path::{Path, PathBuf}; pub struct Section { pub tokens: Vec, pub path: PathBuf, + pub faulty: bool, } impl Section { @@ -13,6 +14,7 @@ impl Section { Self { tokens: Vec::from(tokens), path, + faulty: false, } } } -- cgit v1.2.1