Merge pull request #9 from Yuta1004/#4_RemoveLexIterator

#4 LexIterator 削除

Author: Yuta1004

Cargo.toml

@@ -6,16 +6,16 @@ edition = "2021"
 [dependencies]
 anyhow = { workspace = true }
 thiserror = { workspace = true }
-core = { workspace = true }
-algorithm = { workspace = true }
+pgen_core = { workspace = true }
+pgen_algorithm = { workspace = true }
 
 [dev-dependencies]
 serde = { workspace = true }
 serde_json = "1.0.117"
 
 [features]
 default = []
-derive = ["core/derive"]
+derive = ["pgen_core/derive"]
 
 [workspace]
 resolver = "2"
@@ -31,5 +31,5 @@ thiserror = "1.0.58"
 serde = "1.0.197"
 regex = "1.10.4"
 regex-macro = "0.2.0"
-core = { path = "./crates/core" }
-algorithm = { path = "./crates/algorithm" }
+pgen_core = { package = "core", path = "./crates/core" }
+pgen_algorithm = { package = "algorithm", path = "./crates/algorithm" }

crates/algorithm_lr1/Cargo.toml

@@ -8,4 +8,4 @@ anyhow = { workspace = true }
 thiserror = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 itertools = "0.12.1"
-core = { path = "../core", features = ["derive"] }
+pgen_core = { package = "core", path = "../core", features = ["derive"] }

crates/algorithm_lr1/src/builder.rs

@@ -5,7 +5,7 @@ use std::marker::PhantomData;
 use serde::{Serialize, Deserialize};
 use itertools::Itertools;
 
-use core::cfg::{TokenSet, Syntax, Rule, RuleElem, RuleSet};
+use pgen_core::cfg::{TokenSet, Syntax, Rule, RuleElem, RuleSet};
 
 #[derive(Debug, Serialize, Deserialize)]
 pub(super) enum LRAction<S> {

crates/algorithm_lr1/src/driver.rs

@@ -1,7 +1,8 @@
-use core::cfg::{TokenSet, Syntax};
-use core::lex::LexIterator;
+use pgen_core::cfg::{TokenSet, Syntax};
+use pgen_core::lex::Token;
 
-use super::builder::{LRAction, LR1Configure};
+use crate::error::ParseError;
+use crate::builder::{LRAction, LR1Configure};
 
 pub(super) struct LR1Driver<'a, 'b, T, S> (&'b LR1Configure<'a, T, S>)
 where
@@ -19,7 +20,7 @@ where
 
     pub fn run<'c>(
         &self,
-        lexer: &mut impl LexIterator<'a, 'c, T>,
+        lexer: &mut impl Iterator<Item = Token<'a, 'c, T>>,
     ) -> anyhow::Result<()> {
         let mut stack = vec![0];
         loop {
@@ -29,31 +30,31 @@ where
                 let action = match input {
                     Some(token) => (
                         self.0.action_table[top].get(&token.kind).unwrap(),
-                        Some(token.as_str()),
+                        Some(token),
                     ),
                     None => (
                         &self.0.eof_action_table[top],
                         None
                     ),
                 };
-                match action.0 {
-                    LRAction::Shift(new_state) => {
+                match action {
+                    (LRAction::Shift(new_state), _) => {
                         stack.push(*new_state);
                         break;
                     }
-                    LRAction::Reduce(_, goto, elems_cnt) => {
+                    (LRAction::Reduce(_, goto, elems_cnt), _) => {
                         stack.truncate(stack.len() - elems_cnt);
                         stack.push(self.0.goto_table[stack[stack.len() - 1]][*goto]);
                     }
-                    LRAction::None => {
-                        let pos = lexer.pos();
-                        let pos = match action.1 {
-                            Some(raw) => (pos.0, pos.1 - (raw.len() as u32)),
-                            None => pos,
-                        };
-                        return Err(anyhow::anyhow!("Error at {:?}", pos).into());
+                    (LRAction::Accept, _) => {
+                        return Ok(());
+                    }
+                    (LRAction::None, Some(token)) => {
+                        return Err(ParseError::new_unexpected_token(token).into());
+                    }
+                    (LRAction::None, None) => {
+                        return Err(ParseError::UnexpectedEOF.into());
                     }
-                    LRAction::Accept => return Ok(()),
                 }
             }
         }

crates/algorithm_lr1/src/error.rs

@@ -0,0 +1,27 @@
+use thiserror::Error;
+
+use pgen_core::error::ParseError as SuperParseError;
+use pgen_core::cfg::TokenSet;
+use pgen_core::lex::Token;
+
+#[derive(Debug, Error)]
+pub enum ParseError {
+    #[error("Unexpected token {actual:?} found")]
+    UnexpectedToken {
+        actual: String,
+    },
+    #[error("Unexpected EOF")]
+    UnexpectedEOF,
+}
+
+impl ParseError {
+    pub fn new_unexpected_token<'a, T>(expected: Token<'a, '_, T>) -> SuperParseError
+    where
+        T: TokenSet<'a>,
+    {
+        let err = ParseError::UnexpectedToken {
+            actual: format!("{:?}", expected.kind),
+        };
+        SuperParseError::from(err).with(expected)
+    }
+}

crates/algorithm_lr1/src/lib.rs

@@ -1,11 +1,12 @@
+mod error;
 mod builder;
 mod driver;
 
 use serde::{Serialize, Deserialize};
 
-use core::cfg::{TokenSet, Syntax};
-use core::lex::LexIterator;
-use core::parse::ParserImpl;
+use pgen_core::cfg::{TokenSet, Syntax};
+use pgen_core::lex::Token;
+use pgen_core::parse::ParserImpl;
 
 use builder::LR1Configure;
 use driver::LR1Driver;
@@ -31,16 +32,16 @@ where
 
     fn parse<'b>(
         &self,
-        mut lexer: impl LexIterator<'a, 'b, T>,
+        mut lexer: impl Iterator<Item = Token<'a, 'b, T>>,
     ) -> anyhow::Result<Self::Output> {
         LR1Driver::new(&self.0).run(&mut lexer)
     }
 }
 
 #[cfg(test)]
 mod test {
-    use core::cfg::{TokenSet, Syntax, Rule, RuleElem};
-    use core::Parser;
+    use pgen_core::cfg::{TokenSet, Syntax, Rule, RuleElem};
+    use pgen_core::Parser;
 
     use super::LR1;
 

crates/core/Cargo.toml

@@ -8,8 +8,8 @@ anyhow = { workspace = true }
 thiserror = { workspace = true }
 serde = { workspace = true, features = ["derive"]}
 regex = { workspace = true }
-derive = { package = "core_derive", path = "../core_derive", optional = true }
+pgen_core_derive = { package = "core_derive", path = "../core_derive", optional = true }
 
 [features]
 default = []
-derive = ["dep:derive"]
+derive = ["dep:pgen_core_derive"]

crates/core/src/cfg.rs

@@ -2,7 +2,7 @@ mod token;
 mod syntax;
 
 #[cfg(feature = "derive")]
-pub use derive::{TokenSet, Syntax};
+pub use pgen_core_derive::{TokenSet, Syntax};
 
 pub use token::TokenSet;
 pub use syntax::{Syntax, Rule, RuleElem, RuleSet};

crates/core/src/cfg/syntax.rs

@@ -1,12 +1,13 @@
 use std::collections::HashMap;
+use std::fmt::Debug;
 use std::hash::Hash;
 use std::marker::PhantomData;
 
 use super::token::TokenSet;
 
 pub trait Syntax<'a>
 where
-    Self: Clone + Copy + Sized,
+    Self: Debug + Clone + Copy + Sized,
 {
     type TokenSet: TokenSet<'a>;
 

crates/core/src/cfg/token.rs

@@ -1,10 +1,11 @@
+use std::fmt::Debug;
 use std::hash::Hash;
 
 use regex::Regex;
 
 pub trait TokenSet<'a>
 where
-    Self: Copy + Clone + Hash + Eq,
+    Self: Debug + Copy + Clone + Hash + Eq,
 {
     fn ignore_str() -> &'a str;
     fn enum_iter() -> impl Iterator<Item = Self>;