Merge pull request #17 from Tsukuba-Programming-Lab/develop

Release v0.1.1

Author: Yuta1004

Cargo.lock

@@ -1,21 +1,21 @@
 [package]
 name = "parsergen"
-version = "0.1.0"
+version = "0.1.1"
 edition = "2021"
 
 [dependencies]
 anyhow = { workspace = true }
 thiserror = { workspace = true }
-core = { workspace = true }
-algorithm = { workspace = true }
+pgen_core = { workspace = true }
+pgen_algorithm = { workspace = true }
 
 [dev-dependencies]
 serde = { workspace = true }
 serde_json = "1.0.117"
 
 [features]
 default = []
-derive = ["core/derive"]
+derive = ["pgen_core/derive"]
 
 [workspace]
 resolver = "2"
@@ -31,5 +31,5 @@ thiserror = "1.0.58"
 serde = "1.0.197"
 regex = "1.10.4"
 regex-macro = "0.2.0"
-core = { path = "./crates/core" }
-algorithm = { path = "./crates/algorithm" }
+pgen_core = { package = "core", path = "./crates/core" }
+pgen_algorithm = { package = "algorithm", path = "./crates/algorithm" }

Cargo.toml

@@ -13,9 +13,15 @@ Rust製パーサジェネレータ
 ```
 $ cargo run --example expr
 (10+20)/((30*40)-50)
-Accepted
+Accepted : (Expr (Term (Term (Num "(" (Expr (Expr (Term (Num "10"))) "+" (Term (Num "20"))) ")")) "/" (Num "(" (Expr (Expr (Term (Num "(" (Expr (Term (Term (Num "30")) "*" (Num "40"))) ")"))) "-" (Term (Num "50"))) ")")))
 
 $ cargo run --example expr
 10**
-Rejected: Error at (0, 3)
+-----
+ 1: 10**
+       ^ here
+Error at line 1, column 4.
+-----
+
+Rejected : Unexpected token "Mul" found
 ```

README.md

@@ -1,6 +1,6 @@
 [package]
 name = "algorithm"
-version = "0.1.0"
+version = "0.1.1"
 edition = "2021"
 
 [dependencies]

crates/algorithm/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "algorithm_lr1"
-version = "0.1.0"
+version = "0.1.1"
 edition = "2021"
 
 [dependencies]
 anyhow = { workspace = true }
 thiserror = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 itertools = "0.12.1"
-core = { path = "../core", features = ["derive"] }
+pgen_core = { package = "core", path = "../core", features = ["derive"] }

crates/algorithm_lr1/Cargo.toml

@@ -5,7 +5,7 @@ use std::marker::PhantomData;
 use serde::{Serialize, Deserialize};
 use itertools::Itertools;
 
-use core::cfg::{TokenSet, Syntax, Rule, RuleElem, RuleSet};
+use pgen_core::cfg::{TokenSet, Syntax, Rule, RuleElem, RuleSet};
 
 #[derive(Debug, Serialize, Deserialize)]
 pub(super) enum LRAction<S> {
@@ -37,11 +37,8 @@ where
 {
     pub fn setup() -> anyhow::Result<Self> {
         // 1. Pre-process
-        let rules = S::try_into()?
-            .into_iter()
-            .map(|(rule, _)| rule)
-            .collect::<Vec<_>>();
-        let ruleset = RuleSet::from(rules);
+        let rules = S::into_iter().collect::<Vec<_>>();
+        let ruleset = S::into_ruleset();
         let first_set = ruleset.first_set();
 
         // 2. Generate dummy nonterm
@@ -76,7 +73,7 @@ where
         let mut goto_table: Vec<Vec<usize>> = Vec::with_capacity(dfa.0.len());
         for _ in 0..dfa.0.len() {
             action_table.push(HashMap::from_iter(
-                T::enum_iter()
+                T::into_iter()
                     .map(|token| (token, LRAction::None))
                     .collect::<Vec<(T, LRAction<S>)>>(),
             ));
@@ -85,7 +82,6 @@ where
         }
 
         // 5. Setup tables
-        let rule_table: Vec<S> = S::enum_iter().collect();
         for lritem_set in &dfa.0 {
             for (token, next) in &lritem_set.next {
                 match &token {
@@ -113,7 +109,7 @@ where
                             let id = lritem_set.id as usize;
                             let label = action_table[id].get_mut(&t.0).unwrap();
                             *label = LRAction::Reduce(
-                                rule_table[item.rule.id as usize],
+                                rules[item.rule.id as usize],
                                 *nonterm_table.get(lhs).unwrap(),
                                 item.rule.rhs.len(),
                             );
@@ -124,7 +120,7 @@ where
                                 LRAction::Accept
                             } else {
                                 LRAction::Reduce(
-                                    rule_table[item.rule.id as usize],
+                                    rules[item.rule.id as usize],
                                     *nonterm_table.get(lhs).unwrap(),
                                     item.rule.rhs.len(),
                                 )

crates/algorithm_lr1/src/builder.rs

@@ -1,7 +1,9 @@
-use core::cfg::{TokenSet, Syntax};
-use core::lex::LexIterator;
+use pgen_core::cfg::{TokenSet, Syntax};
+use pgen_core::lex::Token;
+use pgen_core::parse::{SExp, SExpBuilder};
 
-use super::builder::{LRAction, LR1Configure};
+use crate::error::ParseError;
+use crate::builder::{LRAction, LR1Configure};
 
 pub(super) struct LR1Driver<'a, 'b, T, S> (&'b LR1Configure<'a, T, S>)
 where
@@ -19,41 +21,45 @@ where
 
     pub fn run<'c>(
         &self,
-        lexer: &mut impl LexIterator<'a, 'c, T>,
-    ) -> anyhow::Result<()> {
+        lexer: &mut impl Iterator<Item = Token<'a, 'c, T>>,
+    ) -> anyhow::Result<SExp<'a, 'c, T, S>> {
         let mut stack = vec![0];
+        let mut builder = SExpBuilder::new();
         loop {
             let input = lexer.next();
             loop {
                 let top = stack[stack.len() - 1];
                 let action = match input {
                     Some(token) => (
                         self.0.action_table[top].get(&token.kind).unwrap(),
-                        Some(token.as_str()),
+                        Some(token),
                     ),
                     None => (
                         &self.0.eof_action_table[top],
                         None
                     ),
                 };
-                match action.0 {
-                    LRAction::Shift(new_state) => {
+                match action {
+                    (LRAction::Shift(new_state), Some(token)) => {
                         stack.push(*new_state);
+                        builder.push(token);
                         break;
                     }
-                    LRAction::Reduce(_, goto, elems_cnt) => {
+                    (LRAction::Reduce(tag, goto, elems_cnt), _) => {
                         stack.truncate(stack.len() - elems_cnt);
                         stack.push(self.0.goto_table[stack[stack.len() - 1]][*goto]);
+                        builder.wrap(*tag, *elems_cnt);
                     }
-                    LRAction::None => {
-                        let pos = lexer.pos();
-                        let pos = match action.1 {
-                            Some(raw) => (pos.0, pos.1 - (raw.len() as u32)),
-                            None => pos,
-                        };
-                        return Err(anyhow::anyhow!("Error at {:?}", pos).into());
+                    (LRAction::Accept, _) => {
+                        return builder.build();
                     }
-                    LRAction::Accept => return Ok(()),
+                    (LRAction::None, Some(token)) => {
+                        return Err(ParseError::new_unexpected_token(token).into());
+                    }
+                    (LRAction::None, None) => {
+                        return Err(ParseError::UnexpectedEOF.into());
+                    }
+                    _ => unreachable!(),
                 }
             }
         }

crates/algorithm_lr1/src/driver.rs

@@ -0,0 +1,27 @@
+use thiserror::Error;
+
+use pgen_core::error::ParseError as SuperParseError;
+use pgen_core::cfg::TokenSet;
+use pgen_core::lex::Token;
+
+#[derive(Debug, Error)]
+pub enum ParseError {
+    #[error("Unexpected token {actual:?} found")]
+    UnexpectedToken {
+        actual: String,
+    },
+    #[error("Unexpected EOF")]
+    UnexpectedEOF,
+}
+
+impl ParseError {
+    pub fn new_unexpected_token<'a, T>(expected: Token<'a, '_, T>) -> SuperParseError
+    where
+        T: TokenSet<'a>,
+    {
+        let err = ParseError::UnexpectedToken {
+            actual: format!("{:?}", expected.kind),
+        };
+        SuperParseError::from(err).with(expected)
+    }
+}

crates/algorithm_lr1/src/error.rs

@@ -1,11 +1,12 @@
+mod error;
 mod builder;
 mod driver;
 
 use serde::{Serialize, Deserialize};
 
-use core::cfg::{TokenSet, Syntax};
-use core::lex::LexIterator;
-use core::parse::ParserImpl;
+use pgen_core::cfg::{TokenSet, Syntax};
+use pgen_core::lex::Token;
+use pgen_core::parse::{ParserImpl, SExp};
 
 use builder::LR1Configure;
 use driver::LR1Driver;
@@ -23,24 +24,23 @@ where
 {
     type TokenSet = T;
     type Syntax = S;
-    type Output = ();
 
     fn setup() -> anyhow::Result<Self> {
         Ok(LR1(LR1Configure::setup()?))
     }
 
     fn parse<'b>(
         &self,
-        mut lexer: impl LexIterator<'a, 'b, T>,
-    ) -> anyhow::Result<Self::Output> {
+        mut lexer: impl Iterator<Item = Token<'a, 'b, T>>,
+    ) -> anyhow::Result<SExp<'a, 'b, T, S>> {
         LR1Driver::new(&self.0).run(&mut lexer)
     }
 }
 
 #[cfg(test)]
 mod test {
-    use core::cfg::{TokenSet, Syntax, Rule, RuleElem};
-    use core::Parser;
+    use pgen_core::cfg::{TokenSet, Syntax, Rule, RuleElem};
+    use pgen_core::Parser;
 
     use super::LR1;
 
@@ -67,19 +67,14 @@ mod test {
     #[derive(Debug, Clone, Copy, Syntax)]
     enum TestSyntax {
         #[rule("<expr> ::= <expr> Plus <term>")]
-        ExprPlus,
         #[rule("<expr> ::= <expr> Minus <term>")]
-        ExprMinus,
         #[rule("<expr> ::= <term>")]
-        ExprTerm,
+        Expr,
         #[rule("<term> ::= <term> Mul <num>")]
-        TermMul,
         #[rule("<term> ::= <term> Div <num>")]
-        TermDiv,
         #[rule("<term> ::= <num>")]
-        TermNum,
+        Term,
         #[rule("<num> ::= BracketL <expr> BracketR")]
-        NestedNum,
         #[rule("<num> ::= Num")]
         Num,
     }

crates/algorithm_lr1/src/lib.rs

@@ -1,15 +1,15 @@
 [package]
 name = "core"
-version = "0.1.0"
+version = "0.1.1"
 edition = "2021"
 
 [dependencies]
 anyhow = { workspace = true }
 thiserror = { workspace = true }
 serde = { workspace = true, features = ["derive"]}
 regex = { workspace = true }
-derive = { package = "core_derive", path = "../core_derive", optional = true }
+pgen_core_derive = { package = "core_derive", path = "../core_derive", optional = true }
 
 [features]
 default = []
-derive = ["dep:derive"]
+derive = ["dep:pgen_core_derive"]