lark syntax -> llguidance translator

parser/src/earley/lexerspec.rs

@@ -99,6 +99,15 @@ impl LexerSpec {
         SimpleVob::alloc(self.lexemes.len())
     }
 
+    pub fn all_lexemes(&self) -> SimpleVob {
+        let mut v = self.alloc_lexeme_set();
+        self.lexemes[0..self.lexemes.len() - self.num_extra_lexemes]
+            .iter()
+            .enumerate()
+            .for_each(|(idx, _)| v.set(idx, true));
+        v
+    }
+
     pub fn lazy_lexemes(&self) -> SimpleVob {
         let mut v = self.alloc_lexeme_set();
         for (idx, lex) in self.lexemes.iter().enumerate() {

parser/src/earley/mod.rs

@@ -1,6 +1,6 @@
 mod from_guidance;
 mod grammar;
-mod lexer;
+pub(crate) mod lexer;
 mod parser;
 
 pub mod lexerspec;

parser/src/grammar_builder.rs

@@ -2,7 +2,9 @@ use std::{collections::HashMap, sync::atomic::AtomicU32};
 
 use anyhow::{ensure, Result};
 
-use crate::api::{GrammarWithLexer, Node, NodeId, NodeProps, RegexSpec, TopLevelGrammar};
+use crate::api::{
+    GrammarWithLexer, Node, NodeId, NodeProps, RegexId, RegexNode, RegexSpec, TopLevelGrammar,
+};
 
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
 pub struct NodeRef {
@@ -16,6 +18,81 @@ pub struct GrammarBuilder {
     strings: HashMap<String, NodeRef>,
     curr_grammar_id: u32,
     nodes: Vec<Node>,
+    pub regex: RegexBuilder,
+}
+
+pub struct RegexBuilder {
+    node_ids: HashMap<RegexNode, RegexId>,
+    nodes: Vec<RegexNode>,
+}
+
+impl RegexBuilder {
+    pub fn new() -> Self {
+        Self {
+            nodes: vec![],
+            node_ids: HashMap::new(),
+        }
+    }
+
+    pub fn add_node(&mut self, node: RegexNode) -> RegexId {
+        if let Some(id) = self.node_ids.get(&node) {
+            return *id;
+        }
+        let id = RegexId(self.nodes.len());
+        self.nodes.push(node.clone());
+        self.node_ids.insert(node, id);
+        id
+    }
+
+    pub fn regex(&mut self, rx: String) -> RegexId {
+        self.add_node(RegexNode::Regex(rx))
+    }
+
+    pub fn literal(&mut self, s: String) -> RegexId {
+        self.add_node(RegexNode::Literal(s))
+    }
+
+    pub fn concat(&mut self, nodes: Vec<RegexId>) -> RegexId {
+        if nodes.len() == 1 {
+            return nodes[0];
+        }
+        if nodes.len() == 0 {
+            return self.add_node(RegexNode::NoMatch);
+        }
+        self.add_node(RegexNode::Concat(nodes))
+    }
+
+    pub fn select(&mut self, nodes: Vec<RegexId>) -> RegexId {
+        if nodes.len() == 1 {
+            return nodes[0];
+        }
+        if nodes.len() == 0 {
+            return self.add_node(RegexNode::NoMatch);
+        }
+        self.add_node(RegexNode::Or(nodes))
+    }
+
+    pub fn zero_or_more(&mut self, node: RegexId) -> RegexId {
+        self.repeat(node, 0, None)
+    }
+
+    pub fn one_or_more(&mut self, node: RegexId) -> RegexId {
+        self.repeat(node, 1, None)
+    }
+
+    pub fn optional(&mut self, node: RegexId) -> RegexId {
+        self.repeat(node, 0, Some(1))
+    }
+
+    pub fn repeat(&mut self, node: RegexId, min: u32, max: Option<u32>) -> RegexId {
+        self.add_node(RegexNode::Repeat(node, min, max))
+    }
+
+    fn finalize(&mut self) -> Vec<RegexNode> {
+        let r = std::mem::take(&mut self.nodes);
+        *self = Self::new();
+        r
+    }
 }
 
 impl GrammarBuilder {
@@ -49,6 +126,7 @@ impl GrammarBuilder {
             strings: HashMap::new(),
             curr_grammar_id: 0,
             nodes: vec![],
+            regex: RegexBuilder::new(),
         }
     }
 
@@ -62,6 +140,7 @@ impl GrammarBuilder {
                 "no nodes added before add_grammar() or finalize()"
             );
             self.top_grammar.grammars.last_mut().unwrap().nodes = nodes;
+            self.top_grammar.grammars.last_mut().unwrap().rx_nodes = self.regex.finalize();
         }
     }
 
@@ -158,10 +237,19 @@ impl GrammarBuilder {
         self.select(&[value, empty])
     }
 
+    pub fn one_or_more(&mut self, elt: NodeRef) -> NodeRef {
+        let p = self.placeholder();
+        let p_elt = self.join(&[p, elt]);
+        let inner = self.select(&[elt, p_elt]);
+        self.set_placeholder(p, inner);
+        p
+    }
+
     pub fn zero_or_more(&mut self, elt: NodeRef) -> NodeRef {
         let p = self.placeholder();
         let empty = self.empty();
-        let inner = self.select(&[empty, elt]);
+        let p_elt = self.join(&[p, elt]);
+        let inner = self.select(&[empty, p_elt]);
         self.set_placeholder(p, inner);
         p
     }

parser/src/lark/ast.rs

@@ -0,0 +1,97 @@
+/// Represents an item in the grammar (rule, token, or statement).
+#[derive(Debug, Clone)]
+pub enum Item {
+    Rule(Rule),
+    Token(TokenDef),
+    Statement(Statement),
+}
+
+/// Represents a grammar rule.
+#[derive(Debug, Clone)]
+pub struct Rule {
+    pub name: String,
+    pub params: Option<RuleParams>,
+    pub priority: Option<i32>,
+    pub expansions: Expansions,
+}
+
+/// Represents a token definition.
+#[derive(Debug, Clone)]
+pub struct TokenDef {
+    pub name: String,
+    pub params: Option<TokenParams>,
+    pub priority: Option<i32>,
+    pub expansions: Expansions,
+}
+
+/// Represents different types of statements.
+#[derive(Debug, Clone)]
+pub enum Statement {
+    Ignore(Expansions),
+    Import {
+        path: ImportPath,
+        alias: Option<String>,
+    },
+    MultiImport {
+        path: ImportPath,
+        names: Vec<String>,
+    },
+    OverrideRule(Box<Rule>),
+    Declare(Vec<String>),
+}
+
+/// Represents an import path.
+#[derive(Debug, Clone)]
+pub struct ImportPath(pub Vec<String>);
+
+/// Represents parameters for a rule.
+#[derive(Debug, Clone)]
+pub struct RuleParams(pub Vec<String>);
+
+/// Represents parameters for a token.
+#[derive(Debug, Clone)]
+pub struct TokenParams(pub Vec<String>);
+
+/// Represents a list of expansions.
+#[derive(Debug, Clone)]
+pub struct Expansions(pub Vec<Alias>);
+
+/// Represents an alias in the grammar.
+#[derive(Debug, Clone)]
+pub struct Alias {
+    pub expansion: Expansion,
+    pub alias: Option<String>,
+}
+
+/// Represents an expansion consisting of expressions.
+#[derive(Debug, Clone)]
+pub struct Expansion(pub Vec<Expr>);
+
+/// Represents an expression.
+#[derive(Debug, Clone)]
+pub struct Expr {
+    pub atom: Atom,
+    pub op: Option<Op>,
+    pub range: Option<(i32, i32)>,
+}
+
+/// Represents an atom in the grammar.
+#[derive(Debug, Clone)]
+pub enum Atom {
+    Group(Expansions),
+    Maybe(Expansions),
+    Value(Value),
+}
+
+/// Represents different values in the grammar.
+#[derive(Debug, Clone)]
+pub enum Value {
+    LiteralRange(String, String),
+    Name(String),
+    Literal(String),
+    TemplateUsage { name: String, values: Vec<Value> },
+}
+
+/// Represents an operator.
+#[derive(Debug, Clone)]
+pub struct Op(pub String);

parser/src/lark/compiler.rs

@@ -0,0 +1,65 @@
+use std::collections::HashMap;
+
+use anyhow::Result;
+
+use crate::{
+    api::{RegexSpec, TopLevelGrammar},
+    GrammarBuilder, NodeRef,
+};
+
+use super::ast::*;
+
+struct Compiler {
+    builder: GrammarBuilder,
+    items: Vec<Item>,
+    nodes: HashMap<String, NodeInfo>,
+}
+
+struct NodeInfo {
+    id: NodeRef,
+    is_terminal: bool,
+    regex: Option<RegexSpec>,
+}
+
+pub fn lark_to_llguidance(items: Vec<Item>) -> Result<TopLevelGrammar> {
+    let mut c = Compiler {
+        builder: GrammarBuilder::new(),
+        items,
+        nodes: HashMap::new(),
+    };
+    c.execute()?;
+    c.builder.finalize()
+}
+
+impl Compiler {
+    fn execute(&mut self) -> Result<()> {
+        for item in self.items.iter() {
+            match item {
+                Item::Rule(rule) => {
+                    let id = self.builder.placeholder();
+                    self.nodes.insert(
+                        rule.name.clone(),
+                        NodeInfo {
+                            id,
+                            is_terminal: false,
+                            regex: None,
+                        },
+                    );
+                }
+                Item::Token(token_def) => {
+                    let id = self.builder.placeholder();
+                    self.nodes.insert(
+                        token_def.name.clone(),
+                        NodeInfo {
+                            id,
+                            is_terminal: true,
+                            regex: None,
+                        },
+                    );
+                }
+                Item::Statement(statement) => todo!(),
+            }
+        }
+        Ok(())
+    }
+}