Comments re interface (#35)

Co-authored-by: Jeffrey Kegler <[email protected]>
guidance-ai · Oct 30, 2024 · 8de47b9 · 8de47b9
1 parent e9cfc18
commit 8de47b9
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 0 deletions.
diff --git a/parser/src/earley/parser.rs b/parser/src/earley/parser.rs
@@ -624,6 +624,9 @@ impl ParserState {
         }
     }
 
+    // apply_tokens() "pushes" the bytes in 'tokens' into the lexer and parser.  It is a top-level
+    // method in this file.  It is well below llguidance's top-level methods, but in the llguidance
+    // LLinterpreter interface, it is called indirectly via the advance_parser() method.
     pub fn apply_tokens(
         &mut self,
         shared: &mut SharedState,

diff --git a/rust/src/py.rs b/rust/src/py.rs
@@ -27,6 +27,7 @@ struct LLTokenizer {
     tok_bos: Option<u32>,
 }
 
+// This is the interface from llguidance to the LLM's.
 #[pymethods]
 impl LLInterpreter {
     #[new]