Add SKIP lexeme comments (#27)

* Add SKIP lexeme comments * Update parser/src/earley/lexerspec.rs --------- Co-authored-by: Jeffrey Kegler <[email protected]> Co-authored-by: Michał Moskal <[email protected]>
guidance-ai · Oct 21, 2024 · d48dd96 · d48dd96
1 parent 88bd27a
commit d48dd96
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 0 deletions.
diff --git a/parser/src/earley/lexerspec.rs b/parser/src/earley/lexerspec.rs
@@ -32,6 +32,10 @@ pub struct LexemeSpec {
 pub struct LexemeIdx(usize);
 
 impl LexemeIdx {
+    // SKIP is a pseudo-lexeme, a "no-op" which it is sometimes
+    // convenient to insert in the lexeme stream.  As the name
+    // suggests, it is skipped in parsing.
+    // It is typically used for skipping whitespace.
     pub const SKIP: LexemeIdx = LexemeIdx(0);
 
     pub fn new(idx: usize) -> Self {

diff --git a/parser/src/earley/parser.rs b/parser/src/earley/parser.rs
@@ -1350,6 +1350,7 @@ impl ParserState {
         } else {
             self.stats.all_items += row_len;
 
+            // Always accept a SKIP lexeme
             allowed_lexemes.set(LexemeIdx::SKIP.as_usize(), true);
 
             if self.scratch.definitive {
@@ -1613,8 +1614,11 @@ impl ParserState {
         };
 
         let scan_res = if lexeme.idx == LexemeIdx::SKIP {
+            // If this is the SKIP lexeme, then skip it
             self.scan_skip_lexeme(&lexeme)
         } else {
+            // For all but the SKIP lexeme, process this lexeme
+            // with the parser
             self.scan(&lexeme)
         };