From d48dd96ffe59676b12e2117948950b3c9f7774c5 Mon Sep 17 00:00:00 2001
From: v-jkegler <v-jkegler@microsoft.com>
Date: Mon, 21 Oct 2024 12:16:54 -0400
Subject: [PATCH] Add SKIP lexeme comments (#27)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add SKIP lexeme comments

* Update parser/src/earley/lexerspec.rs

---------

Co-authored-by: Jeffrey Kegler <jeffreykegler.notices@protonmail.com>
Co-authored-by: Michał Moskal <michal@moskal.me>
---
 parser/src/earley/lexerspec.rs | 4 ++++
 parser/src/earley/parser.rs    | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/parser/src/earley/lexerspec.rs b/parser/src/earley/lexerspec.rs
index c40a94fe..580d7072 100644
--- a/parser/src/earley/lexerspec.rs
+++ b/parser/src/earley/lexerspec.rs
@@ -32,6 +32,10 @@ pub struct LexemeSpec {
 pub struct LexemeIdx(usize);
 
 impl LexemeIdx {
+    // SKIP is a pseudo-lexeme, a "no-op" which it is sometimes
+    // convenient to insert in the lexeme stream.  As the name
+    // suggests, it is skipped in parsing.
+    // It is typically used for skipping whitespace.
     pub const SKIP: LexemeIdx = LexemeIdx(0);
 
     pub fn new(idx: usize) -> Self {
diff --git a/parser/src/earley/parser.rs b/parser/src/earley/parser.rs
index d1aae963..9b71742e 100644
--- a/parser/src/earley/parser.rs
+++ b/parser/src/earley/parser.rs
@@ -1350,6 +1350,7 @@ impl ParserState {
         } else {
             self.stats.all_items += row_len;
 
+            // Always accept a SKIP lexeme
             allowed_lexemes.set(LexemeIdx::SKIP.as_usize(), true);
 
             if self.scratch.definitive {
@@ -1613,8 +1614,11 @@ impl ParserState {
         };
 
         let scan_res = if lexeme.idx == LexemeIdx::SKIP {
+            // If this is the SKIP lexeme, then skip it
             self.scan_skip_lexeme(&lexeme)
         } else {
+            // For all but the SKIP lexeme, process this lexeme
+            // with the parser
             self.scan(&lexeme)
         };