diff --git a/parser/src/earley/lexerspec.rs b/parser/src/earley/lexerspec.rs index c40a94fe..580d7072 100644 --- a/parser/src/earley/lexerspec.rs +++ b/parser/src/earley/lexerspec.rs @@ -32,6 +32,10 @@ pub struct LexemeSpec { pub struct LexemeIdx(usize); impl LexemeIdx { + // SKIP is a pseudo-lexeme, a "no-op" which it is sometimes + // convenient to insert in the lexeme stream. As the name + // suggests, it is skipped in parsing. + // It is typically used for skipping whitespace. pub const SKIP: LexemeIdx = LexemeIdx(0); pub fn new(idx: usize) -> Self { diff --git a/parser/src/earley/parser.rs b/parser/src/earley/parser.rs index d1aae963..9b71742e 100644 --- a/parser/src/earley/parser.rs +++ b/parser/src/earley/parser.rs @@ -1350,6 +1350,7 @@ impl ParserState { } else { self.stats.all_items += row_len; + // Always accept a SKIP lexeme allowed_lexemes.set(LexemeIdx::SKIP.as_usize(), true); if self.scratch.definitive { @@ -1613,8 +1614,11 @@ impl ParserState { }; let scan_res = if lexeme.idx == LexemeIdx::SKIP { + // If this is the SKIP lexeme, then skip it self.scan_skip_lexeme(&lexeme) } else { + // For all but the SKIP lexeme, process this lexeme + // with the parser self.scan(&lexeme) };