diff --git a/parser/src/earley/slicer.rs b/parser/src/earley/slicer.rs index bf1b0dd..6d24053 100644 --- a/parser/src/earley/slicer.rs +++ b/parser/src/earley/slicer.rs @@ -88,6 +88,7 @@ impl SlicedBiasComputer { tokens.push(vec![]); } } + mask.trim_trailing_zeros(); } let entry = TokenizerSlice { diff --git a/toktrie/src/svob.rs b/toktrie/src/svob.rs index ef65a46..faaad56 100644 --- a/toktrie/src/svob.rs +++ b/toktrie/src/svob.rs @@ -289,12 +289,23 @@ impl SimpleVob { } pub fn or(&mut self, other: &SimpleVob) { - assert_eq!(self.size, other.size); + assert!(self.size >= other.size); for (idx, v) in self.data.iter_mut().zip(other.data.iter()) { *idx |= *v; } } + pub fn trim_trailing_zeros(&mut self) { + let mut idx = self.data.len(); + while idx > 0 && self.data[idx - 1] == 0 { + idx -= 1; + } + if self.data.len() != idx { + self.data.truncate(idx); + self.size = self.data.len() * BITS; + } + } + /// self |= other & !minus pub fn or_minus(&mut self, other: &SimpleVob, minus: &SimpleVob) { assert_eq!(self.size, other.size);