add tokenize_is_approximate() method to TokenizerEnv trait

microsoft · Nov 25, 2024 · e22222c · e22222c
1 parent f4c1f04
commit e22222c
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/core/src/toktree.rs b/core/src/toktree.rs
@@ -161,6 +161,13 @@ pub trait TokenizerEnv: Send {
     fn eos_token(&self) -> TokenId {
         self.tok_trie().eos_token()
     }
+
+    /// If this returns true, this tokenizer may return non-canonical tokenizations
+    /// and should generally not be used for forcing tokens.
+    /// Typically, it will just use TokTrie::greedy_tokenize().
+    fn tokenize_is_approximate(&self) -> bool {
+        false
+    }
 }
 
 pub type TokEnv = Arc<dyn TokenizerEnv + Sync + 'static>;