Add optional toktrie_hf_tokenizers dependency

python_ext/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 
 [dependencies]
 llguidance = { workspace = true }
-# toktrie_hf_tokenizers = { workspace = true }
+toktrie_hf_tokenizers = { workspace = true, optional = true }
 bytemuck = "1.19.0"
 pyo3 = {version = "0.21.2", features = ["extension-module", "abi3-py39"]}
 serde = { version = "1.0.210", features = ["derive"] }
@@ -16,3 +16,7 @@ serde_json = "1.0.132"
 name = "_lib"
 crate-type = ["cdylib"]
 path = "src/lib.rs"
+
+[features]
+default = []
+tokenizers = ["toktrie_hf_tokenizers"]

python_ext/src/py.rs

@@ -157,13 +157,21 @@ impl LLTokenizer {
         slices: Option<Vec<String>>,
     ) -> PyResult<Self> {
         let tok_env: TokEnv = if let Some(_tokenizer_str) = tokenizer.extract::<String>().ok() {
-            let _ = n_vocab;
-            return Err(PyValueError::new_err(
-                "Expecting a TokenizerWrapper() class, not a string",
-            ));
-            // let tok = toktrie_hf_tokenizers::ByteTokenizerEnv::from_name(&tokenizer_str, n_vocab)
-            //     .map_err(val_error)?;
-            // tok.to_env()
+            #[cfg(feature = "tokenizers")]
+            {
+                let tok =
+                    toktrie_hf_tokenizers::ByteTokenizerEnv::from_name(&_tokenizer_str, n_vocab)
+                        .map_err(val_error)?;
+                tok.to_env()
+            }
+
+            #[cfg(not(feature = "tokenizers"))]
+            {
+                let _ = n_vocab;
+                return Err(PyValueError::new_err(
+                    "Expecting a TokenizerWrapper() class, not a string",
+                ));
+            }
         } else {
             Arc::new(PyTokenizer::py_new(tokenizer)?)
         };