update hf tokenizers

microsoft · Nov 6, 2024 · ad6be22 · ad6be22
1 parent 4284824
commit ad6be22
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/core/src/bytes.rs b/core/src/bytes.rs
@@ -2,7 +2,7 @@ use std::mem::size_of;
 
 use anyhow::{anyhow, Result};
 use bytemuck::{NoUninit, Pod};
-use bytemuck_derive::{Pod, Zeroable};
+use bytemuck_derive::Zeroable;
 
 #[derive(Clone, Copy, PartialEq, Eq, Debug, Zeroable, Pod)]
 #[repr(C)]

diff --git a/hf_tokenizers/Cargo.toml b/hf_tokenizers/Cargo.toml
@@ -9,5 +9,5 @@ serde = { version = "1.0.192", features = ["derive"] }
 serde_json = "1.0.108"
 anyhow = "1.0.75"
 rustc-hash = { version = "2.0.0" }
-tokenizers = { version = "0.15.0", features = ["http"] }
+tokenizers = { version = "0.19.1", features = ["http"] }
 log = "0.4.21"