From ad6be22981c3db42a89aa6ff6bf54ffa326054b8 Mon Sep 17 00:00:00 2001
From: Michal Moskal <michal@moskal.me>
Date: Wed, 6 Nov 2024 11:07:42 -0800
Subject: [PATCH] update hf tokenizers

---
 core/src/bytes.rs        | 2 +-
 hf_tokenizers/Cargo.toml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/bytes.rs b/core/src/bytes.rs
index 6aa39fd..8807316 100644
--- a/core/src/bytes.rs
+++ b/core/src/bytes.rs
@@ -2,7 +2,7 @@ use std::mem::size_of;
 
 use anyhow::{anyhow, Result};
 use bytemuck::{NoUninit, Pod};
-use bytemuck_derive::{Pod, Zeroable};
+use bytemuck_derive::Zeroable;
 
 #[derive(Clone, Copy, PartialEq, Eq, Debug, Zeroable, Pod)]
 #[repr(C)]
diff --git a/hf_tokenizers/Cargo.toml b/hf_tokenizers/Cargo.toml
index bef3568..0536d99 100644
--- a/hf_tokenizers/Cargo.toml
+++ b/hf_tokenizers/Cargo.toml
@@ -9,5 +9,5 @@ serde = { version = "1.0.192", features = ["derive"] }
 serde_json = "1.0.108"
 anyhow = "1.0.75"
 rustc-hash = { version = "2.0.0" }
-tokenizers = { version = "0.15.0", features = ["http"] }
+tokenizers = { version = "0.19.1", features = ["http"] }
 log = "0.4.21"