feat: update tokenizer

transitive-bullshit · Mar 18, 2023 · 66be552 · 66be552
1 parent fa8012c
commit 66be552
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 5 deletions.
diff --git a/package.json b/package.json
@@ -37,7 +37,7 @@
     "test:prettier": "prettier '**/*.{js,jsx,ts,tsx}' --check"
   },
   "dependencies": {
-    "@dqbd/tiktoken": "^0.4.0",
+    "@dqbd/tiktoken": "^1.0.2",
     "cac": "^6.7.14",
     "conf": "^11.0.1",
     "eventsource-parser": "^0.0.5",

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml