bug: fix packing bug

Author: zhzLuke96

package.json

@@ -16,7 +16,9 @@
   "license": "AGPL-3.0-only",
   "dependencies": {
     "@anthropic-ai/vertex-sdk": "^0.4.0",
-    "@lenml/tokenizer-claude": "^1.0.4",
+    "@huggingface/jinja": "^0.2.2",
+    "@lenml/tokenizer-claude": "^1.0.10",
+    "@lenml/tokenizers": "^1.0.9",
     "dotenv": "^16.4.5",
     "fastify": "^4.28.1",
     "fastify-sse-v2": "^4.0.0",

pnpm-lock.yaml

@@ -51,7 +51,7 @@ export default {
     }),
     esbuild({
       include: /\.[jt]sx?$/,
-      // exclude: /node_modules/,
+      exclude: [],
       sourceMap: false,
       minify: process.env.NODE_ENV === "production",
       target: "node14",

rollup.config.mjs

@@ -1,7 +1,11 @@
 import { OpenAI_NS } from "./tyeps";
 import { cloneDeep } from "./utils";
 
-import { fromPreTrained } from "@lenml/tokenizer-claude";
+import {
+  tokenizerJSON,
+  tokenizerConfig,
+} from "@lenml/tokenizer-claude/src/data.ts";
+import { TokenizerLoader } from "@lenml/tokenizers/src/main.ts";
 
 const rename_roles = {
   user: "Human",
@@ -10,7 +14,10 @@ const rename_roles = {
   example_assistant: "A",
 } as Record<string, string>;
 
-const tokenizer = fromPreTrained();
+const tokenizer = TokenizerLoader.fromPreTrained({
+  tokenizerConfig,
+  tokenizerJSON,
+});
 
 export enum SystemMergeMode {
   /**

src/PromptMerger.ts

@@ -6,7 +6,9 @@
     "strict": true,
     "esModuleInterop": true,
     "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": true,
+    "allowImportingTsExtensions": true
   },
   "include": ["src"],
   "exclude": ["node_modules", "dist"]