Donwload tiktokenizer model in prod Dockerfile

Dockerfile

@@ -30,6 +30,8 @@ USER $user
 RUN mkdir .sophia
 # Generate the function schemas
 RUN npm run functionSchemas
+# Download the tiktokenizer model
+RUN npm run initTiktokenizer
 
 ENV NODE_ENV=production
 ENV PORT=8080

package.json

@@ -30,6 +30,7 @@
     "watch": "   node --env-file=variables/local.env -r ts-node/register src/cli/watch.ts",
     "build": "    tsc",
     "build:dev": "tsc -project tsconfig.swc.json",
+    "initTiktokenizer": "node --env-file=variables/local.env -r ts-node/register src/initTiktokenizer.ts",
     "functionSchemas": "node --env-file=variables/local.env -r ts-node/register src/generateFunctionSchemas.ts",
     "start": "      node -r ts-node/register src/index.ts",
     "start:local": "node -r ts-node/register --env-file=variables/local.env --inspect=0.0.0.0:9229 src/index.ts",

src/initTiktokenizer.ts

@@ -0,0 +1,7 @@
+import { countTokens } from '#llm/tokens';
+
+// node_modules is read-only for the sophia user in prod, so download in the docker build
+countTokens('hi').catch((err) => {
+	console.error('Failed to download tiktoken model');
+	console.error(err);
+}).finally(() => console.log('Done'));