4.0.0

andrefs · Nov 1, 2024 · 6dd0763 · 6dd0763
1 parent 25d5ec6
commit 6dd0763
Show file tree

Hide file tree

Showing 23 changed files with 77 additions and 67 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "punuy-eval",
-  "version": "3.3.0",
+  "version": "4.0.0",
   "description": "",
   "main": "index.js",
   "scripts": {

diff --git a/src/lib/experiments/aux.ts b/src/lib/experiments/aux.ts
@@ -13,7 +13,7 @@ export function normalizeScale(
   return (
     targetScale.min +
     ((value - sourceScale.min) * (targetScale.max - targetScale.min)) /
-    (sourceScale.max - sourceScale.min)
+      (sourceScale.max - sourceScale.min)
   );
 }
 

diff --git a/src/lib/experiments/compare-prompts/index.ts b/src/lib/experiments/compare-prompts/index.ts
@@ -273,7 +273,8 @@ async function performMulti(
 
   const res = [];
   logger.info(
-    `Preparing to run experiment ${name}, ${trials} times on each variable combination (${trials}x${varCombs.length
+    `Preparing to run experiment ${name}, ${trials} times on each variable combination (${trials}x${
+      varCombs.length
     }):\n${varCombs
       .map(vc => "\t" + JSON.stringify(getVarIds(vc)))
       .join(",\n")}.`
@@ -380,8 +381,8 @@ async function evaluate(exps: ExperimentData<CPExpTypes>[]) {
       `🆚 Comparing ${comp.variables
         .map(v => `[${v}]`)
         .join(" and ")} with fixed variables ${JSON.stringify(
-          comp.fixedValueConfig
-        )}\n${tablePP}`
+        comp.fixedValueConfig
+      )}\n${tablePP}`
     );
   }
 

diff --git a/src/lib/experiments/experiment/aux.ts b/src/lib/experiments/experiment/aux.ts
@@ -127,7 +127,8 @@ export async function saveExpVarCombData<T extends GenericExpTypes>(
     `💾 Saving experiment ${name} with traceId ${traceId} to ${filename}.`
   );
   logger.info(
-    `🥇 It ran successfully ${data.results.raw.length}/${data.meta.trials
+    `🥇 It ran successfully ${data.results.raw.length}/${
+      data.meta.trials
     } times with variables ${JSON.stringify(getVarIds(data.variables))}.`
   );
 

diff --git a/src/lib/experiments/experiment/conversation.ts b/src/lib/experiments/experiment/conversation.ts
@@ -91,7 +91,7 @@ export async function getTurnResponse<T extends GenericExpTypes>(
   const failedAttempts = [];
   logger.info(
     `      👥 ${prompt.pairs.length === 1 ? "pair" : "pairs"} ` +
-    prompt.pairs.map(p => `[${p[0]}, ${p[1]}]`).join(", ")
+      prompt.pairs.map(p => `[${p[0]}, ${p[1]}]`).join(", ")
   );
   while (failedAttempts.length < maxTurnAttempts) {
     const faCount = failedAttempts.length;

diff --git a/src/lib/experiments/experiment/perform.ts b/src/lib/experiments/experiment/perform.ts
@@ -67,10 +67,10 @@ export async function performMulti<T extends GenericExpTypes>(
   for (const [index, vc] of varCombs.entries()) {
     logger.info(
       "⚗️  " +
-      pc.inverse(
-        `Running experiment ${index + 1}/${varCombs.length}: ${this.name}`
-      ) +
-      ` with variables ${JSON.stringify(getVarIds(vc))}.`
+        pc.inverse(
+          `Running experiment ${index + 1}/${varCombs.length}: ${this.name}`
+        ) +
+        ` with variables ${JSON.stringify(getVarIds(vc))}.`
     );
     res.push(await this.perform(vc, trials, Date.now(), folder));
     addUsage(this.totalUsage, res[res.length - 1].usage);
@@ -97,7 +97,8 @@ function startUpLogs(
     throw "🧐 No variable combinations to run experiments with, aborting.";
   }
   logger.info(
-    `🔬 Preparing to run experiment ${name
+    `🔬 Preparing to run experiment ${
+      name
     }, ${trials} times on each variable combination (${trials}x${varCombs.length}): \n${varCombs
       .map(vc => "\t" + JSON.stringify(getVarIds(vc)))
       .join(",\n")}.`

diff --git a/src/lib/experiments/experiment/print.ts b/src/lib/experiments/experiment/print.ts
@@ -20,11 +20,11 @@ export function printUsage<T extends GenericExpTypes>(
   }
   logger.info(
     "📈💸 " +
-    (final ? "Final usage" : "Usage") +
-    " estimate:\n" +
-    Object.values(usage)
-      .map(u => `\t${JSON.stringify(u)} `)
-      .join("\n")
+      (final ? "Final usage" : "Usage") +
+      " estimate:\n" +
+      Object.values(usage)
+        .map(u => `\t${JSON.stringify(u)} `)
+        .join("\n")
   );
 }
 
@@ -112,9 +112,9 @@ function varsToStr(vars: Set<string>, variables: ExpVars) {
   return vars.size === 1
     ? variables[Array.from(vars)[0] as keyof ExpVars]!.id
     : Array.from(vars)
-      .map(v => [v, variables[v as keyof ExpVars]!.id])
-      .map(([v, id]) => `${v}=${id}`)
-      .join(";");
+        .map(v => [v, variables[v as keyof ExpVars]!.id])
+        .map(([v, id]) => `${v}=${id}`)
+        .join(";");
 }
 
 export function generateComparisons(
@@ -229,8 +229,8 @@ export function printExpResTable<T extends GenericExpTypes>(
       `🆚 Comparing ${comp.variables
         .map(v => `[${v}]`)
         .join(" and ")} with fixed variables ${JSON.stringify(
-          comp.fixedValueConfig
-        )} \n${tablePP} \n${csv} `
+        comp.fixedValueConfig
+      )} \n${tablePP} \n${csv} `
     );
   }
 }
diff --git a/src/lib/models/anthropic.ts b/src/lib/models/anthropic.ts
@@ -87,11 +87,11 @@ const buildModel = (
         dataObj: msg,
         usage: msg.usage
           ? {
-            inputTokens: msg.usage.input_tokens,
-            outputTokens: msg.usage.output_tokens,
-            totalTokens: msg.usage.input_tokens + msg.usage.output_tokens,
-            modelId,
-          }
+              inputTokens: msg.usage.input_tokens,
+              outputTokens: msg.usage.output_tokens,
+              totalTokens: msg.usage.input_tokens + msg.usage.output_tokens,
+              modelId,
+            }
           : undefined,
         getDataText: () => {
           let dataText;

diff --git a/src/lib/models/cohere.ts b/src/lib/models/cohere.ts
@@ -47,7 +47,7 @@ const buildModel = (
   modelId: ModelId,
   pricing?: ModelPricing
 ): Model => {
-  const makeRequest = async function(
+  const makeRequest = async function (
     prompt: string,
     toolParams: ModelTool
   ): Promise<CohereModelResponse> {
@@ -74,13 +74,13 @@ const buildModel = (
         dataObj: prediction,
         usage: prediction.usage?.billedUnits
           ? {
-            inputTokens: prediction.usage.billedUnits.inputTokens || 0,
-            outputTokens: prediction.usage.billedUnits.outputTokens || 0,
-            totalTokens:
-              (prediction.usage.billedUnits.inputTokens || 0) +
-              (prediction.usage.billedUnits.outputTokens || 0),
-            modelId,
-          }
+              inputTokens: prediction.usage.billedUnits.inputTokens || 0,
+              outputTokens: prediction.usage.billedUnits.outputTokens || 0,
+              totalTokens:
+                (prediction.usage.billedUnits.inputTokens || 0) +
+                (prediction.usage.billedUnits.outputTokens || 0),
+              modelId,
+            }
           : undefined,
         getDataText: () => {
           let dataText;

diff --git a/src/lib/models/google.ts b/src/lib/models/google.ts
@@ -129,11 +129,11 @@ const buildModel = (
         dataObj: result.response.candidates![0],
         usage: result.response.usageMetadata
           ? {
-            inputTokens: result.response.usageMetadata.promptTokenCount!,
-            outputTokens: result.response.usageMetadata.candidatesTokenCount!,
-            totalTokens: result.response.usageMetadata.totalTokenCount!,
-            modelId,
-          }
+              inputTokens: result.response.usageMetadata.promptTokenCount!,
+              outputTokens: result.response.usageMetadata.candidatesTokenCount!,
+              totalTokens: result.response.usageMetadata.totalTokenCount!,
+              modelId,
+            }
           : undefined,
         getDataText: () => {
           let dataText;

diff --git a/src/lib/models/mistral.ts b/src/lib/models/mistral.ts
@@ -84,11 +84,11 @@ const buildModel = (
         dataObj: chatResponse,
         usage: chatResponse.usage
           ? {
-            inputTokens: chatResponse.usage?.promptTokens,
-            outputTokens: chatResponse.usage?.completionTokens,
-            totalTokens: chatResponse.usage?.totalTokens,
-            modelId,
-          }
+              inputTokens: chatResponse.usage?.promptTokens,
+              outputTokens: chatResponse.usage?.completionTokens,
+              totalTokens: chatResponse.usage?.totalTokens,
+              modelId,
+            }
           : undefined,
         getDataText: () => {
           let dataText;

diff --git a/src/lib/models/openai.ts b/src/lib/models/openai.ts
@@ -41,7 +41,7 @@ const buildModel = (
   modelId: ModelId,
   pricing?: ModelPricing
 ) => {
-  const makeRequest = async function(prompt: string, toolParams: ModelTool) {
+  const makeRequest = async function (prompt: string, toolParams: ModelTool) {
     const req = {
       model: modelId,
       messages: [
@@ -78,11 +78,11 @@ const buildModel = (
         dataObj: completion,
         usage: completion.usage
           ? {
-            inputTokens: completion.usage?.prompt_tokens,
-            outputTokens: completion.usage?.completion_tokens,
-            totalTokens: completion.usage?.total_tokens,
-            modelId,
-          }
+              inputTokens: completion.usage?.prompt_tokens,
+              outputTokens: completion.usage?.completion_tokens,
+              totalTokens: completion.usage?.total_tokens,
+              modelId,
+            }
           : undefined,
         getDataText: () => {
           let dataText;

diff --git a/src/scripts/batchVsSinglePair.ts b/src/scripts/batchVsSinglePair.ts
@@ -33,7 +33,8 @@ const bvsp = async (vars: ExpVarMatrix) => {
   for (const exp of res.experiments) {
     logger.info(
       { ...exp.results.aggregated?.resultTypes },
-      `${exp.meta.name} ${JSON.stringify(getVarIds(exp.variables))} ${exp.results.aggregated?.allDataAvg
+      `${exp.meta.name} ${JSON.stringify(getVarIds(exp.variables))} ${
+        exp.results.aggregated?.allDataAvg
       }`
     );
     logger.debug(

diff --git a/src/scripts/compareMC30.ts b/src/scripts/compareMC30.ts
@@ -45,9 +45,9 @@ const compareMC30 = async () => {
   if (res.usage) {
     logger.info(
       "📈 Usage estimate:\n" +
-      Object.values(res.usage)
-        .map(u => `\t${JSON.stringify(u)}`)
-        .join("\n")
+        Object.values(res.usage)
+          .map(u => `\t${JSON.stringify(u)}`)
+          .join("\n")
     );
   }
 

diff --git a/src/scripts/comparePrompts.ts b/src/scripts/comparePrompts.ts
@@ -34,9 +34,9 @@ const comparePromptsMain = async (vars: ExpVarMatrix) => {
   if (res.usage) {
     logger.info(
       "📈 Usage estimate:\n" +
-      Object.values(res.usage)
-        .map(u => `\t${JSON.stringify(u)}`)
-        .join("\n")
+        Object.values(res.usage)
+          .map(u => `\t${JSON.stringify(u)}`)
+          .join("\n")
     );
   }
 

diff --git a/src/scripts/dsNameFromDsSample.ts b/src/scripts/dsNameFromDsSample.ts
@@ -21,7 +21,8 @@ const nameFromSample = async (vars: ExpVarMatrix) => {
   for (const r of res.experiments) {
     logger.info(
       { ...r.results.aggregated?.resultTypes },
-      `${r.meta.name} ${JSON.stringify(getVarIds(r.variables))} ${r.results.aggregated?.okDataAvg
+      `${r.meta.name} ${JSON.stringify(getVarIds(r.variables))} ${
+        r.results.aggregated?.okDataAvg
       }`
     );
   }

diff --git a/src/scripts/dsPaperFromDsName.ts b/src/scripts/dsPaperFromDsName.ts
@@ -20,7 +20,8 @@ const paperFromName = async (vars: ExpVarMatrix) => {
   for (const r of res.experiments) {
     logger.info(
       { ...r.results.aggregated?.resultTypes },
-      `${r.meta.name} ${JSON.stringify(getVarIds(r.variables))} ${r.results.aggregated?.okDataAvg
+      `${r.meta.name} ${JSON.stringify(getVarIds(r.variables))} ${
+        r.results.aggregated?.okDataAvg
       }`
     );
   }

diff --git a/src/scripts/dsSampleFromDsName.ts b/src/scripts/dsSampleFromDsName.ts
@@ -63,7 +63,8 @@ const sampleFromName = async (vars: ExpVarMatrix) => {
   for (const exp of res.experiments) {
     logger.info(
       { ...exp.results.aggregated?.resultTypes },
-      `${exp.meta.name} ${JSON.stringify(getVarIds(exp.variables))} ${exp.results.aggregated?.okDataAvg
+      `${exp.meta.name} ${JSON.stringify(getVarIds(exp.variables))} ${
+        exp.results.aggregated?.okDataAvg
       }`
     );
     logger.debug(

diff --git a/src/scripts/dsSampleFromDsSample.ts b/src/scripts/dsSampleFromDsSample.ts
@@ -27,7 +27,8 @@ const sampleFromSample = async (vars: ExpVarMatrix) => {
   for (const r of res.experiments) {
     logger.info(
       { ...r.results.aggregated?.resultTypes },
-      `${r.meta.name} ${JSON.stringify(getVarIds(r.variables))} ${r.results.aggregated?.okDataAvg
+      `${r.meta.name} ${JSON.stringify(getVarIds(r.variables))} ${
+        r.results.aggregated?.okDataAvg
       }`
     );
   }

diff --git a/src/scripts/dsValuesExactMatches.ts b/src/scripts/dsValuesExactMatches.ts
@@ -27,7 +27,8 @@ const valuesExactMatch = async (vars: ExpVarMatrix) => {
   for (const exp of res.experiments) {
     logger.info(
       { ...exp.results.aggregated?.resultTypes },
-      `${exp.meta.name} ${JSON.stringify(getVarIds(exp.variables))} ${exp.results.aggregated?.okDataAvg
+      `${exp.meta.name} ${JSON.stringify(getVarIds(exp.variables))} ${
+        exp.results.aggregated?.okDataAvg
       }`
     );
     logger.debug(

diff --git a/src/scripts/fix-eval.ts b/src/scripts/fix-eval.ts
@@ -123,7 +123,7 @@ async function main(dirPath: string, expName?: string) {
     const fixedExpVCData = await reEvalExperiment(expVCData, exp);
     if (
       expVCData.results.aggregated?.allDataAvg !==
-      fixedExpVCData.results.aggregated?.allDataAvg &&
+        fixedExpVCData.results.aggregated?.allDataAvg &&
       isNumber(expVCData.results.aggregated?.allDataAvg) &&
       isNumber(fixedExpVCData.results.aggregated?.allDataAvg)
     ) {

diff --git a/src/scripts/predictionCorrelation.ts b/src/scripts/predictionCorrelation.ts
@@ -45,7 +45,8 @@ const predCorr = async (vars: ExpVarMatrix) => {
   for (const exp of res.experiments) {
     logger.info(
       { ...exp.results.aggregated?.resultTypes },
-      `${exp.meta.name} ${JSON.stringify(getVarIds(exp.variables))} ${exp.results.aggregated?.okDataAvg
+      `${exp.meta.name} ${JSON.stringify(getVarIds(exp.variables))} ${
+        exp.results.aggregated?.okDataAvg
       }`
     );
     logger.debug(