diff --git a/package-lock.json b/package-lock.json index f2461511..6db71b9c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5270,6 +5270,14 @@ } ] }, + "node_modules/queue-promise": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/queue-promise/-/queue-promise-2.2.1.tgz", + "integrity": "sha512-C3eyRwLF9m6dPV4MtqMVFX+Xmc7keZ9Ievm3jJ/wWM5t3uVbFnGsJXwpYzZ4LaIEcX9bss/mdaKzyrO6xheRuA==", + "engines": { + "node": ">=8.12.0" + } + }, "node_modules/queue-tick": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", @@ -6992,6 +7000,7 @@ "p-queue": "^8.0.1", "postgres": "^3.4.3", "prexit": "^2.2.0", + "queue-promise": "^2.2.1", "rouge": "^1.0.3", "samlify": "^2.8.11", "shared": "*", diff --git a/packages/backend/package.json b/packages/backend/package.json index 64e742bc..bc99d10d 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -35,6 +35,7 @@ "p-queue": "^8.0.1", "postgres": "^3.4.3", "prexit": "^2.2.0", + "queue-promise": "^2.2.1", "rouge": "^1.0.3", "samlify": "^2.8.11", "shared": "*", diff --git a/packages/backend/src/api/v1/evaluations/index.ts b/packages/backend/src/api/v1/evaluations/index.ts index bf9fc112..d44c17c8 100644 --- a/packages/backend/src/api/v1/evaluations/index.ts +++ b/packages/backend/src/api/v1/evaluations/index.ts @@ -6,7 +6,8 @@ import sql from "@/src/utils/db" import Context from "@/src/utils/koa" import Router from "koa-router" import { RunEvent } from "lunary/types" -import PQueue from "p-queue" + +import Queue from "queue-promise" import { PassThrough } from "stream" import { runEval } from "./utils" @@ -31,9 +32,9 @@ evaluations.post( Connection: "keep-alive", }) - const queue = new PQueue({ - concurrency: MAX_PARALLEL_EVALS, - timeout: 10000, + const queue = new Queue({ + concurrent: MAX_PARALLEL_EVALS, + start: true, }) const [{ plan }] = @@ -70,16 +71,17 @@ evaluations.post( for (const variation of variations) { for (const provider of evaluation.providers) { count++ - queue.add(() => - runEval({ + queue.enqueue(async () => { + await runEval({ evaluationId: evaluation.id, promptId: prompt.id, variation, provider, prompt: prompt.messages, checklistId, - }), - ) + }) + console.log(`Task ${count} don with model ${provider.model} done`) + }) } } } @@ -89,17 +91,24 @@ evaluations.post( ctx.status = 200 ctx.body = stream - queue.on("active", () => { - const percentDone = ((count - queue.size) / count) * 100 - console.log(`Active: ${queue.size} of ${count} (${percentDone}%)`) + let done = 0 + + queue.on("dequeue", () => { + done++ + const percentDone = (1 - (count - done) / count) * 100 + console.log(`Active: ${done} of ${count} (${percentDone}%)`) stream.write(JSON.stringify({ percentDone }) + "\n") }) - await queue.onIdle() + console.log(`Queue started with ${count} tasks`) - stream.write(JSON.stringify({ id: evaluation?.id }) + "\n") + queue.on("end", () => { + console.log("Queue is empty now") - stream.end() + stream.write(JSON.stringify({ id: evaluation?.id }) + "\n") + + stream.end() + }) }, ) diff --git a/packages/backend/src/api/v1/evaluations/utils.ts b/packages/backend/src/api/v1/evaluations/utils.ts index 1ffa8d84..fc294fc3 100644 --- a/packages/backend/src/api/v1/evaluations/utils.ts +++ b/packages/backend/src/api/v1/evaluations/utils.ts @@ -117,6 +117,8 @@ export async function runEval({ duration, })} ` + + console.log(`Eval for ${provider.model} passed: ${passed}`) } catch (error: any) { await sql` insert into evaluation_result ${sql({ diff --git a/packages/backend/src/utils/license.ts b/packages/backend/src/utils/license.ts index 7654941a..cdf38f6f 100644 --- a/packages/backend/src/utils/license.ts +++ b/packages/backend/src/utils/license.ts @@ -22,7 +22,6 @@ async function licenseMiddleware(ctx: Context, next: Next) { try { if (Date.now() - cache.lastFetch > TWO_HOURS) { - console.log("Fetching") const licenseData = await fetch( `https://license.lunary.ai/v1/licenses/${LICENSE_KEY}`, ).then((res) => res.json()) diff --git a/packages/frontend/components/evals/ResultsMatrix.tsx b/packages/frontend/components/evals/ResultsMatrix.tsx index b9dbbe07..d23f5f8a 100644 --- a/packages/frontend/components/evals/ResultsMatrix.tsx +++ b/packages/frontend/components/evals/ResultsMatrix.tsx @@ -68,7 +68,7 @@ function ResultCell({ result }) { <> {result.status === "success" ? ( - +