fix(js): Separate Jest suites into separate tables when reporting (#1443)

Author: jacoblee93

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [

js/src/index.ts

@@ -18,4 +18,4 @@ export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 
 // Update using yarn bump-version
-export const __version__ = "0.3.1";
+export const __version__ = "0.3.2";

js/src/jest/reporter.ts

@@ -5,11 +5,22 @@ import { printReporterTable } from "../utils/jestlike/reporter.js";
 
 class LangSmithEvalReporter extends DefaultReporter {
   async onTestResult(test: any, testResult: any, aggregatedResults: any) {
+    const groupedTestResults = testResult.testResults.reduce(
+      (groups: Record<string, any>, testResult: any) => {
+        const ancestorTitle = testResult.ancestorTitles.join(" > ");
+        if (groups[ancestorTitle] === undefined) {
+          groups[ancestorTitle] = [];
+        }
+        groups[ancestorTitle].push(testResult);
+        return groups;
+      },
+      {}
+    );
     try {
-      await printReporterTable(
-        testResult.testResults,
-        testResult.failureMessage
-      );
+      for (const testGroupName of Object.keys(groupedTestResults)) {
+        const resultGroup = groupedTestResults[testGroupName];
+        await printReporterTable(resultGroup, testResult.failureMessage);
+      }
     } catch (e: any) {
       console.log("Failed to display LangSmith eval results:", e.message);
       super.onTestResult(test, testResult, aggregatedResults);

js/src/tests/jestlike/jest.test.ts

@@ -192,7 +192,7 @@ const scoreMarketingCopyAgent = async () => {
   };
 };
 
-ls.describe.only("Test Tweet", () => {
+ls.describe("Test Tweet", () => {
   ls.test(
     "should generate a tweet LS",
     {

js/src/utils/jestlike/reporter.ts

@@ -9,6 +9,7 @@ import { ScoreType } from "../../schemas.js";
 import { STRIP_ANSI_REGEX, TEST_ID_DELIMITER } from "./index.js";
 
 const FEEDBACK_COLLAPSE_THRESHOLD = 48;
+const MAX_TEST_PARAMS_LENGTH = 18;
 
 const RESERVED_KEYS = [
   "Name",
@@ -60,7 +61,9 @@ function formatValue(value: unknown) {
         const rawValue = typeof v === "string" ? v : JSON.stringify(v);
         const rawEntry = `${k}: ${rawValue}`;
         const entry =
-          rawEntry.length > 24 ? rawEntry.slice(0, 21) + "..." : rawEntry;
+          rawEntry.length > MAX_TEST_PARAMS_LENGTH
+            ? rawEntry.slice(0, MAX_TEST_PARAMS_LENGTH - 3) + "..."
+            : rawEntry;
         return entry;
       })
       .join("\n");
@@ -223,9 +226,13 @@ export async function printReporterTable(
     minLen?: number;
   }[] = [
     { name: "Test", alignment: "left", maxLen: 36 },
-    { name: "Inputs", alignment: "left", minLen: 24 },
-    { name: "Reference Outputs", alignment: "left", minLen: 24 },
-    { name: "Outputs", alignment: "left", minLen: 24 },
+    { name: "Inputs", alignment: "left", minLen: MAX_TEST_PARAMS_LENGTH },
+    {
+      name: "Reference Outputs",
+      alignment: "left",
+      minLen: MAX_TEST_PARAMS_LENGTH,
+    },
+    { name: "Outputs", alignment: "left", minLen: MAX_TEST_PARAMS_LENGTH },
     { name: "Status", alignment: "left" },
   ];
   if (collapseFeedbackColumn) {
@@ -245,7 +252,7 @@ export async function printReporterTable(
     defaultColumns.push({
       name: "Feedback",
       alignment: "left",
-      minLen: feedbackColumnLength + 10,
+      minLen: feedbackColumnLength + 8,
     });
   }
   console.log();