wip: Display final status for a test run

packages/cli/src/databases/entities/test-run.ee.ts

@@ -1,15 +1,16 @@
-import { Column, Entity, Index, ManyToOne, RelationId } from '@n8n/typeorm';
+import { Column, Entity, Index, ManyToOne, OneToMany, RelationId } from '@n8n/typeorm';
 import { IDataObject } from 'n8n-workflow';
 
 import {
 	datetimeColumnType,
 	jsonColumnType,
 	WithTimestampsAndStringId,
 } from '@/databases/entities/abstract-entity';
+import type { TestCaseExecution } from '@/databases/entities/test-case-execution.ee';
 import { TestDefinition } from '@/databases/entities/test-definition.ee';
 import type { TestRunErrorCode } from '@/evaluation.ee/test-runner/errors.ee';
 
-type TestRunStatus = 'new' | 'running' | 'completed' | 'error' | 'cancelled';
+export type TestRunStatus = 'new' | 'running' | 'completed' | 'error' | 'cancelled';
 
 export type AggregatedTestRunMetrics = Record<string, number | boolean>;
 
@@ -69,4 +70,7 @@ export class TestRun extends WithTimestampsAndStringId {
 	 */
 	@Column(jsonColumnType, { nullable: true })
 	errorDetails: IDataObject;
+
+	@OneToMany('TestCaseExecution', 'testRun')
+	testCaseExecutions: TestCaseExecution[];
 }

packages/cli/src/databases/repositories/test-run.repository.ee.ts

@@ -6,8 +6,13 @@ import type { IDataObject } from 'n8n-workflow';
 import type { AggregatedTestRunMetrics } from '@/databases/entities/test-run.ee';
 import { TestRun } from '@/databases/entities/test-run.ee';
 import type { TestRunErrorCode } from '@/evaluation.ee/test-runner/errors.ee';
+import { getTestRunFinalResult } from '@/evaluation.ee/test-runner/utils.ee';
 import type { ListQuery } from '@/requests';
 
+export type TestRunSummary = TestRun & {
+	finalResult: 'success' | 'error' | 'warning';
+};
+
 @Service()
 export class TestRunRepository extends Repository<TestRun> {
 	constructor(dataSource: DataSource) {
@@ -46,7 +51,10 @@ export class TestRunRepository extends Repository<TestRun> {
 	}
 
 	async markAllIncompleteAsFailed() {
-		return await this.update({ status: In(['new', 'running']) }, { status: 'error' });
+		return await this.update(
+			{ status: In(['new', 'running']) },
+			{ status: 'error', errorCode: 'INTERRUPTED' },
+		);
 	}
 
 	async incrementPassed(id: string) {
@@ -58,16 +66,24 @@ export class TestRunRepository extends Repository<TestRun> {
 	}
 
 	async getMany(testDefinitionId: string, options: ListQuery.Options) {
+		// FIXME: optimize fetching final result of each test run
 		const findManyOptions: FindManyOptions<TestRun> = {
 			where: { testDefinition: { id: testDefinitionId } },
 			order: { createdAt: 'DESC' },
+			relations: ['testCaseExecutions'],
 		};
 
 		if (options?.take) {
 			findManyOptions.skip = options.skip;
 			findManyOptions.take = options.take;
 		}
 
-		return await this.find(findManyOptions);
+		const testRuns = await this.find(findManyOptions);
+
+		return testRuns.map(({ testCaseExecutions, ...testRun }) => {
+			const finalResult =
+				testRun.status === 'completed' ? getTestRunFinalResult(testCaseExecutions) : null;
+			return { ...testRun, finalResult };
+		});
 	}
 }

packages/cli/src/evaluation.ee/test-runner/errors.ee.ts

@@ -28,6 +28,7 @@ export class TestCaseExecutionError extends Error {
 export type TestRunErrorCode =
 	| 'PAST_EXECUTIONS_NOT_FOUND'
 	| 'EVALUATION_WORKFLOW_NOT_FOUND'
+	| 'INTERRUPTED'
 	| 'UNKNOWN_ERROR';
 
 export class TestRunError extends Error {

packages/cli/src/evaluation.ee/test-runner/test-runner.service.ee.ts

@@ -284,13 +284,14 @@ export class TestRunnerService {
 			userId: user.id,
 		};
 
-		const evaluationWorkflow = await this.workflowRepository.findById(test.evaluationWorkflowId);
-		if (!evaluationWorkflow) {
-			throw new TestRunError('EVALUATION_WORKFLOW_NOT_FOUND');
-		}
-
 		const abortSignal = abortController.signal;
 		try {
+			// Get the evaluation workflow
+			const evaluationWorkflow = await this.workflowRepository.findById(test.evaluationWorkflowId);
+			if (!evaluationWorkflow) {
+				throw new TestRunError('EVALUATION_WORKFLOW_NOT_FOUND');
+			}
+
 			///
 			// 1. Make test cases from previous executions
 			///
@@ -472,6 +473,7 @@ export class TestRunnerService {
 				await this.testCaseExecutionRepository.markPendingAsCancelled(testRun.id);
 			} else {
 				const aggregatedMetrics = metrics.getAggregatedMetrics();
+
 				await this.testRunRepository.markAsCompleted(testRun.id, aggregatedMetrics);
 
 				this.logger.debug('Test run finished', { testId: test.id });

packages/cli/src/evaluation.ee/test-runner/utils.ee.ts

@@ -1,6 +1,7 @@
 import assert from 'assert';
 import type { IRunExecutionData, IPinData, IWorkflowBase } from 'n8n-workflow';
 
+import type { TestCaseExecution } from '@/databases/entities/test-case-execution.ee';
 import type { MockedNodeItem } from '@/databases/entities/test-definition.ee';
 import type { WorkflowEntity } from '@/databases/entities/workflow-entity';
 import { TestCaseExecutionError } from '@/evaluation.ee/test-runner/errors.ee';
@@ -61,3 +62,15 @@ export function getPastExecutionTriggerNode(executionData: IRunExecutionData) {
 		return !data[0].source || data[0].source.length === 0 || data[0].source[0] === null;
 	});
 }
+
+export function getTestRunFinalResult(testCaseExecutions: TestCaseExecution[]) {
+	for (const testCaseExecution of testCaseExecutions) {
+		if (testCaseExecution.status === 'error') {
+			return 'error';
+		} else if (testCaseExecution.status === 'warning') {
+			return 'warning';
+		}
+	}
+
+	return 'success';
+}

packages/editor-ui/src/api/testDefinition.ee.ts

@@ -49,6 +49,8 @@ export interface TestRunRecord {
 	updatedAt: string;
 	runAt: string;
 	completedAt: string;
+	errorCode?: string;
+	errorDetails?: Record<string, unknown>;
 }
 
 interface GetTestRunParams {

packages/editor-ui/src/components/TestDefinition/ListRuns/TestRunsTable.vue

@@ -22,13 +22,37 @@ const locale = useI18n();
 const navigateToRunDetail = (run: TestRunRecord) => emit('getRunDetail', run);
 const selectedRows = ref<TestRunRecord[]>([]);
 
+// Combine test run statuses and finalResult to get the final status
+const runSummaries = computed(() => {
+	return props.runs.map(({ status, finalResult, ...run }) => {
+		if (status === 'completed') {
+			return { ...run, status: finalResult };
+		}
+
+		return { ...run, status };
+	});
+});
+
 const metrics = computed(() => {
 	return props.runs.reduce((acc, run) => {
 		const metricKeys = Object.keys(run.metrics ?? {});
 		return [...new Set([...acc, ...metricKeys])];
 	}, [] as string[]);
 });
 
+const getErrorTooltipLinkRoute = computed(() => (row: TestRunRecord) => {
+	if (row.errorCode === 'EVALUATION_WORKFLOW_NOT_FOUND') {
+		return {
+			name: VIEWS.TEST_DEFINITION_EDIT,
+			params: {
+				testId: row.testDefinitionId,
+			},
+		};
+	}
+
+	return undefined;
+});
+
 const columns = computed((): Array<TestTableColumn<TestRunRecord>> => {
 	return [
 		{
@@ -51,6 +75,7 @@ const columns = computed((): Array<TestTableColumn<TestRunRecord>> => {
 				{ text: locale.baseText('testDefinition.listRuns.status.error'), value: 'error' },
 				{ text: locale.baseText('testDefinition.listRuns.status.cancelled'), value: 'cancelled' },
 			],
+			errorRoute: getErrorTooltipLinkRoute.value,
 			filterMethod: (value: string, row: TestRunRecord) => row.status === value,
 		},
 		{
@@ -105,7 +130,7 @@ async function deleteRuns() {
 			</n8n-button>
 		</div>
 		<TestTableBase
-			:data="runs"
+			:data="runSummaries"
 			:columns="columns"
 			selectable
 			@row-click="navigateToRunDetail"