From c3ae1f6bec4dc9505e1363776211ead5635ddb18 Mon Sep 17 00:00:00 2001 From: Matthew Thomas Date: Mon, 26 Feb 2024 20:45:34 +0000 Subject: [PATCH] feat: add test objects --- README.md | 23 +++++++++++---- package.json | 2 +- src/generate-report.ts | 64 +++++++++++++++++++++++++++++++++++++++--- 3 files changed, 79 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 0139a76..7e62ce9 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,8 @@ reporter: [ ['wdio-ctrf-json-reporter', { outputFile: 'custom-name.json', // Optional: Output file name. Defaults to 'ctrf-report.json'. outputDir: 'custom-directory', // Optional: Output directory path. Defaults to 'ctrf'. + minimal: true, // Optional: Generate a minimal report. Defaults to 'false'. Overrides screenshot and testType when set to true + testType: 'e2e', // Optional: Specify the test type (e.g., 'api', 'e2e'). Defaults to 'e2e'. appName: 'MyApp', // Optional: Specify the name of the application under test. appVersion: '1.0.0', // Optional: Specify the version of the application under test. osPlatform: 'linux', // Optional: Specify the OS platform. @@ -98,8 +100,19 @@ reporter: [ The test object in the report includes the following [CTRF properties](https://ctrf.io/docs/schema/test): -| Name | Type | Required | Details | -| ---------- | ------ | -------- | ----------------------------------------------------------------------------------- | -| `name` | String | Required | The name of the test. | -| `status` | String | Required | The outcome of the test. One of: `passed`, `failed`, `skipped`, `pending`, `other`. | -| `duration` | Number | Required | The time taken for the test execution, in milliseconds. | +| Name | Type | Required | Details | +| ------------ | ---------------- | -------- | ----------------------------------------------------------------------------------- | +| `name` | String | Required | The name of the test. | +| `status` | String | Required | The outcome of the test. One of: `passed`, `failed`, `skipped`, `pending`, `other`. | +| `duration` | Number | Required | The time taken for the test execution, in milliseconds. | +| `start` | Number | Optional | The start time of the test as a Unix epoch timestamp. | +| `stop` | Number | Optional | The end time of the test as a Unix epoch timestamp. | +| `suite` | String | Optional | The suite or group to which the test belongs. | +| `message` | String | Optional | The failure message if the test failed. | +| `trace` | String | Optional | The stack trace captured if the test failed. | +| `rawStatus` | String | Optional | The original playwright status of the test before mapping to CTRF status. | +| `type` | String | Optional | The type of test (e.g., `api`, `e2e`). | +| `filepath` | String | Optional | The file path where the test is located in the project. | +| `retry` | Number | Optional | The number of retries attempted for the test. | +| `flake` | Boolean | Optional | Indicates whether the test result is flaky. | +| `browser` | String | Optional | The browser used for the test. | \ No newline at end of file diff --git a/package.json b/package.json index f24d512..cbcb325 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "wdio-ctrf-json-reporter", - "version": "0.0.4", + "version": "0.0.6", "description": "", "repository": { "type": "git", diff --git a/src/generate-report.ts b/src/generate-report.ts index 4fe1a38..bbe042c 100644 --- a/src/generate-report.ts +++ b/src/generate-report.ts @@ -1,5 +1,10 @@ -import WDIOReporter, { type RunnerStats, type TestStats } from '@wdio/reporter' +import WDIOReporter, { + type SuiteStats, + type RunnerStats, + type TestStats, +} from '@wdio/reporter' import { + type CtrfTest, type CtrfEnvironment, type CtrfReport, type CtrfTestState, @@ -30,6 +35,9 @@ class GenerateCtrfReport extends WDIOReporter { private readonly reporterName = 'wdio-ctrf-json-reporter' private readonly defaultOutputFile = 'ctrf-report.json' private readonly defaultOutputDir = 'ctrf' + private currentSuite = '' + private currentSpecFile = '' + private currentBrowser = '' constructor(reporterOptions: ReporterConfigOptions) { super(reporterOptions) @@ -37,7 +45,9 @@ class GenerateCtrfReport extends WDIOReporter { this.reporterConfigOptions = { outputFile: reporterOptions?.outputFile ?? this.defaultOutputFile, outputDir: reporterOptions?.outputDir ?? this.defaultOutputDir, + minimal: reporterOptions?.minimal ?? false, appName: reporterOptions?.appName ?? undefined, + testType: reporterOptions?.testType ?? 'e2e', appVersion: reporterOptions?.appVersion ?? undefined, osPlatform: reporterOptions?.osPlatform ?? undefined, osRelease: reporterOptions?.osRelease ?? undefined, @@ -81,8 +91,20 @@ class GenerateCtrfReport extends WDIOReporter { } } - onRunnerStart(): void { + onSuiteStart(suite: SuiteStats): void { + fs.writeFileSync('suitestats.json', JSON.stringify(suite)) + this.currentSuite = suite.fullTitle + this.currentSpecFile = suite.file + } + + onRunnerStart(runner: RunnerStats): void { this.ctrfReport.results.summary.start = Date.now() + const caps: WebdriverIO.Capabilities = runner.capabilities as any + + const browserName = caps.browserName ?? '' + const browserVersion = caps.browserVersion ?? '' + this.currentBrowser = `${browserName} ${browserVersion}` + this.setEnvironmentDetails(this.reporterConfigOptions ?? {}) if (this.hasEnvironmentDetails(this.ctrfEnvironment)) { this.ctrfReport.results.environment = this.ctrfEnvironment @@ -95,6 +117,8 @@ class GenerateCtrfReport extends WDIOReporter { } onRunnerEnd(runner: RunnerStats): void { + fs.writeFileSync('runner-stats.json', JSON.stringify(runner)) + this.ctrfReport.results.summary.stop = Date.now() const specFilePath = runner.specs[0] const pathParts = specFilePath.split(path.sep) @@ -142,11 +166,29 @@ class GenerateCtrfReport extends WDIOReporter { test: TestStats, status: CtrfTestState ): void { - this.ctrfReport.results.tests.push({ + fs.writeFileSync('teststats.json', JSON.stringify(test)) + const ctrfTest: CtrfTest = { name: test.title, status, duration: test._duration, - }) + } + + if (this.reporterConfigOptions.minimal === false) { + ctrfTest.start = Math.floor(test.start.getTime() / 1000) + ctrfTest.stop = + test.end !== undefined ? Math.floor(test.end.getTime() / 1000) : 0 + ctrfTest.message = this.extractFailureDetails(test).message + ctrfTest.trace = this.extractFailureDetails(test).trace + ctrfTest.rawStatus = test.state + ctrfTest.type = this.reporterConfigOptions.testType ?? 'e2e' + ctrfTest.retry = test.retries + ctrfTest.flake = test.state === 'passed' && (test.retries ?? 0) > 0 + ctrfTest.suite = this.currentSuite + ctrfTest.filePath = this.currentSpecFile + ctrfTest.browser = this.currentBrowser + } + + this.ctrfReport.results.tests.push(ctrfTest) } setEnvironmentDetails(reporterConfigOptions: ReporterConfigOptions): void { @@ -177,6 +219,20 @@ class GenerateCtrfReport extends WDIOReporter { return Object.keys(environment).length > 0 } + extractFailureDetails(testResult: TestStats): Partial { + if (testResult.state === 'failed' && testResult.error !== undefined) { + const failureDetails: Partial = {} + if (testResult.error.message !== undefined) { + failureDetails.message = testResult.error.message + } + if (testResult.error.stack !== undefined) { + failureDetails.trace = testResult.error.stack + } + return failureDetails + } + return {} + } + private writeReportToFile(data: CtrfReport): void { const filePath = path.join( this.reporterConfigOptions.outputDir ?? this.defaultOutputDir,