feat: add test objects

ctrf-io · Feb 26, 2024 · c3ae1f6 · c3ae1f6
1 parent 8f34015
commit c3ae1f6
Show file tree

Hide file tree

Showing 3 changed files with 79 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -83,6 +83,8 @@ reporter: [
     ['wdio-ctrf-json-reporter', {
         outputFile: 'custom-name.json', // Optional: Output file name. Defaults to 'ctrf-report.json'.
         outputDir: 'custom-directory',  // Optional: Output directory path. Defaults to 'ctrf'.
+         minimal: true,                  // Optional: Generate a minimal report. Defaults to 'false'. Overrides screenshot and testType when set to true
+        testType: 'e2e',                // Optional: Specify the test type (e.g., 'api', 'e2e'). Defaults to 'e2e'.
         appName: 'MyApp',               // Optional: Specify the name of the application under test.
         appVersion: '1.0.0',            // Optional: Specify the version of the application under test.
         osPlatform: 'linux',            // Optional: Specify the OS platform.
@@ -98,8 +100,19 @@ reporter: [
 
 The test object in the report includes the following [CTRF properties](https://ctrf.io/docs/schema/test):
 
-| Name       | Type   | Required | Details                                                                             |
-| ---------- | ------ | -------- | ----------------------------------------------------------------------------------- |
-| `name`     | String | Required | The name of the test.                                                               |
-| `status`   | String | Required | The outcome of the test. One of: `passed`, `failed`, `skipped`, `pending`, `other`. |
-| `duration` | Number | Required | The time taken for the test execution, in milliseconds.                             |
+| Name         | Type             | Required | Details                                                                             |
+| ------------ | ---------------- | -------- | ----------------------------------------------------------------------------------- |
+| `name`       | String           | Required | The name of the test.                                                               |
+| `status`     | String           | Required | The outcome of the test. One of: `passed`, `failed`, `skipped`, `pending`, `other`. |
+| `duration`   | Number           | Required | The time taken for the test execution, in milliseconds.                             |
+| `start`      | Number           | Optional | The start time of the test as a Unix epoch timestamp.                               |
+| `stop`       | Number           | Optional | The end time of the test as a Unix epoch timestamp.                                 |
+| `suite`      | String           | Optional | The suite or group to which the test belongs.                                       |
+| `message`    | String           | Optional | The failure message if the test failed.                                             |
+| `trace`      | String           | Optional | The stack trace captured if the test failed.                                        |
+| `rawStatus`  | String           | Optional | The original playwright status of the test before mapping to CTRF status.           |
+| `type`       | String           | Optional | The type of test (e.g., `api`, `e2e`).                                              |
+| `filepath`   | String           | Optional | The file path where the test is located in the project.                             |
+| `retry`      | Number           | Optional | The number of retries attempted for the test.                                       |
+| `flake`      | Boolean          | Optional | Indicates whether the test result is flaky.                                         |
+| `browser`    | String           | Optional | The browser used for the test.                                                      |
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "wdio-ctrf-json-reporter",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "description": "",
   "repository": {
     "type": "git",

diff --git a/src/generate-report.ts b/src/generate-report.ts
@@ -1,5 +1,10 @@
-import WDIOReporter, { type RunnerStats, type TestStats } from '@wdio/reporter'
+import WDIOReporter, {
+  type SuiteStats,
+  type RunnerStats,
+  type TestStats,
+} from '@wdio/reporter'
 import {
+  type CtrfTest,
   type CtrfEnvironment,
   type CtrfReport,
   type CtrfTestState,
@@ -30,14 +35,19 @@ class GenerateCtrfReport extends WDIOReporter {
   private readonly reporterName = 'wdio-ctrf-json-reporter'
   private readonly defaultOutputFile = 'ctrf-report.json'
   private readonly defaultOutputDir = 'ctrf'
+  private currentSuite = ''
+  private currentSpecFile = ''
+  private currentBrowser = ''
 
   constructor(reporterOptions: ReporterConfigOptions) {
     super(reporterOptions)
 
     this.reporterConfigOptions = {
       outputFile: reporterOptions?.outputFile ?? this.defaultOutputFile,
       outputDir: reporterOptions?.outputDir ?? this.defaultOutputDir,
+      minimal: reporterOptions?.minimal ?? false,
       appName: reporterOptions?.appName ?? undefined,
+      testType: reporterOptions?.testType ?? 'e2e',
       appVersion: reporterOptions?.appVersion ?? undefined,
       osPlatform: reporterOptions?.osPlatform ?? undefined,
       osRelease: reporterOptions?.osRelease ?? undefined,
@@ -81,8 +91,20 @@ class GenerateCtrfReport extends WDIOReporter {
     }
   }
 
-  onRunnerStart(): void {
+  onSuiteStart(suite: SuiteStats): void {
+    fs.writeFileSync('suitestats.json', JSON.stringify(suite))
+    this.currentSuite = suite.fullTitle
+    this.currentSpecFile = suite.file
+  }
+
+  onRunnerStart(runner: RunnerStats): void {
     this.ctrfReport.results.summary.start = Date.now()
+    const caps: WebdriverIO.Capabilities = runner.capabilities as any
+
+    const browserName = caps.browserName ?? ''
+    const browserVersion = caps.browserVersion ?? ''
+    this.currentBrowser = `${browserName} ${browserVersion}`
+
     this.setEnvironmentDetails(this.reporterConfigOptions ?? {})
     if (this.hasEnvironmentDetails(this.ctrfEnvironment)) {
       this.ctrfReport.results.environment = this.ctrfEnvironment
@@ -95,6 +117,8 @@ class GenerateCtrfReport extends WDIOReporter {
   }
 
   onRunnerEnd(runner: RunnerStats): void {
+    fs.writeFileSync('runner-stats.json', JSON.stringify(runner))
+
     this.ctrfReport.results.summary.stop = Date.now()
     const specFilePath = runner.specs[0]
     const pathParts = specFilePath.split(path.sep)
@@ -142,11 +166,29 @@ class GenerateCtrfReport extends WDIOReporter {
     test: TestStats,
     status: CtrfTestState
   ): void {
-    this.ctrfReport.results.tests.push({
+    fs.writeFileSync('teststats.json', JSON.stringify(test))
+    const ctrfTest: CtrfTest = {
       name: test.title,
       status,
       duration: test._duration,
-    })
+    }
+
+    if (this.reporterConfigOptions.minimal === false) {
+      ctrfTest.start = Math.floor(test.start.getTime() / 1000)
+      ctrfTest.stop =
+        test.end !== undefined ? Math.floor(test.end.getTime() / 1000) : 0
+      ctrfTest.message = this.extractFailureDetails(test).message
+      ctrfTest.trace = this.extractFailureDetails(test).trace
+      ctrfTest.rawStatus = test.state
+      ctrfTest.type = this.reporterConfigOptions.testType ?? 'e2e'
+      ctrfTest.retry = test.retries
+      ctrfTest.flake = test.state === 'passed' && (test.retries ?? 0) > 0
+      ctrfTest.suite = this.currentSuite
+      ctrfTest.filePath = this.currentSpecFile
+      ctrfTest.browser = this.currentBrowser
+    }
+
+    this.ctrfReport.results.tests.push(ctrfTest)
   }
 
   setEnvironmentDetails(reporterConfigOptions: ReporterConfigOptions): void {
@@ -177,6 +219,20 @@ class GenerateCtrfReport extends WDIOReporter {
     return Object.keys(environment).length > 0
   }
 
+  extractFailureDetails(testResult: TestStats): Partial<CtrfTest> {
+    if (testResult.state === 'failed' && testResult.error !== undefined) {
+      const failureDetails: Partial<CtrfTest> = {}
+      if (testResult.error.message !== undefined) {
+        failureDetails.message = testResult.error.message
+      }
+      if (testResult.error.stack !== undefined) {
+        failureDetails.trace = testResult.error.stack
+      }
+      return failureDetails
+    }
+    return {}
+  }
+
   private writeReportToFile(data: CtrfReport): void {
     const filePath = path.join(
       this.reporterConfigOptions.outputDir ?? this.defaultOutputDir,