chore(e2e): implement extended reporter

.github/workflows/e2e.yml

@@ -8,6 +8,15 @@ on:
 env:
   node-version: 20.x
 
+variables:
+  DIFF_REPORT_BRANCH: diff-report
+
+permissions:
+  contents: write
+  pages: none
+  deployments: none
+  pull-requests: write
+
 jobs:
   e2e-tests:
     name: Automation Testing
@@ -30,6 +39,13 @@ jobs:
         with:
           name: automation-diff-report
           path: e2e/.diff
+      - name: Deploy latest report to branch
+        if: failure()
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: e2e/.diff
+          publish_branch: ${{ env.DIFF_REPORT_BRANCH }}
       - name: Render Report
         if: always()
         shell: bash

e2e/.eslintignore

@@ -3,3 +3,4 @@ jest.config.js
 
 # Common directories
 transformer
+reporters

e2e/jest.config.js

@@ -12,7 +12,11 @@ module.exports = {
   moduleFileExtensions: ['ts', 'js', 'feature'],
   setupFilesAfterEnv: ['./setup/image.ts', './setup/scenarios.ts'],
   reporters: [
-    ["jest-md-dashboard", { title: 'Test Results', outputPath: './.diff/index.md' }],
+    ['./reporters/reporter.js', {
+      diffDir: './.diff',
+      outputPath: './.diff/README.md',
+      outputPublishPath: './.diff/index.md'
+    }],
     ['github-actions', {silent: false}],
     'default'
   ]

e2e/package.json

@@ -10,14 +10,13 @@
     "run:update": "jest -u",
     "run:update:only": "jest -u --no-autorun",
     "run:tests:only": "jest --no-autorun",
-    "run:server": "cd .. && npm run start:test --workspace=site",
+    "run:server": "cd .. && cross-env-shell PORT=3007 npm run start:test --workspace=site",
     "test": "eslint"
   },
   "dependencies": {
     "@types/jest-environment-puppeteer": "^5.0.6",
     "@types/jest-image-snapshot": "^6.4.0",
     "@types/puppeteer": "^7.0.4",
-    "jest-md-dashboard": "^0.7.2",
     "jest-image-snapshot": "^6.4.0",
     "jest-puppeteer": "^9.0.2",
     "puppeteer": "^22.0.0",

e2e/reporters/printers.js

@@ -0,0 +1,60 @@
+const path = require('path');
+
+const printSummary = (stats) => {
+  let text = '\n';
+
+  text += '| :clock10: Start time | :hourglass: Duration |\n';
+  text += '| --- | ---: |\n';
+  text += `|${stats.startTimeStr}|${stats.totalTimeStr}s|\n`;
+  text += '\n';
+
+  text += '| | :white_check_mark: Passed | :x: Failed | :construction: Todo | :white_circle: Total |\n';
+  text += '| --- | ---: | ---: | ---:| ---: |\n';
+  text += `|Test Suites|${stats.numPassedTestSuites}|${stats.numFailedTestSuites}|-|${stats.numTotalTestSuites}|\n`;
+  text += `|Tests|${stats.numPassedTests}|${stats.numFailedTests}|${stats.numTodoTests}|${stats.numTotalTests}|\n`;
+  text += '\n';
+
+  return text;
+};
+
+function printFiles(fileStat, basePath) {
+  let text = '';
+  for (const file of fileStat) {
+    text += `### ${file.filepath}\n`;
+    text += `<table>\n`;
+    text += '<tr><th>Test</th><th>Status</th><th>Time</th></tr>\n';
+    for (const test of file.tests) {
+      const statusTest = test.status === 'passed' ? ':white_check_mark:' : ':x:';
+      const timeStr = test.time < 1 ? `${test.time * 1000}ms` : `${test.time}s`;
+
+      text += `<tr><td>${test.name}:${test.title}</td><td>${statusTest}</td><td>${timeStr}</td></tr>\n`;
+
+      if (test.status !== 'passed' && test.hasSnapshot) {
+        const imgPath = path.join(basePath, test.snapshot);
+        text += `<tr><td colspan="3"><img src="${imgPath}" alt="Test Diff ${test.snapshot}"/></td></tr>`;
+      }
+      if (test.status !== 'passed' && !test.hasSnapshot) {
+        text += `<tr><td colspan="3">\n`;
+        text += '```text\n';
+        text += test.messages.join('\n');
+        text += '\n```\n';
+        text += `</td></tr>\n`;
+      }
+    }
+    text += `<table>\n`;
+  }
+  return text;
+}
+
+function print({stats, files, basePath}) {
+  return `# Test Results
+  ## Summary
+  ${printSummary(stats)}
+
+  ---
+  ## Tests Details
+  ${printFiles(files, basePath)}
+`;
+}
+
+exports.print = print;

e2e/reporters/reporter.js

@@ -0,0 +1,86 @@
+const fs = require('fs');
+const path = require('path');
+
+const {print} = require('./printers');
+
+const sanitize = (str) => (str || '').replace(/\W+/g, '-').toLowerCase();
+
+class SnapshotAwareReporter {
+  constructor(globalConfig, options) {
+    this._globalConfig = globalConfig;
+    this._options = options;
+    this._totalStartTime = performance.now();
+  }
+
+  buildStats(results) {
+    const totalTime = Math.floor(performance.now() - this._totalStartTime);
+    const startTimeStr = new Date(results.startTime).toLocaleString();
+    const totalTimeStr = `${(totalTime / 1000).toFixed(2)}s`;
+
+    return {
+      numFailedTests: results.numFailedTests,
+      numFailedTestSuites: results.numFailedTestSuites,
+      numPassedTests: results.numPassedTests,
+      numPassedTestSuites: results.numPassedTestSuites,
+      numPendingTests: results.numPendingTests,
+      numTodoTests: results.numTodoTests,
+      numPendingTestSuites: results.numPendingTestSuites,
+      numRuntimeErrorTestSuites: results.numRuntimeErrorTestSuites,
+      numTotalTests: results.numTotalTests,
+      numTotalTestSuites: results.numTotalTestSuites,
+      startTime: results.startTime,
+      startTimeStr,
+      totalTime,
+      totalTimeStr
+    };
+  }
+
+  buildTestStat(test, testPath) {
+    const { ancestorTitles, status, title, duration} = test;
+    const filename = path.basename(testPath);
+    const name = ancestorTitles.join(' > ');
+    const statBase = {name, filename, status, title, time: duration};
+
+    if (status === 'passed') return statBase;
+
+    const snapshotParts = [filename, ...ancestorTitles, title, '1-snap', 'diff'];
+    const snapshotName = snapshotParts.map(sanitize).join('-') + '.png'
+    const snapshotPath = path.join(this._options.diffDir, snapshotName);
+    const snapshotExists = fs.existsSync(snapshotPath);
+
+    return Object.assign(statBase, {
+      message: test.failureMessages[0],
+      messages: test.failureMessages,
+      hasSnapshot: snapshotExists,
+      snapshot: snapshotExists ? snapshotName : null
+    });
+  }
+
+  buildTestResults(results) {
+    const testResults = [];
+    const basePath = path.resolve(this._globalConfig.rootDir);
+    for (const result of results.testResults) {
+      const filepath = path.relative(basePath, result.testFilePath);
+      const tests = result.testResults.map(test => this.buildTestStat(test, filepath));
+      testResults.push({filepath, tests});
+    }
+    return testResults;
+  }
+
+  async onRunComplete(contexts, results) {
+    const stats = this.buildStats(results);
+    const files = this.buildTestResults(results);
+    fs.writeFileSync(this._options.outputPath, print({stats, files, basePath: '.'}));
+
+    if (process.env.GITHUB_ACTIONS && process.env.DIFF_REPORT_BRANCH && this._options.outputPublishPath) {
+      const serverUrl = process.env.GITHUB_SERVER_URL;
+      const repository = process.env.GITHUB_REPOSITORY;
+      const branch = process.env.DIFF_REPORT_BRANCH;
+      const basePath = `${serverUrl}/${repository}/blob/${branch}/`;
+      fs.writeFileSync(this._options.outputPublishPath, print({stats, files, basePath}));
+    }
+  }
+}
+
+module.exports = SnapshotAwareReporter;
+module.exports.default = SnapshotAwareReporter;