Merge pull request #396 from guardian/ei/add-comment-with-test-results

Add PR comment with visual test results

.eslintignore

@@ -11,6 +11,7 @@ node_modules
 svelte.config.js
 playwright.config.ts
 /playwright
+/scripts
 
 # Ignore files for PNPM, NPM and YARN
 pnpm-lock.yaml

.github/workflows/visual-regression.yml

@@ -6,6 +6,8 @@ on:
   workflow_dispatch:
   pull_request:
 
+permissions: write-all
+
 jobs:
   test:
     name: Visual regression
@@ -35,4 +37,43 @@ jobs:
         run: pnpm dev & npx wait-on -v -i 1000 http://localhost:7777
 
       - name: Run Playwright
+        continue-on-error: true
         run: pnpm playwright test
+
+      - name: Construct results table
+        run: node scripts/analyse-visual-results
+
+      - uses: actions/github-script@v7
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const { readFileSync, resolve } = require('fs');
+            const resultsTable = readFileSync('./test-results/visual-regression-results-table.txt', 'utf8');
+            const body = `### Visual regression testing results 🔍
+            If any tests are failing, please check that any visual changes are intentional before merging your PR.
+
+            ${resultsTable}`
+            if (context.eventName === 'pull_request') {
+              const comments = await github.rest.issues.listComments({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+              });
+              const existingComment = comments.data.find(comment => comment.body.includes('Visual regression testing results'));
+              if (existingComment) {
+                await github.rest.issues.updateComment({
+                  comment_id: existingComment.id,
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  body
+                });
+                return;
+              } else {
+                const { data } = await github.rest.issues.createComment({
+                  issue_number: context.issue.number,
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  body
+                });
+              }
+            }

playwright.config.ts

@@ -13,6 +13,10 @@ const config: PlaywrightTestConfig = {
 	],
 	testDir: './playwright',
 	snapshotPathTemplate: '{testDir}/reference-images/{arg}{ext}',
+	reporter: [
+		['line'],
+		['json', { outputFile: 'test-results/visual-regression-results.json' }],
+	],
 };
 
 export default config;

playwright/fabric-custom.test.ts

@@ -3,7 +3,7 @@ import { localBaseUrl, referenceBaseUrl, templatePreviewWidths } from './utils';
 
 const viewport = { width: 1600, height: 1000 };
 
-test.describe('Fabric Custom visual regression testing', () => {
+test.describe('Fabric Custom', () => {
 	test('Get reference screenshots', async ({ page }) => {
 		await page.setViewportSize(viewport);
 
@@ -45,7 +45,7 @@ test.describe('Fabric Custom visual regression testing', () => {
 			// compare screenshot to reference
 			await expect(testTemplateLocator).toHaveScreenshot(
 				`Fabric-custom-${width.replace('%', '')}.png`,
-				{ animations: 'disabled', maxDiffPixelRatio: 0.004 },
+				{ animations: 'disabled', maxDiffPixelRatio: 0.006 },
 			);
 		}
 	});

playwright/manual-multiple.test.ts

@@ -3,7 +3,7 @@ import { localBaseUrl, referenceBaseUrl, templatePreviewWidths } from './utils';
 
 const viewport = { width: 1600, height: 1000 };
 
-test.describe('Manual Multiple visual regression testing', () => {
+test.describe('Manual Multiple', () => {
 	test('Get reference screenshots', async ({ page }) => {
 		await page.setViewportSize(viewport);
 
@@ -44,7 +44,7 @@ test.describe('Manual Multiple visual regression testing', () => {
 			// compare screenshot to reference
 			await expect(testTemplateLocator).toHaveScreenshot(
 				`Manual-multiple-${width.replace('%', '')}.png`,
-				{ maxDiffPixelRatio: 0.004 },
+				{ maxDiffPixelRatio: 0.006 },
 			);
 		}
 	});

playwright/manual-single.test.ts

@@ -3,7 +3,7 @@ import { localBaseUrl, referenceBaseUrl, templatePreviewWidths } from './utils';
 
 const viewport = { width: 1600, height: 1000 };
 
-test.describe('Manual Single visual regression testing', () => {
+test.describe('Manual Single', () => {
 	test('Get reference screenshots', async ({ page }) => {
 		await page.setViewportSize(viewport);
 
@@ -44,7 +44,7 @@ test.describe('Manual Single visual regression testing', () => {
 			// compare screenshot to reference
 			await expect(testTemplateLocator).toHaveScreenshot(
 				`Manual-single-${width.replace('%', '')}.png`,
-				{ maxDiffPixelRatio: 0.004 },
+				{ maxDiffPixelRatio: 0.006 },
 			);
 		}
 	});

scripts/analyse-visual-results.js

@@ -0,0 +1,29 @@
+import { readFileSync, writeFileSync } from 'fs';
+import path, { resolve } from 'path';
+import { fileURLToPath } from 'url';
+
+// replicate __dirname functionality as this is an es module
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const filePath = resolve(
+	__dirname,
+	`../test-results/visual-regression-results.json`,
+);
+const fileContent = JSON.parse(readFileSync(filePath, 'utf-8'));
+
+const tests = fileContent.suites;
+
+let resultsTable = `| Template | Visual test status |
+| ------------- | ------------- |`;
+
+for (const test of tests) {
+	const testStatus = Boolean(test.suites[0].specs[1].ok);
+	const testTitle = test.suites[0].title;
+	resultsTable += `\n| ${testTitle} | ${testStatus ? '✅' : '❌'} |`;
+}
+
+writeFileSync(
+	resolve(__dirname, `../test-results/visual-regression-results-table.txt`),
+	resultsTable,
+);