diff --git a/.eslintignore b/.eslintignore index ad99bff0..590e74e8 100644 --- a/.eslintignore +++ b/.eslintignore @@ -11,6 +11,7 @@ node_modules svelte.config.js playwright.config.ts /playwright +/scripts # Ignore files for PNPM, NPM and YARN pnpm-lock.yaml diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 68df3de4..3fba5625 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -6,6 +6,8 @@ on: workflow_dispatch: pull_request: +permissions: write-all + jobs: test: name: Visual regression @@ -35,4 +37,43 @@ jobs: run: pnpm dev & npx wait-on -v -i 1000 http://localhost:7777 - name: Run Playwright + continue-on-error: true run: pnpm playwright test + + - name: Construct results table + run: node scripts/analyse-visual-results + + - uses: actions/github-script@v7 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const { readFileSync, resolve } = require('fs'); + const resultsTable = readFileSync('./test-results/visual-regression-results-table.txt', 'utf8'); + const body = `### Visual regression testing results 🔍 + If any tests are failing, please check that any visual changes are intentional before merging your PR. + + ${resultsTable}` + if (context.eventName === 'pull_request') { + const comments = await github.rest.issues.listComments({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + }); + const existingComment = comments.data.find(comment => comment.body.includes('Visual regression testing results')); + if (existingComment) { + await github.rest.issues.updateComment({ + comment_id: existingComment.id, + owner: context.repo.owner, + repo: context.repo.repo, + body + }); + return; + } else { + const { data } = await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body + }); + } + } diff --git a/playwright.config.ts b/playwright.config.ts index ca1f0088..d1ba7993 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -13,6 +13,10 @@ const config: PlaywrightTestConfig = { ], testDir: './playwright', snapshotPathTemplate: '{testDir}/reference-images/{arg}{ext}', + reporter: [ + ['line'], + ['json', { outputFile: 'test-results/visual-regression-results.json' }], + ], }; export default config; diff --git a/playwright/fabric-custom.test.ts b/playwright/fabric-custom.test.ts index 04251ed1..9ec78007 100644 --- a/playwright/fabric-custom.test.ts +++ b/playwright/fabric-custom.test.ts @@ -3,7 +3,7 @@ import { localBaseUrl, referenceBaseUrl, templatePreviewWidths } from './utils'; const viewport = { width: 1600, height: 1000 }; -test.describe('Fabric Custom visual regression testing', () => { +test.describe('Fabric Custom', () => { test('Get reference screenshots', async ({ page }) => { await page.setViewportSize(viewport); @@ -45,7 +45,7 @@ test.describe('Fabric Custom visual regression testing', () => { // compare screenshot to reference await expect(testTemplateLocator).toHaveScreenshot( `Fabric-custom-${width.replace('%', '')}.png`, - { animations: 'disabled', maxDiffPixelRatio: 0.004 }, + { animations: 'disabled', maxDiffPixelRatio: 0.006 }, ); } }); diff --git a/playwright/manual-multiple.test.ts b/playwright/manual-multiple.test.ts index 773d8462..bd6827dc 100644 --- a/playwright/manual-multiple.test.ts +++ b/playwright/manual-multiple.test.ts @@ -3,7 +3,7 @@ import { localBaseUrl, referenceBaseUrl, templatePreviewWidths } from './utils'; const viewport = { width: 1600, height: 1000 }; -test.describe('Manual Multiple visual regression testing', () => { +test.describe('Manual Multiple', () => { test('Get reference screenshots', async ({ page }) => { await page.setViewportSize(viewport); @@ -44,7 +44,7 @@ test.describe('Manual Multiple visual regression testing', () => { // compare screenshot to reference await expect(testTemplateLocator).toHaveScreenshot( `Manual-multiple-${width.replace('%', '')}.png`, - { maxDiffPixelRatio: 0.004 }, + { maxDiffPixelRatio: 0.006 }, ); } }); diff --git a/playwright/manual-single.test.ts b/playwright/manual-single.test.ts index 21e60712..9b3595c0 100644 --- a/playwright/manual-single.test.ts +++ b/playwright/manual-single.test.ts @@ -3,7 +3,7 @@ import { localBaseUrl, referenceBaseUrl, templatePreviewWidths } from './utils'; const viewport = { width: 1600, height: 1000 }; -test.describe('Manual Single visual regression testing', () => { +test.describe('Manual Single', () => { test('Get reference screenshots', async ({ page }) => { await page.setViewportSize(viewport); @@ -44,7 +44,7 @@ test.describe('Manual Single visual regression testing', () => { // compare screenshot to reference await expect(testTemplateLocator).toHaveScreenshot( `Manual-single-${width.replace('%', '')}.png`, - { maxDiffPixelRatio: 0.004 }, + { maxDiffPixelRatio: 0.006 }, ); } }); diff --git a/scripts/analyse-visual-results.js b/scripts/analyse-visual-results.js new file mode 100644 index 00000000..8d8c946c --- /dev/null +++ b/scripts/analyse-visual-results.js @@ -0,0 +1,29 @@ +import { readFileSync, writeFileSync } from 'fs'; +import path, { resolve } from 'path'; +import { fileURLToPath } from 'url'; + +// replicate __dirname functionality as this is an es module +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const filePath = resolve( + __dirname, + `../test-results/visual-regression-results.json`, +); +const fileContent = JSON.parse(readFileSync(filePath, 'utf-8')); + +const tests = fileContent.suites; + +let resultsTable = `| Template | Visual test status | +| ------------- | ------------- |`; + +for (const test of tests) { + const testStatus = Boolean(test.suites[0].specs[1].ok); + const testTitle = test.suites[0].title; + resultsTable += `\n| ${testTitle} | ${testStatus ? '✅' : '❌'} |`; +} + +writeFileSync( + resolve(__dirname, `../test-results/visual-regression-results-table.txt`), + resultsTable, +);