feat: #52 visual regression tests poc

.github/workflows/playwright.yml

@@ -0,0 +1,71 @@
+name: Visual Regression Tests
+on:
+  pull_request:
+    branches: [main]
+
+  # Allow updating snapshots during manual runs
+  workflow_call:
+    inputs:
+      update-snapshots:
+        description: 'Update snapshots?'
+        type: boolean
+
+  # Allow updating snapshots during automatic runs
+  workflow_dispatch:
+    inputs:
+      update-snapshots:
+        description: 'Update snapshots?'
+        type: boolean
+
+env:
+  NPM_TOKEN: ${{secrets.NPM_TOKEN}}
+  CI: true
+
+jobs:
+  test:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: '18.x'
+
+      - name: Install dependencies
+        run: yarn
+
+      - name: Install Playwright Browsers
+        run: yarn playwright install --with-deps
+
+      - name: Build Storybook
+        run: |
+          yarn build:storybook
+
+      - name: Set up cache
+        id: cache
+        uses: actions/cache@v4
+        with:
+          key: cache/${{github.repository}}/${{github.ref}}
+          restore-keys: cache/${{github.repository}}/refs/heads/main
+          path: .visual-test/**
+
+      - name: Initialize snapshots
+        if: ${{steps.cache.outputs.cache-hit != 'true' || inputs.update-snapshots == 'true'}}
+        run: npx playwright test --update-snapshots --reporter html
+
+      - name: Run Playwright tests
+        continue-on-error: true
+        run: yarn playwright test
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-report
+          path: .visual-test/spec/results/
+          retention-days: 30
+          overwrite: true

.gitignore

@@ -100,3 +100,8 @@ cdk.context.json
 packages/cognito-custom-mail-lambda/**/*.html
 
 .tmp
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/
+.visual-test/**

.storybook/preview.ts

@@ -3,7 +3,6 @@ import '../src/styles/globals'
 
 const preview: Preview = {
   parameters: {
-    actions: { argTypesRegex: '^on[A-Z].*' },
     viewMode: 'docs',
     options: {
       storySort: {

README.md

@@ -7,6 +7,7 @@ A React UI toolkit, with cross platform support for building web applications in
 [![Test PR](https://github.com/reapit/elements/actions/workflows/test-pr.yml/badge.svg)](https://github.com/reapit/elements/actions/workflows/test-pr.yml)
 [![Release Dev](https://github.com/reapit/elements/actions/workflows/release-dev.yml/badge.svg)](https://github.com/reapit/elements/actions/workflows/release-dev.yml)
 [![Release Prod](https://github.com/reapit/elements/actions/workflows/release-prod.yml/badge.svg)](https://github.com/reapit/elements/actions/workflows/release-prod.yml)
+[![Visual Regression Tests](https://github.com/reapit/elements/actions/workflows/playwright.yml/badge.svg)](https://github.com/reapit/elements/actions/workflows/playwright.yml)
 
 ## Documentation
 

jest.config.js

@@ -15,7 +15,7 @@ module.exports = {
     'index.ts',
     'src/styles',
   ],
-  modulePathIgnorePatterns: ['<rootDir>[/\\\\](node_modules|public|dist)[/\\\\]'],
+  modulePathIgnorePatterns: ['<rootDir>[/\\\\](node_modules|public|dist|visual-tests)[/\\\\]'],
   moduleNameMapper: {
     '^.+\\.svg$': join(__dirname, './scripts/jest/svg-transform.js'),
     '@/(.*)': '<rootDir>/src/$1',

package.json

@@ -37,7 +37,8 @@
     "publish": "yarn npm publish --access public",
     "deploy": "elements-deploy deploy",
     "start": "storybook dev -p 6006",
-    "test": "cross-env TZ=UTC jest --watch --colors --silent"
+    "test": "cross-env TZ=UTC jest --watch --colors --silent",
+    "visual-test": "yarn build:storybook && yarn playwright test"
   },
   "dependencies": {
     "@linaria/core": "^5.0.2",
@@ -54,28 +55,29 @@
     "@linaria/vite": "^5.0.4",
     "@linaria/webpack-loader": "^5.0.4",
     "@mdx-js/react": "^1.6.22",
+    "@playwright/test": "^1.42.1",
     "@rollup/plugin-babel": "^6.0.4",
     "@rollup/plugin-commonjs": "^24.1.0",
     "@rollup/plugin-node-resolve": "^15.2.3",
     "@rollup/plugin-terser": "^0.4.4",
     "@rollup/plugin-typescript": "^11.1.5",
     "@rollup/plugin-url": "^8.0.2",
-    "@storybook/addon-a11y": "^8.0.0-rc.0",
-    "@storybook/addon-actions": "^8.0.0-rc.0",
-    "@storybook/addon-docs": "^8.0.0-rc.0",
-    "@storybook/addon-essentials": "^8.0.0-rc.0",
-    "@storybook/addon-links": "^8.0.0-rc.0",
-    "@storybook/addon-storysource": "^8.0.0-rc.0",
-    "@storybook/blocks": "^8.0.0-rc.0",
-    "@storybook/cli": "^8.0.0-rc.0",
-    "@storybook/csf": "0.1.2-next.0",
-    "@storybook/icons": "^1.2.5",
-    "@storybook/manager-api": "^8.0.0-rc.0",
-    "@storybook/preview-api": "^8.0.0-rc.0",
-    "@storybook/react": "^8.0.0-rc.0",
-    "@storybook/react-dom-shim": "^8.0.0-rc.0",
-    "@storybook/react-vite": "^8.0.0-rc.0",
-    "@storybook/theming": "^8.0.0-rc.0",
+    "@storybook/addon-a11y": "^8.0.4",
+    "@storybook/addon-actions": "^8.0.4",
+    "@storybook/addon-docs": "^8.0.4",
+    "@storybook/addon-essentials": "^8.0.4",
+    "@storybook/addon-links": "^8.0.4",
+    "@storybook/addon-storysource": "^8.0.4",
+    "@storybook/blocks": "^8.0.4",
+    "@storybook/cli": "^8.0.4",
+    "@storybook/csf": "^0.1.3",
+    "@storybook/icons": "^1.2.9",
+    "@storybook/manager-api": "^8.0.4",
+    "@storybook/preview-api": "^8.0.4",
+    "@storybook/react": "^8.0.4",
+    "@storybook/react-dom-shim": "^8.0.4",
+    "@storybook/react-vite": "^8.0.4",
+    "@storybook/theming": "^8.0.4",
     "@svgr/rollup": "^8.1.0",
     "@svgr/webpack": "^8.1.0",
     "@testing-library/react": "^14.1.2",
@@ -105,7 +107,7 @@
     "eslint-plugin-prettier": "^5.1.3",
     "eslint-plugin-react": "^7.33.2",
     "eslint-plugin-react-hooks": "^4.6.0",
-    "eslint-plugin-storybook": "^0.6.15",
+    "eslint-plugin-storybook": "^0.8.0",
     "jest": "^29.7.0",
     "jest-coverage-badges": "^1.1.2",
     "jest-environment-jsdom": "^29.7.0",
@@ -121,7 +123,7 @@
     "rollup": "^3.29.4",
     "rollup-plugin-css-only": "^4.5.2",
     "snyk": "^1.1259.0",
-    "storybook": "^8.0.0-rc.0",
+    "storybook": "^8.0.4",
     "ts-jest": "^29.1.1",
     "ts-node": "^10.9.1",
     "tslib": "^2.6.2",

playwright.config.ts

@@ -0,0 +1,88 @@
+import { defineConfig, devices } from '@playwright/test'
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// require('dotenv').config();
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './visual-tests',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 3 : 1,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: [
+    [
+      'html',
+      {
+        outputFolder: '.visual-test/spec/results',
+        open: 'never',
+      },
+    ],
+    process.env.CI ? ['github'] : ['line'],
+  ],
+
+  outputDir: '.visual-test/spec/output',
+  snapshotPathTemplate: '.visual-test/spec/snaps/{projectName}/{testFilePath}/{arg}{ext}',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://127.0.0.1:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  // 2 min a bit high but want to ensure we doesn't timeout on CI
+  timeout: 120000,
+  /* Configure projects for major browsers */
+  // Just adding Chrome for now, we can look at browser matrix later
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+
+    // {
+    //   name: 'firefox',
+    //   use: { ...devices['Desktop Firefox'] },
+    // },
+
+    // {
+    //   name: 'webkit',
+    //   use: { ...devices['Desktop Safari'] },
+    // },
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    // },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  webServer: undefined,
+})

src/storybook/changelog.mdx

@@ -16,6 +16,11 @@ We will publish release version history and changes here. Where possible, we wil
 
 Beta versions should be relatively stable but subject to occssional breaking changes.
 
+### **4.1.3 - 27/03/24**
+
+- Updated Storybook to latest
+- Adds visual regression tests
+
 ### **4.1.2 - 07/03/24**
 
 - Added docs for layout stories missed at past release
@@ -27,7 +32,7 @@ Beta versions should be relatively stable but subject to occssional breaking cha
 
 ### **4.1.0 - 22/02/24**
 
-- Upgrade to Stoybook v8
+- Upgrade to Storybook v8
 - Upgraded prettier to latest
 - Migrate storybook components and stories to v8
 

visual-tests/flakey-test-list.ts

@@ -0,0 +1,27 @@
+/**
+ * Add test names here to exclude from test run. The below typically use placeholder images which vary at each load
+ * e.g. Avatar, hence removing - we should ensure the images are static in the story ideally and then can be removed
+ * from this list. Also, if the component uses a Portal, it doesn't render correctly in the storybook iframe.
+ */
+export const flakeyTestList = [
+  'avatar--with-src',
+  'avatar--with-image',
+  'card--card-with-avatar',
+  'card--card-with-image',
+  'card--card-focussed',
+  'card--card-complete-example',
+  'card--react-shorthand-avatar-body',
+  'card--react-shorthand-complete',
+  'layouts--complete-combined-example',
+  'layouts--complete-combined-example-max-width',
+  'pageheader--basic-usage',
+  'table--basic-usage',
+  'table--column-widths',
+  'table--basic-customisation-table-cells',
+  'table--react-shorthand-usage',
+  'drawer--basic-usage-closed',
+  'mobilecontrols--basic-usage',
+  'modal--basic-usage-closed',
+  'persistentnotification--full-react-example-fixed-position',
+  'portal--portal-usage',
+]

visual-tests/index.spec.ts

@@ -0,0 +1,24 @@
+import { test, expect } from '@playwright/test'
+import buildManifest from '../public/dist/index.json'
+import { navigate } from './utils'
+import { flakeyTestList } from './flakey-test-list'
+/** Credit to
+ * https://jamesiv.es/blog/frontend/testing/2024/03/11/visual-testing-storybook-with-playwright and
+ * https://www.browsercat.com/post/ultimate-guide-visual-testing-playwright#visual-testing-in-cicd
+ * for implementation
+ */
+
+const storyNames = Object.keys(buildManifest.entries).filter(
+  (entry) => buildManifest.entries[entry].tags.includes('story') && !flakeyTestList.includes(entry),
+)
+
+storyNames.forEach((story) => {
+  test(story, async ({ page }, meta) => {
+    await page.setViewportSize({ width: 1920, height: 1080 })
+    await navigate(page, 'https://elements.dev.paas.reapit.cloud', meta.title)
+
+    const screenshot = await page.screenshot()
+
+    expect(screenshot).toMatchSnapshot(`${meta.title}.png`, { maxDiffPixels: 10 })
+  })
+})

visual-tests/utils.ts

@@ -0,0 +1,22 @@
+import { Page } from '@playwright/test'
+
+export const getStoryUrl = (storybookUrl: string, id: string): string => {
+  const params = new URLSearchParams({
+    id,
+    viewMode: 'story',
+    nav: '0',
+  })
+
+  return `${storybookUrl}/iframe.html?${params.toString()}`
+}
+
+export const navigate = async (page: Page, storybookUrl: string, id: string): Promise<void> => {
+  try {
+    const url = getStoryUrl(storybookUrl, id)
+    await page.goto(url)
+    await page.waitForLoadState('networkidle')
+    await page.waitForSelector('#storybook-root')
+  } catch (error) {
+    console.error('Error navigating to storybook', error)
+  }
+}