epic: integration tests (v5+)

.eslintrc.yml

@@ -25,6 +25,7 @@ parser: '@typescript-eslint/parser'
 parserOptions:
   project:
     - './tsconfig.json'
+    - './e2e/tsconfig.json'
     - './site/tsconfig.json'
     - './eslint/tsconfig.json'
   sourceType: module

.github/workflows/e2e.yml

@@ -0,0 +1,86 @@
+name: Automation Testing (Draft)
+
+on:
+  push:
+    branches: [ main, main-beta, epic/* ]
+  workflow_dispatch:
+    inputs:
+      updateSnapshots:
+        type: boolean
+        default: false
+        required: false
+        description: 'Update snapshots during the run on the current brunch'
+
+env:
+  node-version: 20.x
+  DIFF_REPORT_BRANCH: diff-report
+
+permissions:
+  contents: write
+  pages: none
+  deployments: none
+  pull-requests: write
+
+jobs:
+  e2e-tests:
+    name: Automation Testing
+    runs-on: ubuntu-latest
+    if: ${{ !inputs.updateSnapshots }}
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node v${{ env.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          cache: 'npm'
+          node-version: ${{ env.node-version }}
+      - name: Install NPM Dependencies
+        run: npm ci
+      - name: Run Tests
+        run: npm run test:e2e
+      - name: Collect Report & Upload artifact
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: automation-diff-report
+          path: e2e/.diff
+      - name: Deploy latest issued report to branch
+        if: failure()
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: e2e/.diff
+          publish_branch: ${{ env.DIFF_REPORT_BRANCH }}
+      - name: Render Report
+        if: always()
+        shell: bash
+        run: |
+          cat e2e/.diff/index.md >> $GITHUB_STEP_SUMMARY
+
+  e2e-tests-update-snapshots:
+    name: Update Automation Testing Snapshots
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'workflow_dispatch' && inputs.updateSnapshots }}
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node v${{ env.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          cache: 'npm'
+          node-version: ${{ env.node-version }}
+      - name: Install NPM Dependencies
+        run: npm ci
+      - name: Update Test Snapshots
+        run: npm run test:e2e:update
+      - name: Commit & Push Snapshot Changes
+        uses: actions/github-script@v5
+        with:
+          script: |
+            const git = require('simple-git')();
+            const currentBranch = '${{ github.ref }}'.split('/').pop();
+            await git.addConfig('user.email', '${{ github.actor }}@users.noreply.github.com');
+            await git.addConfig('user.name', '${{ github.actor }}');
+            await git.add('*.png');
+            await git.commit('test(e2e): update snapshots (via GitHub Actions))');
+            await git.push('origin', currentBranch);

.gitignore

@@ -28,3 +28,4 @@ npm-debug.log
 # Jest & Sonar
 /.report
 /.scannerwork
+/e2e/.diff

e2e/.eslintignore

@@ -0,0 +1,6 @@
+# Common configuration
+jest.config.js
+
+# Common directories
+transformer
+reporters

e2e/README.md

@@ -0,0 +1,106 @@
+# ESL Internal Automation Testing 
+
+## Introduction
+This is an internal package for visual automation testing the ESL. 
+It is designed to be used by the ESL team to test the ESL library by checking the visual consistency of `@exadel/esl-website` submodule.
+
+## Installation
+
+There are no extra pre-conditions to develop new visual tests locally, 
+everything to start will be already installed with root package installation.
+
+**⚠️ Except for one critical thing: all final snapshots should be created on Linux based OS (the latest version of Ubuntu is recommended) ⚠️**
+
+Visual testing depends on font rendering so snapshots created on different OS may be different.
+The main (remote) environment for visual testing runs via GitHub Workflows on the latest version of Ubuntu.
+
+### Creating snapshots and running existing tests on Windows
+
+To create snapshots on Windows, you need to use WSL (Windows Subsystem for Linux) and follow the next steps:
+
+1. Install WSL (https://docs.microsoft.com/en-us/windows/wsl/install)
+2. Install Ubuntu from Microsoft Store (https://www.microsoft.com/en-us/p/ubuntu/9nblggh4msv6)
+3. Run Ubuntu and install Node.js and npm with subsystem CLI. 
+   To simplify the process, you can use `nvm` (Node Version Manager). 
+   Use the following commands to install `nvm` and the latest Node.js version:
+   ```bash
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash
+    nvm install node@20
+    nvm use node@20
+   ```
+4. Install Chromium browser (https://learn.microsoft.com/en-us/windows/wsl/tutorials/gui-apps#install-google-chrome-for-linux):
+   ```bash
+   sudo apt-get update
+   sudo apt-get install -y chromium-browser
+   ```
+5. Install additional dependencies to run puppeteer:
+   ```bash
+   sudo apt install libgtk-3-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2
+   ```
+   Note: you might need to restart your subsystem after steps 4 and 5.
+6. Open the project folder via WSL terminal and run `npm install` to ensure all dependencies are installed.
+7. Use `e2e` package scripts to create snapshots and run tests:
+   - `npm run test:e2e` (in the root package) or `npm run run` (in the sub-package) to run all tests (missing snapshots will be created automatically)
+   - `npm run test:e2e:update` (in the root package) or `npm run run:update` (in sub-package) to update all snapshots
+
+### Creating snapshots and running existing tests on Mac
+
+TODO: add instructions for Mac OS
+
+### Creating snapshots and running existing tests on Linux
+
+To create snapshots on Linux, you can use any Linux distribution, but the latest version of Ubuntu is recommended.
+
+1. Install the ESL root package and all dependencies according to the root package README.md.
+2. Ensure you have Chromium browser installed:
+   ```bash
+   sudo apt-get update
+   sudo apt-get install -y chromium-browser
+   ```
+3. Use `e2e` package scripts to create snapshots and run tests:
+   - `npm run test:e2e` (in the root package) or `npm run run` (in sub-package) to run all tests (missing snapshots will be created automatically)
+   - `npm run test:e2e:update` (in the root package) or `npm run run:update` (in sub-package) to update all snapshots
+
+
+### Full list of available commands
+
+All mentioned commands are available in the root package and in the `e2e` package only. 
+Use explicit workspace name to run following commands from the root package.  
+
+- `npm run run` - run all visual tests (run server automatically, create missing snapshots)
+- `npm run run:update` - update all snapshots (run server automatically)
+- `npm run run:server` - shortcut to run server for visual tests (uses `esl-website` package)
+- `npm run run:update:only` - update all snapshots, does not run server (ensure you run server manually)
+- `npm run run:server:only` - run server for visual tests, does not run tests
+
+Note: default server port for visual tests is `3007`.
+
+## Updating snapshots using GitHub project workflow
+
+The Automated tests workflow allows you to trigger it manually, it will update all snapshots and commit changes on the branch you specify.
+
+Make sure you have permission to run workflows in the ESL repository or ask the ESL Maintainers team to run the proper update for you.
+
+To trigger the workflow manually, follow these steps:
+
+1. Open the ESL repository in your browser.
+2. Go to the "Actions" tab.
+3. Find the "Automated tests" workflow and click on it.
+4. Click on the "Run workflow" button.
+5. Specify the branch you want to update
+6. Make sure you check the "Update snapshots" checkbox
+7. Click on the "Run workflow" button. 
+
+The update-commit will be created and pushed to the specified branch under your GitHub account.
+
+## Check Automation Test Results
+
+Automated tests run inside GitHub Workflows environment automatically on every push or pull request to the `main`, `main-beta` and `epic:*` branches.
+
+To check the results of the latest run, follow these steps:
+
+1. Open the ESL repository in your browser.
+2. Go to the "Actions" tab or find the "Automated tests" workflow in the "Pull requests" checks.
+3. Find the latest run and click on it.
+
+You can find the test results in the Summary section or you can download full results as workflow run artifacts.

e2e/jest-puppeteer.config.js

@@ -0,0 +1,32 @@
+/* eslint-disable no-undef */
+
+if (!process.argv.includes('--no-autorun') && !process.env.PORT) {
+  process.env.PORT = '3007';
+}
+
+/** @type {import('jest-environment-puppeteer').JestPuppeteerConfig} */
+module.exports = {
+  launch: {
+    headless: 'new',
+    product: 'chrome',
+    args: [
+      '--no-sandbox',
+      '--disable-gpu',
+      '--disable-setuid-sandbox',
+      '--disable-background-timer-throttling',
+      '--disable-backgrounding-occluded-windows',
+      '--disable-renderer-backgrounding',
+      '--font-render-hinting=none'
+    ]
+  },
+  server: {
+    command: 'npm run run:server',
+    port: process.env.PORT,
+    launchTimeout: 120000,
+    debug: true
+  }
+};
+
+if (process.argv.includes('--no-autorun')) {
+  delete module.exports.server;
+}

e2e/jest.config.js

@@ -0,0 +1,23 @@
+// Test env in dev mode produces a big amount of stdin/out listeners, so limit increased
+require('events').EventEmitter.defaultMaxListeners  = 50;
+
+module.exports = {
+  preset: "jest-puppeteer",
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+    '^.+\\.feature$': './transformer/gherkin.js'
+  },
+  roots: ['./tests/'],
+  testRegex: ['(.+)\\.(spec|test)\\.ts$', '(.+).feature'],
+  moduleFileExtensions: ['ts', 'js', 'feature'],
+  setupFilesAfterEnv: ['./setup/image.ts', './setup/scenarios.ts'],
+  reporters: [
+    ['./reporters/reporter.js', {
+      diffDir: './.diff',
+      outputPath: './.diff/README.md',
+      outputPublishPath: './.diff/index.md'
+    }],
+    ['github-actions', {silent: false}],
+    'default'
+  ]
+};

e2e/package.json

@@ -0,0 +1,25 @@
+{
+  "private": true,
+  "name": "@exadel/esl-snapshot-tests",
+  "version": "4.14.1",
+  "description": "Integration snapshot tests for ESL",
+  "homepage": "https://esl-ui.com/",
+  "license": "MIT",
+  "scripts": {
+    "run": "jest",
+    "run:update": "jest -u",
+    "run:update:only": "jest -u --no-autorun",
+    "run:tests:only": "jest --no-autorun",
+    "run:server": "cd .. && cross-env-shell PORT=3007 npm run start:test --workspace=site",
+    "test": "eslint"
+  },
+  "dependencies": {
+    "@types/jest-environment-puppeteer": "^5.0.6",
+    "@types/jest-image-snapshot": "^6.4.0",
+    "@types/puppeteer": "^7.0.4",
+    "jest-image-snapshot": "^6.4.0",
+    "jest-puppeteer": "^9.0.2",
+    "puppeteer": "^22.0.0",
+    "stucumber": "^0.19.0"
+  }
+}

e2e/reporters/printers.js

@@ -0,0 +1,65 @@
+const printSummary = (stats) => {
+  let text = '\n';
+
+  text += '| :clock10: Start time | :hourglass: Duration |\n';
+  text += '| --- | ---: |\n';
+  text += `|${stats.startTimeStr}|${stats.totalTimeStr}|\n`;
+  text += '\n';
+
+  text += '| | :white_check_mark: Passed | :x: Failed | :construction: Todo | :white_circle: Total |\n';
+  text += '| --- | ---: | ---: | ---:| ---: |\n';
+  text += `|Test Suites|${stats.numPassedTestSuites}|${stats.numFailedTestSuites}|-|${stats.numTotalTestSuites}|\n`;
+  text += `|Tests|${stats.numPassedTests}|${stats.numFailedTests}|${stats.numTodoTests}|${stats.numTotalTests}|\n`;
+  text += '\n';
+
+  return text;
+};
+
+const resolveURL = (basePath, snapshot) => {
+  if (!basePath) return snapshot;
+  let path = basePath + (basePath.endsWith('/') ? '' : '/') + snapshot;
+  if (basePath.includes('github')) path += '?raw=true';
+  return path.replace(/\\/g, '/');
+};
+
+function printFiles(fileStat, basePath) {
+  let text = '';
+  for (const file of fileStat) {
+    text += `### ${file.filepath}\n`;
+    text += `<table>\n`;
+    text += '<tr><th>Test</th><th>Status</th><th>Time</th></tr>\n';
+    for (const test of file.tests) {
+      const statusTest = test.status === 'passed' ? ':white_check_mark:' : ':x:';
+      const timeStr = test.time < 1000 ? `${test.time}ms` : `${test.time / 1000}s`;
+
+      text += `<tr><td>${test.name}:${test.title}</td><td>${statusTest}</td><td>${timeStr}</td></tr>\n`;
+
+      if (test.status !== 'passed' && test.hasSnapshot) {
+        text += `<tr><td colspan="3"><img src="${resolveURL(basePath, test.snapshot)}" alt="Test Diff ${test.snapshot}"/></td></tr>`;
+      }
+      if (test.status !== 'passed' && !test.hasSnapshot) {
+        text += `<tr><td colspan="3">\n`;
+        text += '```text\n';
+        text += test.messages.join('\n');
+        text += '\n```\n';
+        text += `</td></tr>\n`;
+      }
+    }
+    text += `</table>`;
+    text += '\n\n';
+  }
+  return text;
+}
+
+function print({stats, files, basePath}) {
+  return `# Test Results
+  ## Summary
+  ${printSummary(stats)}
+
+  ---
+  ## Tests Details
+  ${printFiles(files, basePath)}
+`;
+}
+
+exports.print = print;