feat(add-automation): added Azure pipeline config, shifted screenshot…

… function to puppeteer, accepted most German cookies

.achecker.yml

@@ -71,5 +71,4 @@ baselineFolder: .accessibility-checker/test/baselines
 cacheFolder: .accessibility-checker/tmp
 
 headless: "shell"
-maxTabs: 7
 perfMetrics: false

.mocharc.yml

@@ -1,3 +1,3 @@
 require: tsx
 # 1 hour for report generation
-timeout: 3600000
+timeout: 3600000

.prettierignore

@@ -1,6 +1,9 @@
 # folders
 public
 node_modules
+power-bi-desktop
+automation
+.vscode
 
 # files
 *.html

automation/azure/dev-ops/azure-pipelines.yml

@@ -5,13 +5,13 @@ resources:
     - repository: senacor-main
       type: github
       endpoint: senacor
-      ref: test/add-2-pipeline
+      ref: feature/add-automation
       name: senacor/digital-access-reporting-tool
     - repository: senacor-screenshots
       type: github
-      endpoint: pan-senacor
+      endpoint: senacor
       ref: feature/screenshots
-      name: pan-senacor/digital-access-reporting-tool
+      name: senacor/digital-access-reporting-tool
 
 trigger:
   branches:

backend/gen/generateReport.spec.ts

@@ -1,32 +1,28 @@
-import fs from "node:fs"
 import { assert, expect } from "chai"
 import { before, after, describe, it } from "mocha"
 
 import getValidUrlOrNull from "../utils/getValidUrlOrNull"
 import takeScreenshot from "../utils/takeScreenshot"
 import generateMultiPageReport from "../utils/report-generation/generateMultiPageReport"
 
-const SCREENSHOT_BASE_URL = "https://raw.githubusercontent.com/pan-senacor/digital-access-reporting-tool/refs/heads/feature/screenshots"
 let url: URL | null
 let logoUrl: URL | null
 
 before(() => {
-    assert.isNotEmpty(process.env.URL, "URL is required")
-    assert.isNotEmpty(process.env.LOGO_URL, "Logo URL is required")
-    url = getValidUrlOrNull(process.env.URL || "")
-    logoUrl = getValidUrlOrNull(process.env.LOGO_URL || "")
+  assert.isNotEmpty(process.env.URL, "URL is required")
+  assert.isNotEmpty(process.env.LOGO_URL, "Logo URL is required")
+  url = getValidUrlOrNull(process.env.URL || "")
+  logoUrl = getValidUrlOrNull(process.env.LOGO_URL || "")
 })
 
-after(() => {
-})
+after(() => {})
 
 describe("Generate equal access multi-page report", () => {
-    it("WCAG", async () => {
-        const screenshotPath = await takeScreenshot(url!)
-        assert.isNotEmpty(screenshotPath, `Failed to create screenshot for URL: ${url}`)
-        const { multiPageReport } = await generateMultiPageReport(url!, logoUrl!, `${SCREENSHOT_BASE_URL}/${screenshotPath}`)
-        assert.isNotEmpty(multiPageReport)
-        expect(multiPageReport.pageCount).to.be.above(0)
-        fs.writeFileSync("./.accessibility-checker/accessibility-report.json", JSON.stringify({ report: multiPageReport }))
-    })
+  it("WCAG", async () => {
+    const screenshotUrl = await takeScreenshot(url!)
+    assert.isNotEmpty(screenshotUrl, `Failed to create screenshot for URL: ${url}`)
+    const { multiPageReport } = await generateMultiPageReport(url!, logoUrl!, screenshotUrl)
+    assert.isNotEmpty(multiPageReport)
+    expect(multiPageReport.pageCount).to.be.above(0)
+  })
 })

backend/route-handlers/createAccessibilityReportHandler.ts

@@ -47,12 +47,12 @@ export default async function createAccessibilityReportHandler(
     return res.status(400).send({ data: null, formErrors: [formError], serverError })
   }
 
-  const screenshotPath = await takeScreenshot(url)
-  if (!screenshotPath) {
+  const screenshotUrl = await takeScreenshot(url)
+  if (!screenshotUrl) {
     serverError = { message: "Failed to create screenshot for URL " + url.href }
   }
 
-  const { multiPageReport } = await generateMultiPageReport(url, logoUrl, screenshotPath)
+  const { multiPageReport } = await generateMultiPageReport(url, logoUrl, screenshotUrl)
   const data: Data = { report: multiPageReport }
 
   return res.send({ data, formErrors: null, serverError })

backend/utils/report-generation/generateMultiPageReport.ts

@@ -1,12 +1,17 @@
+import os from "os"
+import fs from "node:fs"
 import * as accessibilityChecker from "accessibility-checker"
 import { crawlDomainUrlsRecursively } from "./crawlDomainUrlsRecursively"
 import { createMultiPageReport } from "./report-aggregation/createMultiPageReport"
 import { AccessibilityCheckerReport } from "./types"
 
+// limit number of browser tabs to number of CPUs
+const cpuCount = os.cpus().length
+
 export default async function generateMultiPageReport(
   url: URL,
   logoUrl: URL,
-  screenshotPath: string | null,
+  screenshotUrl: string | null,
 ) {
   // List of all accessibility checker reports that are generated for each URL
   const accessibilityCheckerReports: AccessibilityCheckerReport[] = []
@@ -42,10 +47,18 @@ export default async function generateMultiPageReport(
   const multiPageReport = createMultiPageReport(
     url,
     logoUrl,
-    screenshotPath,
+    screenshotUrl,
     accessibilityCheckerReports,
   )
-
+  await fs.writeFile(
+    "./.accessibility-checker/accessibility-report.json",
+    JSON.stringify({ report: multiPageReport }),
+      (error) => {
+        if (error) {
+          console.log(`🔥 ${error}`)
+        }
+      },
+  )
   console.log("🚢 Shipping aggregated report!")
   return {
     multiPageReport,
@@ -81,7 +94,7 @@ class ReportCreationSet extends Set<string> {
    * @param reportCallback Callback function that is called with the generated report
    * @param parallelCreationsLimit Maximum number of parallel report creations that are being handled at any time
    */
-  constructor({ url, reportCallback, parallelCreationsLimit = 7 }: ReportCreationSetArgs) {
+  constructor({ url, reportCallback, parallelCreationsLimit = cpuCount }: ReportCreationSetArgs) {
     super()
     this.#reportCallback = reportCallback
     this.#parallelCreationsLimit = parallelCreationsLimit
@@ -130,6 +143,7 @@ class ReportCreationSet extends Set<string> {
         this.#reportCallback(null)
       } finally {
         --this.#runningReportCreationsCount
+        console.log(`✔ Completed reporting for ${url}!`)
       }
     }
 

backend/utils/report-generation/report-aggregation/createMultiPageReport.ts

@@ -14,13 +14,13 @@ import { LevelIssueCount, MultiPageReport, SinglePageReport, TreePageReport } fr
 export const createMultiPageReport = (
   url: URL,
   logoUrl: URL,
-  screenshotPath: string | null,
+  screenshotUrl: string | null,
   accessibilityCheckerReports: AccessibilityCheckerReport[],
 ) => {
   const multiPageReport: MultiPageReport = {
     url: url.origin,
     logoUrl: logoUrl.href,
-    screenshotPath,
+    screenshotUrl,
     pageCount: 0,
     pageReports: [],
     categoryIssueCounts: [],
@@ -113,7 +113,6 @@ function aggregateLevelIssueCounts(
 }
 
 function createTreePageReport(pages: SinglePageReport[]) {
-
   const base: TreePageReport = emptyTreePageReport("/")
 
   for (const page of pages) {
@@ -127,16 +126,16 @@ function createTreePageReport(pages: SinglePageReport[]) {
 
     path?.forEach((_e, i) => {
       // prepend the protocl (e.g. https: and '/' to the current path)
-      const currPath = protocol + "/" + path?.slice(0, i+1).join("")
-      const child = curr.children.find(e => e.url === currPath)
+      const currPath = protocol + "/" + path?.slice(0, i + 1).join("")
+      const child = curr.children.find((e) => e.url === currPath)
 
       if (child) {
         curr = child
       } else {
         const treeReport = emptyTreePageReport(currPath)
         curr.children.push(treeReport)
-        curr = curr.children[curr.children.length-1]
-        curr.children = curr.children.sort((a,b) => a.url.localeCompare(b.url))
+        curr = curr.children[curr.children.length - 1]
+        curr.children = curr.children.sort((a, b) => a.url.localeCompare(b.url))
       }
       if (fullPath === currPath) {
         curr.page = page
@@ -161,7 +160,7 @@ function emptyTreePageReport(url: string): TreePageReport {
       levelIssueCounts: [],
       elementCount: 0,
       elementWithViolationCount: 0,
-      elementsWithNoViolationsPercentage: 0
+      elementsWithNoViolationsPercentage: 0,
     },
   }
 }
@@ -202,19 +201,14 @@ function calculateStats(treePage: TreePageReport) {
     // Here we aggregate the simpler values of the multi page report
     treePage.summary.totalIssueCount += child.summary.totalIssueCount
 
-    aggregateLevelIssueCounts(
-      treePage.summary.levelIssueCounts,
-      child.summary.levelIssueCounts,
-    )
+    aggregateLevelIssueCounts(treePage.summary.levelIssueCounts, child.summary.levelIssueCounts)
 
     treePage.summary.elementCount += child.summary.elementCount
-    treePage.summary.elementWithViolationCount +=
-      child.summary.elementWithViolationCount
+    treePage.summary.elementWithViolationCount += child.summary.elementWithViolationCount
   }
 
-  treePage.summary.elementsWithNoViolationsPercentage =
-    calculateElementsWithNoViolationsPercentage(
-      treePage.summary.elementCount,
-      treePage.summary.elementWithViolationCount,
-    )
-}
+  treePage.summary.elementsWithNoViolationsPercentage = calculateElementsWithNoViolationsPercentage(
+    treePage.summary.elementCount,
+    treePage.summary.elementWithViolationCount,
+  )
+}

backend/utils/report-generation/report-aggregation/types.ts

@@ -119,7 +119,7 @@ export type SinglePageReport = {
 // }
 export type MultiPageReport = SinglePageReport & {
   logoUrl: string
-  screenshotPath: string | null
+  screenshotUrl: string | null
   pageCount: number
   pageReports: SinglePageReport[]
   treePageReport: TreePageReport | null
@@ -128,4 +128,4 @@ export type MultiPageReport = SinglePageReport & {
 export type TreePageReport = SinglePageReport & {
   page: SinglePageReport | null
   children: TreePageReport[]
-}
+}

backend/utils/takeScreenshot.ts

@@ -1,32 +1,37 @@
-import { chromium } from "playwright-extra"
+import puppeteer from "puppeteer-extra"
+import AdblockerPlugin from "puppeteer-extra-plugin-adblocker"
 import StealthPlugin from "puppeteer-extra-plugin-stealth"
+import * as accessibilityChecker from "accessibility-checker"
 import logger from "./logger"
 
+const SCREENSHOT_BASE_URL =
+  "https://raw.githubusercontent.com/senacor/digital-access-reporting-tool/refs/heads/feature/screenshots"
 const screenshotPath = "screenshots/"
 const screenshotType = "png"
 
 const createScreenshotPath = (url: URL) => {
   return screenshotPath + url.hostname + "." + screenshotType
 }
 
-// register the Stealth plugin in Playwright
-chromium.use(StealthPlugin())
-
 export default async function takeScreenshot(url: URL) {
+  // register the Stealth and Ad-Blocker plugins with Puppeteer
+  puppeteer.use(AdblockerPlugin()).use(StealthPlugin())
   try {
-    const browser = await chromium.launch()
     const screenshotPath = createScreenshotPath(url)
+    const acConfig = await accessibilityChecker.getConfigUnsupported()
+    const browser = await puppeteer.launch({
+      headless: acConfig.headless,
+      defaultViewport: { width: 1920, height: 1080 },
+    })
     const page = await browser.newPage()
-    page.on('dialog', async dialog => { await dialog.dismiss() })
-
-    await page.setViewportSize({ width: 1920, height: 1080 })
-    await page.goto(url.href)
+    await page.goto(url.href, { waitUntil: 'domcontentloaded' })
+    await acceptCookies(page)
     await page.screenshot({ path: screenshotPath, type: screenshotType })
     await browser.close()
 
     console.log("📸 Screenshot taken")
 
-    return screenshotPath
+    return `${SCREENSHOT_BASE_URL}/${screenshotPath}`
   } catch (error) {
     logger.print("error", "Failed to take screenshot")
 
@@ -42,3 +47,29 @@ export default async function takeScreenshot(url: URL) {
     return null
   }
 }
+
+async function acceptCookies(page: any) {
+  try {
+      const element = await page
+        .locator(
+          `a::-p-text(uswählen),
+          a::-p-text(kzeptieren),
+          :scope >>> a::-p-text(kzeptieren),
+          :scope >>> a::-p-text(ustimmen),
+          a::-p-text(eht klar),
+          :scope >>> button::-p-text(ustimmen),
+          :scope >>> button::-p-text(inverstanden),
+          :scope >>> button::-p-text(kzeptieren),
+          :scope >>> button::-p-text(ccept)
+        `)
+        .setTimeout(3000)
+      await element.click()
+      await page.waitForNetworkIdle({ timeout: 3000, idleTime: 1000 })
+    // Cookies have been accepted successfully
+    return true
+  } catch (error) {
+    // An error occurred while accepting cookies
+    console.log(`🔥 Error handling the cookie popup: ${error}`)
+    return false
+  }
+}