[TASK] Add TYPO3 14 Support

.devbox/composer.json

@@ -15,32 +15,31 @@
         "issues": "https://github.com/tomasnorre/crawler/issues"
     },
     "require": {
-        "tnm/crawler-devbox-sitepackage": "^0.0.3",
-        "tomasnorre/crawler": "*@dev",
-        "typo3/cms-belog": "^13.4" ,
-        "typo3/cms-beuser": "^13.4",
-        "typo3/cms-felogin": "^13.4",
-        "typo3/cms-indexed-search": "^13.4",
-        "typo3/cms-info": "^13.4",
-        "typo3/cms-tstemplate": "^13.4",
-        "typo3/minimal": "^13.4",
-        "typo3/cms-install": "^13.4",
         "georgringer/news": "^14",
-        "typo3/cms-reports": "^13.4",
-        "symfony/dependency-injection": "7.3.6"
+        "tnm/crawler-devbox-sitepackage": "^0.0.5",
+        "tomasnorre/crawler": "*@dev",
+        "typo3/cms-belog": "^13.4 || ^14.0",
+        "typo3/cms-beuser": "^13.4 || ^14.0",
+        "typo3/cms-felogin": "^13.4 || ^14.0",
+        "typo3/cms-indexed-search": "^13.4 || ^14.0",
+        "typo3/cms-info": "^13.4 || ^14.0",
+        "typo3/cms-install": "^13.4 || ^14.0",
+        "typo3/cms-tstemplate": "^13.4 || ^14.0",
+        "typo3/minimal": "^13.4 || ^14.0"
     },
     "require-dev": {
         "roave/security-advisories": "dev-latest"
     },
     "conflict": {
+        "symfony/dependency-injection": "7.4.0",
         "tomasnorre/crawler": "<12.0.0"
     },
     "minimum-stability": "dev",
     "prefer-stable": true,
     "config": {
         "allow-plugins": {
-            "typo3/cms-composer-installers": true,
-            "typo3/class-alias-loader": true
+            "typo3/class-alias-loader": true,
+            "typo3/cms-composer-installers": true
         },
         "bin-dir": "bin",
         "vendor-dir": "vendor"

.github/workflows/Acceptance.yml

@@ -15,6 +15,7 @@ jobs:
       matrix:
         typo3:
           - ^13.4
+          - ^14.0
         php:
           - '8.2'
           - '8.3'
@@ -34,6 +35,15 @@ jobs:
         shell: bash
         run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF})" >> $GITHUB_ENV
 
+      - name: Set Playwright tag
+        shell: bash
+        run: |
+          if [[ "${{ matrix.typo3 }}" == "^13.4" ]]; then
+            echo "PW_GREP=@v13" >> $GITHUB_ENV
+          elif [[ "${{ matrix.typo3 }}" == "^14.0" ]]; then
+            echo "PW_GREP=@v14" >> $GITHUB_ENV
+          fi
+
       - uses: actions/checkout@v6
       - name: Patch composer.json for forked PR
         if: >
@@ -72,12 +82,12 @@ jobs:
       - name: Install dependencies
         run: cd Tests/Acceptance && npm ci
       - name: Install Playwright Browsers
-        run: cd Tests/Acceptance && npx playwright install --with-deps
+        run: cd Tests/Acceptance && npx playwright install chromium --with-deps
       - name: Run Playwright tests
-        run: cd Tests/Acceptance && npx playwright test
+        run: cd Tests/Acceptance && npx playwright test --grep "$PW_GREP"
       - uses: actions/upload-artifact@v6
         if: ${{ !cancelled() }}
         with:
-          name: playwright-report
-          path: playwright-report/
-          retention-days: 30
+          name: playwright-report-${{ matrix.typo3 }}-${{ matrix.php }}
+          path: Tests/Acceptance/playwright-report/
+          retention-days: 10

.github/workflows/StaticAnalysis.yaml

@@ -91,6 +91,5 @@ jobs:
                     coverage: none
             -   name: "Install dependencies"
                 uses: ramsey/composer-install@v3
-            -   run: composer cs-fix
             -   run: .Build/bin/psalm --no-cache --shepherd
 

.github/workflows/Tests.yml

@@ -17,6 +17,7 @@ jobs:
       matrix:
         typo3:
           - '^13.4'
+          - '^14.0'
         php:
           - '8.2'
           - '8.3'

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Added
 * PHP 8.5 Support [@tomasnorre](https://github.com/tomasnorre)
+* TYPO3 14 Support [@tomasnorre](https://github.com/tomasnorre)
 
 ### Changed
 * Updated symfony-components dependencies to ^7.2

Classes/Command/BuildQueueCommand.php

@@ -234,7 +234,7 @@ private function outputModeExec(OutputInterface $output, array $queueRows): void
             $result = $this->crawlerController->readUrlFromArray($queueRec);
 
             $resultContent = $result['content'] ?? '';
-            $requestResult = json_decode($resultContent, true);
+            $requestResult = json_decode((string) $resultContent, true);
 
             $progressBar->clear();
             if (is_array($requestResult)) {

Classes/Service/ConfigurationService.php

@@ -387,7 +387,7 @@ private function addValuesInRange(array $reg, array $paramArray, int|string $par
     private function expandPidList(array $treeCache, int $pid, int $depth, PageTreeView $tree, array $pidList): array
     {
         if (empty($treeCache[$pid][$depth])) {
-            $tree->reset();
+            //$tree->reset();
             $tree->getTree($pid, $depth);
             $treeCache[$pid][$depth] = $tree->tree;
         }

Configuration/Backend/Modules.php

@@ -21,12 +21,22 @@
 use AOE\Crawler\Controller\Backend\BackendModuleCrawlerProcessController;
 use AOE\Crawler\Controller\Backend\BackendModuleStartCrawlingController;
 
+if ((new \TYPO3\CMS\Core\Information\Typo3Version())->getMajorVersion() === 14) {
+    $parent = 'content';
+    $position = [
+        'after' => 'content_status',
+    ];
+} else {
+    $parent = 'web';
+    $position = [
+        'after' => 'web_info',
+    ];
+}
+
 return [
     'web_site_crawler' => [
-        'parent' => 'web',
-        'position' => [
-            'after' => 'web_info',
-        ],
+        'parent' => $parent,
+        'position' => $position,
         'access' => 'user',
         'workspaces' => 'live',
         'path' => '/module/page/crawler',
@@ -43,7 +53,6 @@
         'parent' => 'web_site_crawler',
         'access' => 'user',
         'path' => '/module/page/crawler/start',
-        'iconIdentifier' => 'crawler-start',
         'labels' => [
             'title' => 'Start',
         ],
@@ -57,7 +66,6 @@
         'parent' => 'web_site_crawler',
         'access' => 'user',
         'path' => '/module/page/crawler/process',
-        'iconIdentifier' => 'crawler-process',
         'labels' => [
             'title' => 'Process',
         ],
@@ -71,7 +79,6 @@
         'parent' => 'web_site_crawler',
         'access' => 'user',
         'path' => '/module/page/crawler/log',
-        'iconIdentifier' => 'crawler-log',
         'labels' => [
             'title' => 'Log',
         ],

Tests/Acceptance/Tests/backend_crawler_configuration.spec.js

@@ -1,7 +1,7 @@
 import { test, expect } from '@playwright/test';
 import * as helpers from './helpers';
 
-test('test', async ({ page }) => {
+test('Able to create and save crawler configuration TYPO3-13', { tag: '@v13' }, async ({ page}) => {
     await helpers.loginBackend(page)
     await page.getByTitle('List', { exact: true }).click();
     await page.locator('div.node:nth-child(2)').click();
@@ -11,3 +11,15 @@ test('test', async ({ page }) => {
     await page.locator('iframe[name="list_frame"]').contentFrame().getByLabel('Name').fill('Test Configuration');
     await page.locator('iframe[name="list_frame"]').contentFrame().getByRole('button', { name: 'Save' }).click();
 });
+
+test('Able to create and save crawler configuration TYPO3-14', { tag: '@wip' }, async ({ page}) => {
+    await helpers.loginBackend(page)
+    await page.getByTitle('Records', { exact: true }).click();
+    await page.waitForLoadState('networkidle');
+    await page.locator('div').filter({ hasText: /^Welcome$/ }).first().click();
+    await page.locator('iframe[name="list_frame"]').contentFrame().getByRole('button', { name: 'New Crawler Configuration' }).click();
+    await expect(page.locator('iframe[name="list_frame"]').contentFrame().locator('h1')).toContainText('Create new Crawler Configuration on page "Welcome"');
+    //await page.locator('iframe[name="list_frame"]').contentFrame().getByLabel('Name').click();
+    await page.locator('iframe[name="list_frame"]').contentFrame().getByLabel('Name').fill('Test Configuration');
+    await page.locator('iframe[name="list_frame"]').contentFrame().getByRole('button', { name: 'Save' }).click();
+});

Tests/Acceptance/Tests/backend_module_basic.spec.js

@@ -3,25 +3,25 @@ const {test, expect} = require('@playwright/test');
 import * as helpers from './helpers';
 
 
-test('Can see crawler backend module', async ({page}) => {
+test('Can see crawler backend module', { tag: ['@v13', '@v14'] }, async ({page}) => {
     await helpers.loginBackend(page)
     await helpers.openCrawlerModule(page);
     await expect(page).toHaveURL('/typo3/module/page/crawler');
 });
 
-test('Can select and see crawler module start crawling', async ({page}) => {
+test('Can select and see crawler module start crawling', { tag: ['@v13','@v14'] }, async ({page}) => {
     await helpers.loginBackend(page)
     await helpers.openCrawlerModuleStartCrawling(page);
     await expect(page).toHaveURL('/typo3/module/page/crawler/start?id=1');
 });
 
-test('Can select and see crawler module log', async ({page}) => {
+test('Can select and see crawler module log', { tag: ['@v13','@v14'] }, async ({page}) => {
     await helpers.loginBackend(page)
     await helpers.openCrawlerModuleCrawlerLog(page);
     await expect(page).toHaveURL('/typo3/module/page/crawler/log?id=1');
 })
 
-test('Can select and see crawler module process', async ({page}) => {
+test('Can select and see crawler module process', { tag: ['@v13','@v14'] }, async ({page}) => {
     await helpers.loginBackend(page)
     await helpers.openCrawlerModuleCrawlerProcesses(page)
     await expect(page).toHaveURL('/typo3/module/page/crawler/process?id=1');

Tests/Acceptance/Tests/backend_module_log.spec.js

@@ -1,7 +1,7 @@
 import {test, expect} from '@playwright/test';
 import * as helpers from './helpers';
 
-test('Can crawl manually in log module, and keep selected log depth', async ({page}) => {
+test('Can crawl manually in log module, and keep selected log depth', { tag: ['@v13'] },async ({page}) => {
     await helpers.loginBackend(page)
     await helpers.addQueueEntries(page, 'default', '99');
     await helpers.openCrawlerModuleCrawlerLog(page)

Tests/Acceptance/Tests/backend_module_process.spec.js

@@ -11,7 +11,7 @@ async function addQueueEntries(page, config, depth = '0') {
     await expect(page.locator('#nprogress')).toHaveCount(0);
 }
 
-test('Can Flush all processes', async ({page}) => {
+test('Can Flush all processes', { tag: ['@v13'] },async ({page}) => {
     await helpers.loginBackend(page)
     await addQueueEntries(page, 'default', '99');
     await helpers.openCrawlerModuleCrawlerProcesses(page)
@@ -22,7 +22,7 @@ test('Can Flush all processes', async ({page}) => {
     await expect(page.locator('iframe[name="list_frame"]').contentFrame().locator('#processes tbody tr')).toHaveCount(0)
 });
 
-test('Can disable and enable crawler', async ({page}) => {
+test('Can disable and enable crawler', { tag: ['@v13','@v14'] },async ({page}) => {
     await helpers.loginBackend(page)
     await helpers.openCrawlerModuleCrawlerProcesses(page);
     await page.locator('iframe[name="list_frame"]').contentFrame().getByRole('button', {name: 'Stop all processes and'}).click();
@@ -31,7 +31,7 @@ test('Can disable and enable crawler', async ({page}) => {
     await expect(page.locator('iframe[name="list_frame"]').contentFrame().locator('body')).toContainText('Stop all processes and disable crawling');
 });
 
-test('Can add process', async ({page}) => {
+test('Can add process', { tag: ['@v13'] },async ({page}) => {
     await helpers.loginBackend(page)
     await addQueueEntries(page, 'default')
     await expect(page.locator('iframe[name="list_frame"]').contentFrame().getByText('URLs submitted')).toContainText('1 URLs submitted');
@@ -41,7 +41,7 @@ test('Can add process', async ({page}) => {
     await expect(page.locator('iframe[name="list_frame"]').contentFrame().getByText('New process has been started')).toContainText('New process has been started');
 });
 
-test('Process successful', async ({page}) => {
+test('Process successful', { tag: ['@v13'] },async ({page}) => {
     await helpers.loginBackend(page)
     await addQueueEntries(page, 'default')
     await expect(page.locator('iframe[name="list_frame"]').contentFrame().getByText('URLs submitted')).toContainText('1 URLs submitted');

Tests/Acceptance/Tests/backend_module_start_crawling.spec.js

@@ -1,7 +1,7 @@
 import { test, expect } from '@playwright/test';
 import * as helpers from './helpers';
 
-test('Update URL button', async ({ page }) => {
+test('Update URL button', { tag: ['@v13'] },async ({ page }) => {
     await helpers.loginBackend(page)
     await helpers.openCrawlerModuleStartCrawling(page)
     await page.locator('div.node:nth-child(2)').click();
@@ -12,7 +12,7 @@ test('Update URL button', async ({ page }) => {
     await expect(page.locator('iframe[name="list_frame"]').contentFrame().getByText('Count')).toContainText('Count: 1');
 });
 
-test('CrawlerConfigurationWithExcludePageSixPlusThree', async ({ page }) => {
+test('CrawlerConfigurationWithExcludePageSixPlusThree', { tag: ['@v13'] }, async ({ page }) => {
     await helpers.loginBackend(page)
     await helpers.openCrawlerModuleStartCrawling(page)
     await page.locator('div.node:nth-child(2)').click();
@@ -24,7 +24,7 @@ test('CrawlerConfigurationWithExcludePageSixPlusThree', async ({ page }) => {
     await expect(page.locator('iframe[name="list_frame"]').contentFrame().getByRole('document')).not.toContainText('TypeError');
 });
 
-test('UpdateUrlButtonSetDepth', async ({ page }) => {
+test('UpdateUrlButtonSetDepth', { tag: ['@v13'] }, async ({ page }) => {
     await helpers.loginBackend(page)
     await helpers.openCrawlerModuleStartCrawling(page)
     await page.locator('div.node:nth-child(2)').click();

Tests/Acceptance/Tests/frontend.spec.js

@@ -1,18 +1,18 @@
 // @ts-check
 const {test, expect} = require('@playwright/test');
 
-test('Can see homepage', async ({page}) => {
+test('Can see homepage',{ tag: ['@v13','@v14'] }, async ({page}) => {
     await page.goto('/');
     await expect(page.getByRole('list').first()).toContainText('Search');
     await expect(page.getByRole('list').first()).toContainText('Login');
 });
 
-test('Can see news page', async ({page}) => {
+test('Can see news page',{ tag: ['@v13','@v14'] }, async ({page}) => {
     await page.goto('/news');
     await expect(page.getByRole('document')).toContainText('No news available.');
 });
 
-test.skip('Can see search page and search for Tomasnorre', async ({page}) => {
+test.skip('Can see search page and search for Tomasnorre', { tag: ['@v13','@v14'] },async ({page}) => {
 
     await page.goto('/search');
     await expect(page.getByRole('document')).toContainText('Search');

Tests/Acceptance/Tests/helpers.js

@@ -8,7 +8,7 @@ export async function loginBackend(page) {
 }
 
 export async function openCrawlerModule(page) {
-    await page.getByText('ext_icon_crawler Crawler').click();
+    await page.getByText('Crawler module').click();
 }
 
 export async function openCrawlerModuleCrawlerProcesses(page) {

Tests/Acceptance/playwright.config.js

@@ -21,25 +21,42 @@ module.exports = defineConfig({
   /* Opt out of parallel tests on CI. */
   workers: process.env.CI ? 1 : undefined,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: 'dot',
+    reporter: [
+        ['html', { outputFolder: 'playwright-report', open: 'never' }],
+        ['dot'],
+    ],
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     /* Base URL to use in actions like `await page.goto('/')`. */
     baseURL: 'https://crawler-devbox.ddev.site/',
 
     /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
-    trace: 'on-first-retry',
+      trace: 'retain-on-failure',
 
     /* Whether to ignore HTTPS errors when sending network requests. See https://playwright.dev/docs/api/class-testoptions#test-options-ignore-https-errors*/
     ignoreHTTPSErrors: true,
+
+    // 📸 Screenshot on failure
+    screenshot: 'only-on-failure',
+
+    // 🎥 Video recording
+    video: 'retain-on-failure',
+
+    browserName: 'chromium',
+    launchOptions: {
+      args: [
+          '--no-sandbox',
+          '--disable-dev-shm-usage',
+      ],
+    },
   },
   /* Configure projects for major browsers */
   projects: [
     {
       name: 'chromium',
       use: { ...devices['Desktop Chrome'] },
     },
-/*
+      /*
     {
       name: 'firefox',
       use: { ...devices['Desktop Firefox'] },
@@ -48,7 +65,8 @@ module.exports = defineConfig({
     {
       name: 'webkit',
       use: { ...devices['Desktop Safari'] },
-    },*/
+    },
+    */
   ],
 
   /* Run your local dev server before starting the tests */