diff --git a/.github/workflows/test-translationtool.yml b/.github/workflows/test-translationtool.yml
new file mode 100644
index 0000000..81881a8
--- /dev/null
+++ b/.github/workflows/test-translationtool.yml
@@ -0,0 +1,66 @@
+# This workflow is provided via the organization template repository
+#
+# https://github.com/nextcloud/.github
+# https://docs.github.com/en/actions/learn-github-actions/sharing-workflows-with-your-organization
+#
+# SPDX-FileCopyrightText: 2021-2024 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: MIT
+
+name: Test translationtool
+
+on: pull_request
+
+permissions:
+  contents: read
+
+concurrency:
+  group: lint-php-cs-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest-low
+    permissions:
+      contents: read
+      pull-requests: read
+
+    outputs:
+      src: ${{ steps.changes.outputs.src}}
+
+    steps:
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
+        id: changes
+        continue-on-error: true
+        with:
+          filters: |
+            src:
+              - '.github/workflows/**'
+              - 'translations/translationtool/**'
+
+  test:
+    runs-on: ubuntu-latest
+
+    needs: [changes]
+    name: Test translationtool
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Set up php8.3
+        uses: shivammathur/setup-php@c541c155eee45413f5b09a52248675b1a2575231 # v2.31.1
+        with:
+          php-version: 8.3
+          extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite
+          coverage: none
+          ini-file: development
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install dependencies
+        working-directory: translations/translationtool/
+        run: composer i
+
+      - name: Create POT files
+        working-directory: translations/translationtool/
+        run: composer run test
diff --git a/translations/translationtool/.gitignore b/translations/translationtool/.gitignore
index c8b802d..3511f07 100644
--- a/translations/translationtool/.gitignore
+++ b/translations/translationtool/.gitignore
@@ -2,3 +2,4 @@ vendor
 composer.phar
 .vscode
 .php-cs-fixer.cache
+/tests/translationfiles/templates/tests.pot
diff --git a/translations/translationtool/composer.json b/translations/translationtool/composer.json
index cca1d85..66530fa 100644
--- a/translations/translationtool/composer.json
+++ b/translations/translationtool/composer.json
@@ -21,6 +21,8 @@
 	"scripts": {
 		"cs:check": "php-cs-fixer fix --dry-run --diff",
 		"cs:fix": "php-cs-fixer fix",
+		"test": "cd tests && bash test.sh",
+
 		"post-install-cmd": [
 			"@composer bin all install --ansi"
 		],
diff --git a/translations/translationtool/tests/expected.pot b/translations/translationtool/tests/expected.pot
new file mode 100644
index 0000000..fbbbccc
--- /dev/null
+++ b/translations/translationtool/tests/expected.pot
@@ -0,0 +1,168 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the Nextcloud package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+
+#. TRANSLATORS PHP string with plain text
+#: lib/php.php:12
+msgid "PHP String"
+msgstr ""
+
+#. TRANSLATORS PHP string with parameters
+#: lib/php.php:14
+#, php-format
+msgid "PHP String with %s"
+msgstr ""
+
+#. TRANSLATORS PHP string with positional parameters
+#: lib/php.php:16
+#, php-format
+msgid "PHP String with positional %1$s"
+msgstr ""
+
+#. TRANSLATORS PHP plural with plain text
+#: lib/php.php:21
+msgid "PHP %n Plural"
+msgid_plural "PHP %n Plurals"
+msgstr[0] ""
+msgstr[1] ""
+
+#. TRANSLATORS PHP plural with parameters
+#: lib/php.php:23
+msgid "PHP %n Plural with %s"
+msgid_plural "PHP %n Plurals with %s"
+msgstr[0] ""
+msgstr[1] ""
+
+#. TRANSLATORS PHP plural with positional parameters
+#: lib/php.php:27
+msgid "PHP %n Plural with positional %1$s"
+msgid_plural "PHP %n Plurals with positional %1$s"
+msgstr[0] ""
+msgstr[1] ""
+
+#. TRANSLATORS VUE string with plain text (src/vue.vue:8)
+#: specialVueFakeDummyForL10nScript.js:2
+msgid "VUE String"
+msgstr ""
+
+#. TRANSLATORS VUE string with parameters (src/vue.vue:10)
+#: specialVueFakeDummyForL10nScript.js:4
+msgid "VUE String with inline {parameter}"
+msgstr ""
+
+#. TRANSLATORS VUE string with wrapped parameters (src/vue.vue:12)
+#: specialVueFakeDummyForL10nScript.js:6
+msgid "VUE String with wrapped {parameter}"
+msgstr ""
+
+#. TRANSLATORS VUE Fix for https://github.com/nextcloud/docker-ci/pull/663 - Part 1 (src/vue.vue:26)
+#: specialVueFakeDummyForL10nScript.js:8
+msgid "VUE String testing"
+msgstr ""
+
+#: specialVueFakeDummyForL10nScript.js:8
+msgid "VUE String not"
+msgstr ""
+
+#. TRANSLATORS VUE Fix for https://github.com/nextcloud/docker-ci/pull/663 - Part 2 (src/vue.vue:29)
+#: specialVueFakeDummyForL10nScript.js:10
+msgid "VUE String with followup double quotes"
+msgstr ""
+
+#. TRANSLATORS VUE plural with plain text (src/vue.vue:17)
+#: specialVueFakeDummyForL10nScript.js:12
+msgid "VUE %n Plural"
+msgid_plural "VUE %n Plurals"
+msgstr[0] ""
+msgstr[1] ""
+
+#. TRANSLATORS VUE plural with wrapped parameters (src/vue.vue:21)
+#: specialVueFakeDummyForL10nScript.js:16
+msgid "VUE %n Plural with wrapped %s"
+msgid_plural "VUE %n Plurals with wrapped %s"
+msgstr[0] ""
+msgstr[1] ""
+
+#. TRANSLATORS VUE plural with parameters (src/vue.vue:19)
+#: specialVueFakeDummyForL10nScript.js:14
+msgid "VUE %n Plural with %s"
+msgid_plural "VUE %n Plurals with %s"
+msgstr[0] ""
+msgstr[1] ""
+
+#. TRANSLATORS JS string with plain text
+#: src/js.js:7
+msgid "JS String"
+msgstr ""
+
+#. TRANSLATORS JS string with parameters
+#: src/js.js:9
+msgid "JS String with inline {parameter}"
+msgstr ""
+
+#. TRANSLATORS JS string with wrapped parameters
+#: src/js.js:11
+msgid "JS String with wrapped {parameter}"
+msgstr ""
+
+#. TRANSLATORS JS plural with plain text
+#: src/js.js:16
+msgid "JS %n Plural"
+msgid_plural "JS %n Plurals"
+msgstr[0] ""
+msgstr[1] ""
+
+#. TRANSLATORS JS plural with parameters
+#: src/js.js:18
+msgid "JS %n Plural with %s"
+msgid_plural "JS %n Plurals with %s"
+msgstr[0] ""
+msgstr[1] ""
+
+#. TRANSLATORS JS plural with wrapped parameters
+#: src/js.js:20
+msgid "JS %n Plural with wrapped %s"
+msgid_plural "JS %n Plurals with wrapped %s"
+msgstr[0] ""
+msgstr[1] ""
+
+#. TRANSLATORS TS string with plain text
+#: src/ts.ts:7
+msgid "TS String"
+msgstr ""
+
+#. TRANSLATORS TS string with parameters
+#: src/ts.ts:9
+msgid "TS String with inline {parameter}"
+msgstr ""
+
+#. TRANSLATORS TS string with wrapped parameters
+#: src/ts.ts:11
+msgid "TS String with wrapped {parameter}"
+msgstr ""
+
+#. TRANSLATORS TS plural with plain text
+#: src/ts.ts:16
+msgid "TS %n Plural"
+msgid_plural "TS %n Plurals"
+msgstr[0] ""
+msgstr[1] ""
+
+#. TRANSLATORS TS plural with parameters
+#: src/ts.ts:18
+msgid "TS %n Plural with %s"
+msgid_plural "TS %n Plurals with %s"
+msgstr[0] ""
+msgstr[1] ""
+
+#. TRANSLATORS TS plural with wrapped parameters
+#: src/ts.ts:20
+msgid "TS %n Plural with wrapped %s"
+msgid_plural "TS %n Plurals with wrapped %s"
+msgstr[0] ""
+msgstr[1] ""
diff --git a/translations/translationtool/tests/l10n/.gitkeep b/translations/translationtool/tests/l10n/.gitkeep
new file mode 100644
index 0000000..7a2ef8e
--- /dev/null
+++ b/translations/translationtool/tests/l10n/.gitkeep
@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: CC0-1.0
diff --git a/translations/translationtool/tests/lib/php.php b/translations/translationtool/tests/lib/php.php
new file mode 100644
index 0000000..615cca4
--- /dev/null
+++ b/translations/translationtool/tests/lib/php.php
@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+$l = \OCP\Server::get(\OCP\IL10N::class);
+
+// TRANSLATORS PHP string with plain text
+$l->t('PHP String');
+// TRANSLATORS PHP string with parameters
+$l->t('PHP String with %s', 'Parameter');
+// TRANSLATORS PHP string with positional parameters
+$l->t('PHP String with positional %1$s', [
+	'Parameter',
+]);
+
+// TRANSLATORS PHP plural with plain text
+$l->n('PHP %n Plural', 'PHP %n Plurals', 6);
+// TRANSLATORS PHP plural with parameters
+$l->n('PHP %n Plural with %s', 'PHP %n Plurals with %s', 6, [
+	'Parameter',
+]);
+// TRANSLATORS PHP plural with positional parameters
+$l->n('PHP %n Plural with positional %1$s', 'PHP %n Plurals with positional %1$s', 6, [
+	'Parameter',
+]);
diff --git a/translations/translationtool/tests/src/js.js b/translations/translationtool/tests/src/js.js
new file mode 100644
index 0000000..8278a95
--- /dev/null
+++ b/translations/translationtool/tests/src/js.js
@@ -0,0 +1,22 @@
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+// TRANSLATORS JS string with plain text
+t('test', 'JS String');
+// TRANSLATORS JS string with parameters
+t('test', 'JS String with inline {parameter}', { parameter: 'Parameter' });
+// TRANSLATORS JS string with wrapped parameters
+t('test', 'JS String with wrapped {parameter}', {
+	parameter: 'Parameter',
+});
+
+// TRANSLATORS JS plural with plain text
+n('test', 'JS %n Plural', 'JS %n Plurals', 6);
+// TRANSLATORS JS plural with parameters
+n('test', 'JS %n Plural with %s', 'JS %n Plurals with %s', 6, { parameter: 'Parameter' });
+// TRANSLATORS JS plural with wrapped parameters
+n('test', 'JS %n Plural with wrapped %s', 'JS %n Plurals with wrapped %s', 6,  {
+	parameter: 'Parameter',
+});
diff --git a/translations/translationtool/tests/src/ts.ts b/translations/translationtool/tests/src/ts.ts
new file mode 100644
index 0000000..e8c931d
--- /dev/null
+++ b/translations/translationtool/tests/src/ts.ts
@@ -0,0 +1,22 @@
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+// TRANSLATORS TS string with plain text
+t('test', 'TS String')
+// TRANSLATORS TS string with parameters
+t('test', 'TS String with inline {parameter}', { parameter: 'Parameter' });
+// TRANSLATORS TS string with wrapped parameters
+t('test', 'TS String with wrapped {parameter}', {
+	parameter: 'Parameter',
+});
+
+// TRANSLATORS TS plural with plain text
+n('test', 'TS %n Plural', 'TS %n Plurals', 6)
+// TRANSLATORS TS plural with parameters
+n('test', 'TS %n Plural with %s', 'TS %n Plurals with %s', 6, { parameter: 'Parameter' });
+// TRANSLATORS TS plural with wrapped parameters
+n('test', 'TS %n Plural with wrapped %s', 'TS %n Plurals with wrapped %s', 6,  {
+	parameter: 'Parameter',
+});
diff --git a/translations/translationtool/tests/src/vue.vue b/translations/translationtool/tests/src/vue.vue
new file mode 100644
index 0000000..9a7b1f8
--- /dev/null
+++ b/translations/translationtool/tests/src/vue.vue
@@ -0,0 +1,30 @@
+<!--
+  - SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+  - SPDX-License-Identifier: AGPL-3.0-or-later
+-->
+
+<template>
+	<!-- TRANSLATORS VUE string with plain text -->
+	{{ t('test', 'VUE String') }}
+	<!-- TRANSLATORS VUE string with parameters -->
+	{{ t('test', 'VUE String with inline {parameter}', { parameter: 'Parameter' }) }}
+	<!-- TRANSLATORS VUE string with wrapped parameters -->
+	{{ t('test', 'VUE String with wrapped {parameter}', {
+		parameter: 'Parameter',
+	}) }}
+
+	<!-- TRANSLATORS VUE plural with plain text -->
+	{{ n('test', 'VUE %n Plural', 'VUE %n Plurals', 6) }}
+	<!-- TRANSLATORS VUE plural with parameters -->
+	{{ n('test', 'VUE %n Plural with %s', 'VUE %n Plurals with %s', 6, { parameter: 'Parameter' }) }}
+	<!-- TRANSLATORS VUE plural with wrapped parameters -->
+	{{ n('test', 'VUE %n Plural with wrapped %s', 'VUE %n Plurals with wrapped %s', 6,  {
+		parameter: 'Parameter',
+	}) }}
+
+	<!-- TRANSLATORS VUE Fix for https://github.com/nextcloud/docker-ci/pull/663 - Part 1 -->
+	{{ test ? t('test', 'VUE String testing') : t('test', 'VUE String not') }}
+
+	<!-- TRANSLATORS VUE Fix for https://github.com/nextcloud/docker-ci/pull/663 - Part 2 -->
+	{{ t("test", "VUE String with followup double quotes") }}: {{ data["random property"] }}
+</template>
diff --git a/translations/translationtool/tests/test.sh b/translations/translationtool/tests/test.sh
new file mode 100644
index 0000000..d551bde
--- /dev/null
+++ b/translations/translationtool/tests/test.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+php ../src/translationtool.php create-pot-files
+
+# Remove unstable data like "POT-Creation-Date: 2024-09-10 14:25+0200\n"
+# And remove the development path
+cat translationfiles/templates/tests.pot | grep -v '^"' | sed -e "s/${PWD//\//\\/}\\///" > expected.pot
+
+CHANGED_LINES=$(git diff expected.pot | wc -l)
+
+if ! [[ "$CHANGED_LINES" = "0" ]]; then
+	echo 'POT file changed'
+	git diff expected.pot
+	exit 1
+fi