From 53833c7b9015e48decf96e768dadaa983c922706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Vo=C5=99=C3=AD=C5=A1ek?= Date: Thu, 30 May 2024 16:45:21 +0200 Subject: [PATCH 1/2] Remove "NO OTHERS" from reserved keywords "NO" or "OTHERS" are not reserved keywords per MySQL grammar --- src/Tokenizer.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Tokenizer.php b/src/Tokenizer.php index cad0eb8..b723a54 100644 --- a/src/Tokenizer.php +++ b/src/Tokenizer.php @@ -182,7 +182,6 @@ final class Tokenizer 'MYISAM', 'NAMES', 'NATURAL', - 'NO OTHERS', 'NOT', 'NULL', 'OFFSET', From ff667cc161259e189f0a0de910840e842307f7bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Vo=C5=99=C3=AD=C5=A1ek?= Date: Thu, 30 May 2024 16:48:45 +0200 Subject: [PATCH 2/2] Make sure keywords are single upper words The "reserved" keywords are special as they need to be always quoted and per MySQL/SQLite spec they are always single word and case insensitive. --- src/Tokenizer.php | 7 +++---- tests/TokenizerTest.php | 14 ++++++++++++++ tests/clihighlight.txt | 14 +++++++------- tests/format-highlight.html | 14 +++++++------- tests/highlight.html | 10 +++++----- 5 files changed, 36 insertions(+), 23 deletions(-) diff --git a/src/Tokenizer.php b/src/Tokenizer.php index b723a54..82ace8c 100644 --- a/src/Tokenizer.php +++ b/src/Tokenizer.php @@ -50,7 +50,7 @@ final class Tokenizer 'CASE', 'CHANGE', 'CHANGED', - 'CHARACTER SET', + 'CHARACTER', 'CHARSET', 'CHECK', 'CHECKSUM', @@ -68,7 +68,7 @@ final class Tokenizer 'CONVERT', 'CREATE', 'CROSS', - 'CURRENT ROW', + 'CURRENT', 'CURRENT_TIMESTAMP', 'DATABASE', 'DATABASES', @@ -186,8 +186,6 @@ final class Tokenizer 'NULL', 'OFFSET', 'ON', - 'ON DELETE', - 'ON UPDATE', 'OPEN', 'OPTIMIZE', 'OPTION', @@ -241,6 +239,7 @@ final class Tokenizer 'SEPARATOR', 'SERIALIZABLE', 'SESSION', + 'SET', 'SHARE', 'SHOW', 'SHUTDOWN', diff --git a/tests/TokenizerTest.php b/tests/TokenizerTest.php index a10838d..9f82319 100644 --- a/tests/TokenizerTest.php +++ b/tests/TokenizerTest.php @@ -9,7 +9,10 @@ use PHPUnit\Framework\TestCase; use ReflectionClass; +use function array_filter; +use function preg_match; use function sort; +use function strtoupper; final class TokenizerTest extends TestCase { @@ -44,6 +47,17 @@ public function testInternalKeywordListsAreSortedForEasierMaintenance(): void } } + public function testKeywordsReservedAreSingleUpperWord(): void + { + $tokenizerReserved = $this->getTokenizerList('reserved'); + + $kwsDiff = array_filter($tokenizerReserved, static function ($v) { + return $v !== strtoupper($v) || preg_match('~^\w+$~', $v) !== 1; + }); + + self::assertSame([], $kwsDiff); + } + #[DoesNotPerformAssertions] public function testThereAreNoRegressions(): void { diff --git a/tests/clihighlight.txt b/tests/clihighlight.txt index ab3bb04..c98a763 100644 --- a/tests/clihighlight.txt +++ b/tests/clihighlight.txt @@ -915,23 +915,23 @@ c GROUPS BETWEEN UNBOUNDED PRECEDING - AND CURRENT ROW - EXCLUDE NO OTHERS + AND CURRENT ROW + EXCLUDE NO OTHERS ) AS no_others, GROUP_CONCAT(b, '.') OVER ( ORDER BY c GROUPS BETWEEN UNBOUNDED PRECEDING - AND CURRENT ROW - EXCLUDE CURRENT ROW + AND CURRENT ROW + EXCLUDE CURRENT ROW ) AS current_row, GROUP_CONCAT(b, '.') OVER ( ORDER BY c GROUPS BETWEEN UNBOUNDED PRECEDING - AND CURRENT ROW + AND CURRENT ROW EXCLUDE GROUP ) AS grp, GROUP_CONCAT(b, '.') OVER ( @@ -939,7 +939,7 @@ c GROUPS BETWEEN UNBOUNDED PRECEDING - AND CURRENT ROW + AND CURRENT ROW EXCLUDE TIES ) AS tie, GROUP_CONCAT(b, '.') FILTER ( @@ -990,7 +990,7 @@ ORDER BY RevenueYear ROWS - BETWEEN CURRENT ROW + BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING ) AS MinRevenueBeyond FROM diff --git a/tests/format-highlight.html b/tests/format-highlight.html index 106d5a2..8d6521c 100644 --- a/tests/format-highlight.html +++ b/tests/format-highlight.html @@ -915,23 +915,23 @@ c GROUPS BETWEEN UNBOUNDED PRECEDING - AND CURRENT ROW - EXCLUDE NO OTHERS + AND CURRENT ROW + EXCLUDE NO OTHERS ) AS no_others, GROUP_CONCAT(b, '.') OVER ( ORDER BY c GROUPS BETWEEN UNBOUNDED PRECEDING - AND CURRENT ROW - EXCLUDE CURRENT ROW + AND CURRENT ROW + EXCLUDE CURRENT ROW ) AS current_row, GROUP_CONCAT(b, '.') OVER ( ORDER BY c GROUPS BETWEEN UNBOUNDED PRECEDING - AND CURRENT ROW + AND CURRENT ROW EXCLUDE GROUP ) AS grp, GROUP_CONCAT(b, '.') OVER ( @@ -939,7 +939,7 @@ c GROUPS BETWEEN UNBOUNDED PRECEDING - AND CURRENT ROW + AND CURRENT ROW EXCLUDE TIES ) AS tie, GROUP_CONCAT(b, '.') FILTER ( @@ -990,7 +990,7 @@ ORDER BY RevenueYear ROWS - BETWEEN CURRENT ROW + BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING ) AS MinRevenueBeyond FROM diff --git a/tests/highlight.html b/tests/highlight.html index fe4fcb3..6eb6da8 100644 --- a/tests/highlight.html +++ b/tests/highlight.html @@ -288,10 +288,10 @@ WHERE cte1.a = cte2.c; --- 
SELECT a,
-    GROUP_CONCAT(b, '.') OVER (ORDER BY c GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE NO OTHERS) AS no_others,
-    GROUP_CONCAT(b, '.') OVER (ORDER BY c GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE CURRENT ROW) AS current_row,
-    GROUP_CONCAT(b, '.') OVER (ORDER BY c GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE GROUP) AS grp,
-    GROUP_CONCAT(b, '.') OVER (ORDER BY c GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE TIES) AS tie,
+    GROUP_CONCAT(b, '.') OVER (ORDER BY c GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE NO OTHERS) AS no_others,
+    GROUP_CONCAT(b, '.') OVER (ORDER BY c GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE CURRENT ROW) AS current_row,
+    GROUP_CONCAT(b, '.') OVER (ORDER BY c GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE GROUP) AS grp,
+    GROUP_CONCAT(b, '.') OVER (ORDER BY c GROUPS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE TIES) AS tie,
     GROUP_CONCAT(b, '.') FILTER (WHERE c != 'two') OVER (ORDER BY a) AS filtered,
     CONVERT(VARCHAR(20), AVG(SalesYTD) OVER (PARTITION BY TerritoryID ORDER BY DATEPART(yy, ModifiedDate)), 1) AS MovingAvg,
     AVG(starting_salary) OVER w2 AVG,
@@ -299,7 +299,7 @@
     MAX(starting_salary) OVER (w1 ORDER BY hire_date),
     LISTAGG(arg, ',') OVER (PARTITION BY part ORDER BY ord ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS LISTAGG_ROWS,
     LISTAGG(arg, ',') OVER (PARTITION BY part ORDER BY ord RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS LISTAGG_RANGE,
-    MIN(Revenue) OVER (PARTITION BY DepartmentID ORDER BY RevenueYear ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS MinRevenueBeyond
+    MIN(Revenue) OVER (PARTITION BY DepartmentID ORDER BY RevenueYear ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS MinRevenueBeyond
 FROM t1
 WINDOW w1 AS (PARTITION BY department, division), w2 AS (w1 ORDER BY hire_date);
---