feat: update grammar to use full utf-8 range

Author: calebdw

common/define-grammar.js

@@ -109,7 +109,7 @@ module.exports = function defineGrammar(dialect) {
     extras: $ => {
       const extras = [
         $.comment,
-        /[\s\uFEFF\u2060\u200B\u00A0]/,
+        /[\s\u00A0\u200B\u2060\uFEFF]/,
       ];
 
       if (dialect === 'php') {
@@ -1517,7 +1517,7 @@ module.exports = function defineGrammar(dialect) {
         $._expression,
       ),
 
-      name: _ => /[_a-zA-Z\u00A1-\u00ff][_a-zA-Z\u00A1-\u00ff\d]*/,
+      name: _ => name(),
 
       _reserved_identifier: _ => choice(
         'self',
@@ -1605,3 +1605,21 @@ function pipeSep1(rule) {
 function ampSep1(rule) {
   return seq(rule, repeat(seq(token('&'), rule)));
 }
+
+/**
+ * Creates a regex that matches PHP identifiers.
+ *
+ * Note that PHP officially only supports the following character regex
+ * for identifiers: ^[a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*$
+ * However, there is a "bug" in how PHP parses multi-byte characters that allows
+ * for a much larger range of characters to be used in identifiers.
+
+ * @see https://www.php.net/manual/en/language.variables.basics.php
+ *
+ * @return {RexExp}
+ */
+function name() {
+  // We need to side step around the whitespace characters in the extras array.
+  const range = String.raw`\u0080-\u009f\u00a1-\u200a\u200c-\u205f\u2061-\ufefe\uff00-\uffff`;
+  return new RegExp(`[_a-zA-Z${range}][_a-zA-Z${range}\\d]*`);
+}

common/test/corpus/expressions.txt

@@ -1558,3 +1558,37 @@ Foo::{$bar};
     (class_constant_access_expression
       (name)
       (name (variable_name (name))))))
+
+===============================================
+UTF-8 identifiers
+===============================================
+
+<?php
+$漢字;
+
+<<<漢漢字
+  This is a heredoc
+漢漢字;
+
+<<<'字漢'
+  This is a nowdoc.
+字漢;
+
+---
+
+(program
+  (php_tag)
+  (expression_statement
+    (variable_name (name)))
+    (expression_statement
+      (heredoc
+        (heredoc_start)
+        (heredoc_body
+          (string_value))
+        (heredoc_end)))
+    (expression_statement
+      (nowdoc
+        (heredoc_start)
+        (nowdoc_body
+          (nowdoc_string))
+        (heredoc_end))))