Support subscript characters in identifiers

Author: triallax

numbat/src/parser.rs

@@ -1687,6 +1687,7 @@ mod tests {
         parse_as_expression(&["foo_bar"], identifier!("foo_bar"));
         parse_as_expression(&["MeineSchöneVariable"], identifier!("MeineSchöneVariable"));
         parse_as_expression(&["°"], identifier!("°"));
+        parse_as_expression(&["Mass_H₂O"], identifier!("Mass_H₂O"));
     }
 
     #[test]

numbat/src/tokenizer.rs

@@ -171,6 +171,13 @@ fn is_other_allowed_identifier_char(c: char) -> bool {
     c == '%'
 }
 
+fn is_subscript_char(c: char) -> bool {
+    let c_u32 = c as u32;
+
+    // See https://en.wikipedia.org/wiki/Unicode_subscripts_and_superscripts#Superscripts_and_subscripts_block
+    (0x2080..=0x209CF).contains(&c_u32)
+}
+
 fn is_identifier_start(c: char) -> bool {
     unicode_ident::is_xid_start(c)
         || is_numerical_fraction_char(c)
@@ -182,6 +189,7 @@ fn is_identifier_start(c: char) -> bool {
 
 fn is_identifier_continue(c: char) -> bool {
     (unicode_ident::is_xid_continue(c)
+        || is_subscript_char(c)
         || is_currency_char(c)
         || is_other_allowed_identifier_char(c))
         && !is_exponent_char(c)
@@ -1049,3 +1057,12 @@ fn test_is_currency_char() {
 
     assert!(!is_currency_char('E'));
 }
+
+#[test]
+fn test_is_subscript_char() {
+    assert!(is_subscript_char('₅'));
+    assert!(is_subscript_char('₁'));
+    assert!(is_subscript_char('ₓ'));
+    assert!(is_subscript_char('ₘ'));
+    assert!(is_subscript_char('₎'));
+}