Merge pull request #20 from ai-kana/main

More tokenizer fixes, code generator fix
NotNite · Nov 8, 2024 · 6be5c14 · 6be5c14
2 parents cf0175c + 3b82b98
commit 6be5c14
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 16 deletions.
diff --git a/GDWeave.Dumper/CodeGenerator.cs b/GDWeave.Dumper/CodeGenerator.cs
@@ -2,10 +2,18 @@
 
 public class CodeGenerator(List<Token> tokens, List<string> identifiers) {
     public void Generate(StreamWriter writer) {
+        var onNewLine = false;
         foreach (var token in tokens) {
             var tabs = 0u;
             var gen = this.GenerateToken(token, ref tabs);
 
+            onNewLine = gen == "\n";
+
+            if (!onNewLine)
+            {
+                gen += ' ';
+            }
+
             writer.Write(gen);
 
             for (var i = 0; i < tabs; i++) {

diff --git a/GDWeave/Script/ScriptTokenizer.cs b/GDWeave/Script/ScriptTokenizer.cs
@@ -100,7 +100,7 @@ public static class ScriptTokenizer {
         {"|=", TokenType.OpAssignBitOr},
         {"^=", TokenType.OpAssignBitXor},
 
-        {"+", TokenType.OpAnd},
+        {"+", TokenType.OpAdd},
         {"-", TokenType.OpSub},
         {"*", TokenType.OpMul},
         {"/", TokenType.OpDiv},
@@ -139,7 +139,7 @@ public static class ScriptTokenizer {
         "*=",
         "/=",
         "%=",
-        "&=",
+       "&=",
         "|=",
         "^=",
 
@@ -205,7 +205,8 @@ private static void BuildIdentifierName(IEnumerator<string> enumerator, List<Tok
         found = "_" + enumerator.Current;
     }
 
-    private static void BuildNumber(IEnumerator<string> enumerator, List<Token> toFlush) {
+    private static void BuildNumber(IEnumerator<string> enumerator, List<Token> toFlush, out bool foundFull) {
+        foundFull = true;
         int sign = 1;
 
         if (enumerator.Current == "-") {
@@ -215,13 +216,15 @@ private static void BuildNumber(IEnumerator<string> enumerator, List<Token> toFl
 
         if (!long.TryParse(enumerator.Current, out long upper)) {
             toFlush.Add(new Token(TokenType.OpSub));
+            foundFull = false;
             return;
         }
 
         if (!enumerator.MoveNext()) return;
 
         if (enumerator.Current != ".") {
             toFlush.Add(new ConstantToken(new IntVariant(upper * sign)));
+            foundFull = false;
             return;
         }
 
@@ -249,15 +252,17 @@ public static IEnumerable<Token> Tokenize(string gdScript, uint baseIndent = 0)
         var toFlush = new List<Token>(2);
         finalTokens.Add(new Token(TokenType.Newline, baseIndent));
         var enumerator = tokens.GetEnumerator();
-        while (enumerator.MoveNext()) {
-            var current = enumerator.Current;
-            if (current == "\n") {
+        var reparse = false;
+        while (reparse ? true : enumerator.MoveNext()) {
+            reparse = false;
+
+            if (enumerator.Current == "\n") {
                 InsertNewLine(enumerator, baseIndent, toFlush);
                 endAndFlushId();
                 continue;
             }
 
-            if (current == "_") {
+            if (enumerator.Current == "_") {
                 BuildIdentifierName(enumerator, toFlush, out string? found);
                 if (found == string.Empty) {
                     endAndFlushId();
@@ -270,37 +275,39 @@ public static IEnumerable<Token> Tokenize(string gdScript, uint baseIndent = 0)
                 continue;
             }
 
-            if (current == "-" || char.IsDigit(current[0])) {
-                BuildNumber(enumerator, toFlush);
+            if (enumerator.Current == "-" || char.IsDigit(enumerator.Current[0])) {
+                BuildNumber(enumerator, toFlush, out bool foundFull);
+                reparse = !foundFull;
                 endAndFlushId();
                 continue;
             }
 
-            if (BuiltinFunctions.Contains(current)) {
-                toFlush.Add(new Token(TokenType.BuiltInFunc, (uint?) BuiltinFunctions.IndexOf(current)));
+            if (BuiltinFunctions.Contains(enumerator.Current)) {
+                toFlush.Add(new Token(TokenType.BuiltInFunc, (uint?) BuiltinFunctions.IndexOf(enumerator.Current)));
                 endAndFlushId();
                 continue;
             }
 
-            if (Tokens.TryGetValue(current, out var type)) {
+            if (Tokens.TryGetValue(enumerator.Current, out var type)) {
                 toFlush.Add(new Token(type));
                 endAndFlushId();
                 continue;
             }
 
-            if (current.StartsWith('"')) {
+            if (enumerator.Current.StartsWith('"')) {
+                var current = enumerator.Current;
                 toFlush.Add(new ConstantToken(new StringVariant(current.Substring(1, current.Length - 2))));
                 endAndFlushId();
                 continue;
             }
 
-            if (bool.TryParse(current, out var resultB)) {
+            if (bool.TryParse(enumerator.Current, out var resultB)) {
                 toFlush.Add(new ConstantToken(new BoolVariant(resultB)));
                 endAndFlushId();
                 continue;
             }
 
-            idName += current;
+            idName += enumerator.Current;
 
             end();
 
@@ -323,7 +330,6 @@ void endAndFlushId() {
         finalTokens.Add(new(TokenType.Newline, baseIndent));
 
         foreach (var t in finalTokens) yield return t;
-
     }
 
     private static IEnumerable<string> SanitizeInput(IEnumerable<string> tokens) {