From 8fd328164cc2fc812ae53d9322f20329a57d9a33 Mon Sep 17 00:00:00 2001 From: ai-kana Date: Tue, 5 Nov 2024 13:59:07 -0800 Subject: [PATCH 1/4] Fix improper token returns There is so much broken still,,, all nighter is not treating me well --- GDWeave/Script/ScriptTokenizer.cs | 34 +++++++++++++++++-------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/GDWeave/Script/ScriptTokenizer.cs b/GDWeave/Script/ScriptTokenizer.cs index b87ecec..55b7303 100644 --- a/GDWeave/Script/ScriptTokenizer.cs +++ b/GDWeave/Script/ScriptTokenizer.cs @@ -139,7 +139,7 @@ public static class ScriptTokenizer { "*=", "/=", "%=", - "&=", + "&=", "|=", "^=", @@ -205,7 +205,8 @@ private static void BuildIdentifierName(IEnumerator enumerator, List enumerator, List toFlush) { + private static void BuildNumber(IEnumerator enumerator, List toFlush, out bool foundFull) { + foundFull = true; int sign = 1; if (enumerator.Current == "-") { @@ -215,6 +216,7 @@ private static void BuildNumber(IEnumerator enumerator, List toFl if (!long.TryParse(enumerator.Current, out long upper)) { toFlush.Add(new Token(TokenType.OpSub)); + foundFull = false; return; } @@ -222,6 +224,7 @@ private static void BuildNumber(IEnumerator enumerator, List toFl if (enumerator.Current != ".") { toFlush.Add(new ConstantToken(new IntVariant(upper * sign))); + foundFull = false; return; } @@ -250,14 +253,13 @@ public static IEnumerable Tokenize(string gdScript, uint baseIndent = 0) finalTokens.Add(new Token(TokenType.Newline, baseIndent)); var enumerator = tokens.GetEnumerator(); while (enumerator.MoveNext()) { - var current = enumerator.Current; - if (current == "\n") { + if (enumerator.Current == "\n") { InsertNewLine(enumerator, baseIndent, toFlush); endAndFlushId(); continue; } - if (current == "_") { + if (enumerator.Current == "_") { BuildIdentifierName(enumerator, toFlush, out string? found); if (found == string.Empty) { endAndFlushId(); @@ -270,37 +272,40 @@ public static IEnumerable Tokenize(string gdScript, uint baseIndent = 0) continue; } - if (current == "-" || char.IsDigit(current[0])) { - BuildNumber(enumerator, toFlush); + if (enumerator.Current == "-" || char.IsDigit(enumerator.Current[0])) { + BuildNumber(enumerator, toFlush, out bool foundFull); endAndFlushId(); - continue; + if (foundFull) { + continue; + } } - if (BuiltinFunctions.Contains(current)) { - toFlush.Add(new Token(TokenType.BuiltInFunc, (uint?) BuiltinFunctions.IndexOf(current))); + if (BuiltinFunctions.Contains(enumerator.Current)) { + toFlush.Add(new Token(TokenType.BuiltInFunc, (uint?) BuiltinFunctions.IndexOf(enumerator.Current))); endAndFlushId(); continue; } - if (Tokens.TryGetValue(current, out var type)) { + if (Tokens.TryGetValue(enumerator.Current, out var type)) { toFlush.Add(new Token(type)); endAndFlushId(); continue; } - if (current.StartsWith('"')) { + if (enumerator.Current.StartsWith('"')) { + var current = enumerator.Current; toFlush.Add(new ConstantToken(new StringVariant(current.Substring(1, current.Length - 2)))); endAndFlushId(); continue; } - if (bool.TryParse(current, out var resultB)) { + if (bool.TryParse(enumerator.Current, out var resultB)) { toFlush.Add(new ConstantToken(new BoolVariant(resultB))); endAndFlushId(); continue; } - idName += current; + idName += enumerator.Current; end(); @@ -323,7 +328,6 @@ void endAndFlushId() { finalTokens.Add(new(TokenType.Newline, baseIndent)); foreach (var t in finalTokens) yield return t; - } private static IEnumerable SanitizeInput(IEnumerable tokens) { From 199a4c2ca8eee80cfb2c09a7fa1a9e78c487c9e7 Mon Sep 17 00:00:00 2001 From: ai-kana Date: Thu, 7 Nov 2024 20:31:30 -0800 Subject: [PATCH 2/4] Update CodeGenerator.cs --- GDWeave.Dumper/CodeGenerator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GDWeave.Dumper/CodeGenerator.cs b/GDWeave.Dumper/CodeGenerator.cs index c0e4cbd..28a3b6a 100644 --- a/GDWeave.Dumper/CodeGenerator.cs +++ b/GDWeave.Dumper/CodeGenerator.cs @@ -6,7 +6,7 @@ public void Generate(StreamWriter writer) { var tabs = 0u; var gen = this.GenerateToken(token, ref tabs); - writer.Write(gen); + writer.Write(gen + ' '); for (var i = 0; i < tabs; i++) { writer.Write('\t'); From 26dba4af7ded1c747c6256928f760a95b72bf255 Mon Sep 17 00:00:00 2001 From: ai-kana Date: Thu, 7 Nov 2024 21:03:53 -0800 Subject: [PATCH 3/4] Fix negative literal edge cases --- GDWeave/Script/ScriptTokenizer.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/GDWeave/Script/ScriptTokenizer.cs b/GDWeave/Script/ScriptTokenizer.cs index 55b7303..014c741 100644 --- a/GDWeave/Script/ScriptTokenizer.cs +++ b/GDWeave/Script/ScriptTokenizer.cs @@ -100,7 +100,7 @@ public static class ScriptTokenizer { {"|=", TokenType.OpAssignBitOr}, {"^=", TokenType.OpAssignBitXor}, - {"+", TokenType.OpAnd}, + {"+", TokenType.OpAdd}, {"-", TokenType.OpSub}, {"*", TokenType.OpMul}, {"/", TokenType.OpDiv}, @@ -252,7 +252,10 @@ public static IEnumerable Tokenize(string gdScript, uint baseIndent = 0) var toFlush = new List(2); finalTokens.Add(new Token(TokenType.Newline, baseIndent)); var enumerator = tokens.GetEnumerator(); - while (enumerator.MoveNext()) { + var reparse = false; + while (reparse ? true : enumerator.MoveNext()) { + reparse = false; + if (enumerator.Current == "\n") { InsertNewLine(enumerator, baseIndent, toFlush); endAndFlushId(); @@ -274,10 +277,9 @@ public static IEnumerable Tokenize(string gdScript, uint baseIndent = 0) if (enumerator.Current == "-" || char.IsDigit(enumerator.Current[0])) { BuildNumber(enumerator, toFlush, out bool foundFull); + reparse = !foundFull; endAndFlushId(); - if (foundFull) { - continue; - } + continue; } if (BuiltinFunctions.Contains(enumerator.Current)) { From 468b5f517f5312a87a30347cc32ad50f15dee589 Mon Sep 17 00:00:00 2001 From: ai-kana Date: Thu, 7 Nov 2024 21:13:10 -0800 Subject: [PATCH 4/4] Remove extra space on newline --- GDWeave.Dumper/CodeGenerator.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/GDWeave.Dumper/CodeGenerator.cs b/GDWeave.Dumper/CodeGenerator.cs index 28a3b6a..cfeb7c3 100644 --- a/GDWeave.Dumper/CodeGenerator.cs +++ b/GDWeave.Dumper/CodeGenerator.cs @@ -2,11 +2,19 @@ public class CodeGenerator(List tokens, List identifiers) { public void Generate(StreamWriter writer) { + var onNewLine = false; foreach (var token in tokens) { var tabs = 0u; var gen = this.GenerateToken(token, ref tabs); - writer.Write(gen + ' '); + onNewLine = gen == "\n"; + + if (!onNewLine) + { + gen += ' '; + } + + writer.Write(gen); for (var i = 0; i < tabs; i++) { writer.Write('\t');