Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

More tokenizer fixes, code generator fix #20

Merged
merged 5 commits into from
Nov 8, 2024
Merged

More tokenizer fixes, code generator fix #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8fd3281
Fix improper token returns
ai-kana Nov 5, 2024
199a4c2
Update CodeGenerator.cs
ai-kana Nov 8, 2024
26dba4a
Fix negative literal edge cases
ai-kana Nov 8, 2024
468b5f5
Remove extra space on newline
ai-kana Nov 8, 2024
3b82b98
Merge branch 'NotNite:main' into main
ai-kana Nov 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions GDWeave.Dumper/CodeGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,18 @@

public class CodeGenerator(List<Token> tokens, List<string> identifiers) {
public void Generate(StreamWriter writer) {
var onNewLine = false;
foreach (var token in tokens) {
var tabs = 0u;
var gen = this.GenerateToken(token, ref tabs);

onNewLine = gen == "\n";

if (!onNewLine)
{
gen += ' ';
}

writer.Write(gen);

for (var i = 0; i < tabs; i++) {
Expand Down
38 changes: 22 additions & 16 deletions GDWeave/Script/ScriptTokenizer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ public static class ScriptTokenizer {
{"|=", TokenType.OpAssignBitOr},
{"^=", TokenType.OpAssignBitXor},

{"+", TokenType.OpAnd},
{"+", TokenType.OpAdd},
{"-", TokenType.OpSub},
{"*", TokenType.OpMul},
{"/", TokenType.OpDiv},
Expand Down Expand Up @@ -139,7 +139,7 @@ public static class ScriptTokenizer {
"*=",
"/=",
"%=",
"&=",
"&=",
"|=",
"^=",

Expand Down Expand Up @@ -205,7 +205,8 @@ private static void BuildIdentifierName(IEnumerator<string> enumerator, List<Tok
found = "_" + enumerator.Current;
}

private static void BuildNumber(IEnumerator<string> enumerator, List<Token> toFlush) {
private static void BuildNumber(IEnumerator<string> enumerator, List<Token> toFlush, out bool foundFull) {
foundFull = true;
int sign = 1;

if (enumerator.Current == "-") {
Expand All @@ -215,13 +216,15 @@ private static void BuildNumber(IEnumerator<string> enumerator, List<Token> toFl

if (!long.TryParse(enumerator.Current, out long upper)) {
toFlush.Add(new Token(TokenType.OpSub));
foundFull = false;
return;
}

if (!enumerator.MoveNext()) return;

if (enumerator.Current != ".") {
toFlush.Add(new ConstantToken(new IntVariant(upper * sign)));
foundFull = false;
return;
}

Expand Down Expand Up @@ -249,15 +252,17 @@ public static IEnumerable<Token> Tokenize(string gdScript, uint baseIndent = 0)
var toFlush = new List<Token>(2);
finalTokens.Add(new Token(TokenType.Newline, baseIndent));
var enumerator = tokens.GetEnumerator();
while (enumerator.MoveNext()) {
var current = enumerator.Current;
if (current == "\n") {
var reparse = false;
while (reparse ? true : enumerator.MoveNext()) {
reparse = false;

if (enumerator.Current == "\n") {
InsertNewLine(enumerator, baseIndent, toFlush);
endAndFlushId();
continue;
}

if (current == "_") {
if (enumerator.Current == "_") {
BuildIdentifierName(enumerator, toFlush, out string? found);
if (found == string.Empty) {
endAndFlushId();
Expand All @@ -270,37 +275,39 @@ public static IEnumerable<Token> Tokenize(string gdScript, uint baseIndent = 0)
continue;
}

if (current == "-" || char.IsDigit(current[0])) {
BuildNumber(enumerator, toFlush);
if (enumerator.Current == "-" || char.IsDigit(enumerator.Current[0])) {
BuildNumber(enumerator, toFlush, out bool foundFull);
reparse = !foundFull;
endAndFlushId();
continue;
}

if (BuiltinFunctions.Contains(current)) {
toFlush.Add(new Token(TokenType.BuiltInFunc, (uint?) BuiltinFunctions.IndexOf(current)));
if (BuiltinFunctions.Contains(enumerator.Current)) {
toFlush.Add(new Token(TokenType.BuiltInFunc, (uint?) BuiltinFunctions.IndexOf(enumerator.Current)));
endAndFlushId();
continue;
}

if (Tokens.TryGetValue(current, out var type)) {
if (Tokens.TryGetValue(enumerator.Current, out var type)) {
toFlush.Add(new Token(type));
endAndFlushId();
continue;
}

if (current.StartsWith('"')) {
if (enumerator.Current.StartsWith('"')) {
var current = enumerator.Current;
toFlush.Add(new ConstantToken(new StringVariant(current.Substring(1, current.Length - 2))));
endAndFlushId();
continue;
}

if (bool.TryParse(current, out var resultB)) {
if (bool.TryParse(enumerator.Current, out var resultB)) {
toFlush.Add(new ConstantToken(new BoolVariant(resultB)));
endAndFlushId();
continue;
}

idName += current;
idName += enumerator.Current;

end();

Expand All @@ -323,7 +330,6 @@ void endAndFlushId() {
finalTokens.Add(new(TokenType.Newline, baseIndent));

foreach (var t in finalTokens) yield return t;

}

private static IEnumerable<string> SanitizeInput(IEnumerable<string> tokens) {
Expand Down
Loading