Skip to content

Commit

Permalink
editing ReadSelectExpressionLexes
Browse files Browse the repository at this point in the history
  • Loading branch information
@mk3008
mk3008 committed Nov 3, 2024
1 parent 6d6999c commit f0adba8
Show file tree
Hide file tree
Showing 12 changed files with 809 additions and 29 deletions.
10 changes: 10 additions & 0 deletions src/Carbunql.LexicalAnalyzer/Lex.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,16 @@ public Lex(ReadOnlyMemory<char> memory, LexType type, int position, int length)
cachedValue = null;
}

public Lex(ReadOnlyMemory<char> memory, LexType type, int position, int length, string value)
{
Memory = memory;
Type = type;
Position = position;
Length = length;
EndPosition = Position + Length;
cachedValue = value;
}

public Lex(ReadOnlyMemory<char> memory, LexType type, int position, int length, int endPosition)
{
Memory = memory;
Expand Down
282 changes: 282 additions & 0 deletions src/Carbunql.LexicalAnalyzer/LexPosition.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,282 @@
namespace Carbunql.LexicalAnalyzer;

public readonly struct LexPosition(ReadOnlyMemory<char> memory, int position, int length)
{
public ReadOnlyMemory<char> Memory { get; } = memory;

public int Position { get; } = position;

public int Length { get; } = length;

public int EndPosition => Position + Length;

public string Value => GetValue();

private readonly string? cachedValue;

private string GetValue()
{
if (cachedValue == null)
{
if (Position + Length > Memory.Length)
throw new ArgumentOutOfRangeException(nameof(Length), "Position and Length exceed memory bounds.");

return Memory.Slice(Position, Length).ToString();
}
return cachedValue;
}
}




//private static bool IsKeyword(ReadOnlySpan<char> sql, int position, string keyword)
//{
// // Check if the keyword matches at the given position
// return sql.Slice(position, keyword.Length).ToString().Equals(keyword, StringComparison.OrdinalIgnoreCase);
//}


//private static LexType FindLex(ReadOnlyMemory<char> memory, char firstChar, ref int pos)
//{
// if (char.IsDigit(firstChar))
// {
// }

// return LexType.Unknown; // Default return
//}
//if (pos < length)
//{
// char current = sql[pos];
// LexType lexType = current switch
// {
// ';' => LexType.QueryTerminator,
// ',' => LexType.ValueSeparator,
// '.' => LexType.IdentifierSeparator,
// '(' => LexType.LeftParen,
// ')' => LexType.RightParen,
// '[' => LexType.LeftSquareBracket,
// ']' => LexType.RightSquareBracket,
// _ => FindLex(memory, current, ref pos)
// };

// if (lexType == LexType.Unknown)
// {
// throw new InvalidOperationException($"Unexpected character '{current}' at position {pos}.");
// }

// return new Lex(memory, lexType, start, pos - start + 1); // Adjusted length calculation
//}


//public static IEnumerable<SqlToken> Tokenize(string sql)
//{
// return Tokenize(sql.AsMemory(), pos: 0, length: sql.Length, parenthesisLevel: 0, bracketLevel: 0);
//}

//internal static IEnumerable<SqlToken> Tokenize(ReadOnlyMemory<char> memory, int pos, int length, int parenthesisLevel, int bracketLevel)
//{
// var sql = memory.Span;
// int start;

// while (pos < length)
// {
// start = pos;
// char current = sql[pos];
// switch (current)
// {
// case ' ' or '\t' or '\n':
// pos++; // skip whitespace
// break;

// case '.':
// pos++;

// yield return new SqlToken(TokenType.Dot, memory.Slice(start, pos - start), start, pos, parenthesisLevel, bracketLevel, ".");
// break;

// case ',':
// pos++;

// yield return new SqlToken(TokenType.Comma, memory.Slice(start, pos - start), start, pos, parenthesisLevel, bracketLevel, "\'");
// break;

// case '(':
// pos++;

// yield return new SqlToken(TokenType.LeftParen, memory.Slice(start, pos - start), start, pos, parenthesisLevel, bracketLevel, "(");
// parenthesisLevel++;
// break;

// case ')':
// pos++;

// parenthesisLevel--;
// yield return new SqlToken(TokenType.RightParen, memory.Slice(start, pos - start), start, pos, parenthesisLevel, bracketLevel, ")");
// break;

// case '[':
// pos++;

// yield return new SqlToken(TokenType.LeftBracket, memory.Slice(start, pos - start), start, pos, parenthesisLevel, bracketLevel, "[");
// bracketLevel++;
// break;

// case ']':
// pos++;

// bracketLevel--;
// yield return new SqlToken(TokenType.RightBracket, memory.Slice(start, pos - start), start, pos, parenthesisLevel, bracketLevel, "]");
// break;

// case '/':
// if (TokenParser.TryParseHintCommentToken(memory, pos, length, parenthesisLevel, ref bracketLevel, out var startHint, out var hint, out var endHint))
// {
// pos = endHint.EndPosition;
// yield return startHint;
// yield return hint;
// yield return endHint;
// break;
// }

// if (TokenParser.TryParseBlockCommentToken(memory, pos, length, parenthesisLevel, ref bracketLevel, out var startBlockComment, out var blockComment, out var endBlockComment))
// {
// pos = endBlockComment.EndPosition;
// yield return startBlockComment;
// yield return blockComment;
// yield return endBlockComment;
// break;
// }

// if (TokenParser.TryParseSymbolOperatorToken(memory, pos, length, parenthesisLevel, ref bracketLevel, out var slashOperator))
// {
// pos = slashOperator.EndPosition;
// yield return slashOperator;
// break;
// }

// throw new InvalidOperationException();

// case '-':
// if (TokenParser.TryParseLineCommentToken(memory, pos, length, parenthesisLevel, ref bracketLevel, out var startLineComment, out var lineComment))
// {
// pos = lineComment.EndPosition;
// yield return startLineComment;
// yield return lineComment;
// break;
// }

// if (TokenParser.TryParseSymbolOperatorToken(memory, pos, length, parenthesisLevel, ref bracketLevel, out var minusOperator))
// {
// pos = minusOperator.EndPosition;
// yield return minusOperator;
// break;
// }

// throw new InvalidOperationException();

// case '\'':
// if (TokenParser.TryParseSingleQuoteIdentifierToken(memory, pos, length, parenthesisLevel, ref bracketLevel, out var singleQuotedIdentifer))
// {
// pos = singleQuotedIdentifer.EndPosition;
// yield return singleQuotedIdentifer;
// }

// throw new InvalidOperationException();

// case '"':
// if (TokenParser.TryParseDoubleQuoteIdentifierToken(memory, pos, length, parenthesisLevel, ref bracketLevel, out var doubleQuotedIdentifer))
// {
// pos = doubleQuotedIdentifer.EndPosition;
// yield return doubleQuotedIdentifer;
// }

// throw new InvalidOperationException();

// default:
// if (char.IsDigit(current))
// {
// if (TokenParser.TryParseNumericToken(memory, pos, length, parenthesisLevel, ref bracketLevel, out var numericToken))
// {
// pos = numericToken.EndPosition;
// yield return numericToken;
// }
// }
// else if (char.IsLetter(current))
// {
// var identifier = TokenParser.ReadLowercaseLetterIdentifier(memory, pos, length);

// if (TokenParser.TryParseSpecialValueToken(memory, ref pos, length, parenthesisLevel, ref bracketLevel, identifier, out var specialValueToken))
// {
// pos = specialValueToken.EndPosition;
// yield return specialValueToken;
// break;

// }

// if (TokenParser.TryParseTextOperatorToken(memory, pos, length, parenthesisLevel, ref bracketLevel, identifier, out var logicalToken))
// {
// pos = logicalToken.EndPosition;
// yield return logicalToken;
// break;
// }

// // type

// if (TokenParser.TryParseIntervalTypeToken(memory, ref pos, length, parenthesisLevel, ref bracketLevel, identifier, out var intervalToken))
// {
// pos = intervalToken.EndPosition;
// yield return intervalToken;
// break;
// }

// if (TokenParser.TryParseDateTimeTypeToken(memory, pos, length, parenthesisLevel, bracketLevel, identifier, out var datetimeTypeToken))
// {
// pos = datetimeTypeToken.EndPosition;
// yield return datetimeTypeToken;
// break;
// }


// // ddl command

// if (TokenParser.TryParseCreateIdentifierToken(memory, ref pos, length, parenthesisLevel, ref bracketLevel, identifier, out var createToken))
// {
// pos = createToken.EndPosition;
// yield return createToken;
// break;
// }

// if (TokenParser.TryParseAlterIdentifierToken(memory, ref pos, length, parenthesisLevel, ref bracketLevel, identifier, out var alterToken))
// {
// pos = alterToken.EndPosition;
// yield return alterToken;
// break;
// }

// if (TokenParser.TryParseDropIdentifierToken(memory, ref pos, length, parenthesisLevel, ref bracketLevel, identifier, out var dropToken))
// {
// pos = dropToken.EndPosition;
// yield return dropToken;
// break;
// }
// }
// else
// {
// // not letter and not digit
// if (TokenParser.TryParseSymbolOperatorToken(memory, pos, length, parenthesisLevel, ref bracketLevel, out var defaultOperator))
// {
// pos = defaultOperator.EndPosition;
// yield return defaultOperator;
// break;
// }
// }

// yield return new SqlToken(TokenType.Identifier, memory.Slice(start, pos - start), start, pos, parenthesisLevel, bracketLevel);
// break;
// }
// }
//}



5 changes: 5 additions & 0 deletions src/Carbunql.LexicalAnalyzer/LexType.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ public enum LexType : byte

QueryTerminator = 1,

As,

Operator,

//alter command
AlterSchema,
AlterTable,
Expand Down Expand Up @@ -51,6 +55,7 @@ public enum LexType : byte

WildCard,
Value,
SchemaOrTable,
SchemaOrTableOrColumn,
Column,
Function,
Expand Down
29 changes: 28 additions & 1 deletion src/Carbunql.LexicalAnalyzer/Lexer.Comment.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,28 @@ namespace Carbunql.LexicalAnalyzer;

public static partial class Lexer
{
internal static void SkipWhiteSpacesAndComment(ReadOnlyMemory<char> memory, ref int position)
{
SkipWhiteSpaces(memory, ref position);
SkipComment(memory, ref position);
SkipWhiteSpaces(memory, ref position);
}

internal static void SkipWhiteSpaces(ReadOnlyMemory<char> memory, ref int position)
{
var span = memory.Span;

while (position < span.Length && char.IsWhiteSpace(span[position]))
{
position++;
}
}

internal static void SkipComment(ReadOnlyMemory<char> memory, ref int position)
{
position = Math.Max(ParseUntilNonComment(memory, position).Last().EndPosition, position);
}

[MemberNotNullWhen(true)]
private static bool TryParseCommentStartLex(ReadOnlyMemory<char> memory, ref int position, out Lex lex)
{
Expand All @@ -19,7 +41,7 @@ private static bool TryParseCommentStartLex(ReadOnlyMemory<char> memory, ref int
/// <param name="memory">The string to be parsed.</param>
/// <param name="previous">The previous Lex indicating a non-comment state, or null if no previous state exists.</param>
/// <returns>An enumeration of Lexes after comments have been removed.</returns>
private static IEnumerable<Lex> ParseUntilNonComment(ReadOnlyMemory<char> memory, Lex? previous = null)
internal static IEnumerable<Lex> ParseUntilNonComment(ReadOnlyMemory<char> memory, Lex? previous = null)
{
// Invalid if the previous Lex is in a comment state
if (previous?.Type == LexType.LineCommentStart
Expand All @@ -33,6 +55,11 @@ private static IEnumerable<Lex> ParseUntilNonComment(ReadOnlyMemory<char> memory
// Start position is 0 if previous is null
int position = previous?.EndPosition ?? 0;

return ParseUntilNonComment(memory, position);
}

internal static IEnumerable<Lex> ParseUntilNonComment(ReadOnlyMemory<char> memory, int position)
{
while (true)
{
Lex lex;
Expand Down
Loading

0 comments on commit f0adba8

Please sign in to comment.