diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index debad51..9e08316 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,21 +1,21 @@ name: Test on: - workflow_call: + workflow_call: jobs: - test: - name: Run Test Suite - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 + test: + name: Run Test Suite + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 - - name: Setup tooling - uses: ./.github/actions/setup + - name: Setup tooling + uses: ./.github/actions/setup - - name: Run Turbo build target - run: bun x turbo run build --cache-dir=node_modules/.cache/turbo + - name: Run Turbo build target + run: bun x turbo run build --cache-dir=node_modules/.cache/turbo - - name: Run Turbo test target - run: bun x turbo run test --cache-dir=node_modules/.cache/turbo + - name: Run Turbo test target + run: bun x turbo run test --cache-dir=node_modules/.cache/turbo diff --git a/.gitignore b/.gitignore index ceac576..120be7a 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,11 @@ build/ *.dylib .test-query.kql +# Generated grammar files +packages/kql-lezer/src/kql.grammar +packages/kql-lezer/src/parser.ts +packages/kql-lezer/src/parser.terms.ts + # Turbo .turbo diff --git a/AGENTS.md b/AGENTS.md index 836d92a..f1bb613 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,6 +3,7 @@ Instructions for AI agents on the Fossiq codebase, organized by priority. ## CRITICAL: System & Safety Rules + **Must be followed without exception:** - Never install global tools (`brew install`, `npm install -g`, etc.) without explicit user approval. @@ -20,19 +21,23 @@ Instructions for AI agents on the Fossiq codebase, organized by priority. - Never suppress, hide, or eliminate issues (e.g., silence warnings/errors, delete logs, modify configs to hide problems). Always ask user instead. ## HIGH: Communication & Output + - **C4 Rule (Most Important)**: All content creation and thoughts must be Clear, Concise, Correct, Complete, and Confident/Assertive. - Keep responses very concise; avoid redundancy. - No large summaries or excessive apologies. - Use markdown with code blocks, e.g., `path/to/file.ts#L1-10`. ## HIGH: Code Style & Architecture + ### Runtime & Tools + - Use Bun: `bun x` (not `npx`), `bun run` (not `npm run`). - TypeScript ESM; prefer functional programming and pure functions. - Use `$` for single-line shell operations in scripts; move conditionals to TypeScript. - Never assume library/spec behavior—always search official docs first (via WebSearch or context7 MCP). ### Code Quality + - Small, single-responsibility functions with descriptive names. - Keep files <150 lines; split large ones. - Organize related code in subdirectories. @@ -43,6 +48,7 @@ Instructions for AI agents on the Fossiq codebase, organized by priority. - Avoid barrel files (index.ts files that re-export everything); import directly from specific modules. ### Template Usage (Eta.js) + - **Whitespace Control**: Eta templates preserve all whitespace, including newlines. Use `<%- %>` to trim whitespace before/after tags for precise output control. - **Template Loading**: Load and compile templates once at module initialization; cache for performance. - **Section-Based Rendering**: For complex outputs, split into separate templates per section and join results in TypeScript to maintain control over separators. @@ -50,12 +56,15 @@ Instructions for AI agents on the Fossiq codebase, organized by priority. - **Separation of Concerns**: Keep logic in TypeScript; use templates only for string formatting and iteration. ### Architecture + - Monorepo with `packages/` workspaces. - Clear package boundaries; separate concerns. - Add features only when requested. ## HIGH: Debugging Context & Efficiency + Provide upfront context to minimize exploration: + - Exact file paths and relationships. - Git status/branch/SHAs. - Full error messages/stack traces. @@ -66,6 +75,7 @@ Provide upfront context to minimize exploration: **Avoid forcing discovery** of repo structure, branches, tests, dependencies, labels, or build steps. **Example context:** + ``` Working on between operator in kql-lezer. - Files: packages/kql-lezer/src/kql.grammar (L261-263), packages/kql-to-duckdb/src/translator.ts @@ -75,7 +85,9 @@ Working on between operator in kql-lezer. ``` ## HIGH: Development Workflow + ### GitHub Interactions + - Use `gh` CLI exclusively. - **Mandatory disclaimer** on all issues/PRs/comments: - Get username: `gh api user -q .login` @@ -87,23 +99,27 @@ Working on between operator in kql-lezer. - Forgetting this is a critical failure. ### GitHub Actions Debugging + 1. `gh run view ` 2. `gh run view --job=` 3. `gh run view --log-failed --job=` 4. Check workflow YAML and repo files as needed. ### Before Changes + - Always read files first. - Research facts upfront. - Limit fix attempts (1-2), then defer to user. ### Testing + - No testing during development; test only after completion if source changed. - Use `bun test`. ### Documentation (After Any Feature) + - Mark checklists complete. -Add discovered patterns/gotchas to guides. + Add discovered patterns/gotchas to guides. ## HIGH: MCP Servers @@ -121,38 +137,51 @@ Available MCP (Model Context Protocol) servers for enhanced functionality: Always prefer MCP servers over manual searches when available, especially for documentation (context7), linting (ESLint), and task management (taskmanager). ## MEDIUM: Tool Usage + - Limit file reads; pipe large outputs (`head`, `tail`, `rg`). - Never create standalone setup/explanation files or boilerplate unless asked. ## Package-Specific Guides + ### @fossiq/kql-lezer + - Purpose: Real-time KQL highlighting (no WASM). -- Key files: `src/kql.grammar`, `src/parser.ts` (generated), `src/index.ts`. -- Build: `lezer-generator src/kql.grammar -o src/parser.ts`. -- Status: 77 tests passing. +- Grammar sources: `src/grammar/` (TypeScript files defining tokens, rules, precedence) +- Generated files (DO NOT EDIT): `src/kql.grammar`, `src/parser.ts`, `src/parser.terms.ts` +- Grammar workflow: + 1. Edit TypeScript sources in `src/grammar/` (tokens, rules, plugins) + 2. Run `bun run build` - auto-generates grammar → parser → compiles TS + 3. Update `src/parser/cst-to-ast/` if adding new constructs + 4. Run `bun test` to verify +- Generated files are gitignored - always regenerated on build +- Status: 110 tests passing. ### @fossiq/kql-ast + - Purpose: Shared AST types. - Status: Core complete. ### @fossiq/ui + - Stack: SolidJS, Vite, PicoCSS, CodeMirror 6, DuckDB WASM, TanStack Table. - Gotchas: DuckDB files in `public/`; theme via DOM classes; grid truncation needs `min-width: 0`. - Status: Core complete (polishing). ## Monorepo Management + - Packages: `@fossiq/kebab-case`; internal deps `workspace:*`. - Adding packages: Create dir, `package.json`, copy `tsconfig.json`, minimal `src/index.ts`. - Versioning: `bun run changeset`, then `version`/`release`. - Issues: Add `agent` label; use prefixes (`[ui]`, etc.); include disclaimer. ## Quick Reference -| Task | Command | -|-------------------|--------------------------------------| -| Install deps | `bun install` | -| Build all | `bun run build` | -| Lint | `bun run lint` | -| Lint fix | `bun run lint:fix` | -| Test package | `cd packages/ && bun test` | -| Changeset | `bun run changeset` | -| UI dev | `cd packages/ui && bun run dev` | + +| Task | Command | +| ------------ | ------------------------------- | +| Install deps | `bun install` | +| Build all | `bun run build` | +| Lint | `bun run lint` | +| Lint fix | `bun run lint:fix` | +| Test package | `cd packages/ && bun test` | +| Changeset | `bun run changeset` | +| UI dev | `cd packages/ui && bun run dev` | diff --git a/packages/kql-ast/src/expressions.ts b/packages/kql-ast/src/expressions.ts index 7b78368..fd561d4 100644 --- a/packages/kql-ast/src/expressions.ts +++ b/packages/kql-ast/src/expressions.ts @@ -11,7 +11,8 @@ export type ExpressionType = | "Literal" | "ParenthesizedExpression" | "NumberLiteral" - | "StringLiteral"; + | "StringLiteral" + | "ArrayLiteral"; export interface BinaryExpression extends ASTNode { type: "BinaryExpression"; @@ -60,6 +61,11 @@ export interface ParenthesizedExpression extends ASTNode { expression: Expression; } +export interface ArrayLiteral extends ASTNode { + type: "ArrayLiteral"; + elements: Expression[]; +} + export type Expression = | BinaryExpression | UnaryExpression @@ -69,4 +75,5 @@ export type Expression = | ParenthesizedExpression | NumberLiteral | StringLiteral + | ArrayLiteral | ErrorNode; diff --git a/packages/kql-ast/src/index.ts b/packages/kql-ast/src/index.ts index 7a89ed4..e8bbaa6 100644 --- a/packages/kql-ast/src/index.ts +++ b/packages/kql-ast/src/index.ts @@ -9,6 +9,7 @@ export type { StringLiteral, UnaryExpression, ParenthesizedExpression, + ArrayLiteral, Expression, } from "./expressions"; export type { diff --git a/packages/kql-lezer/README.md b/packages/kql-lezer/README.md index 9c3d9e2..0bfe336 100644 --- a/packages/kql-lezer/README.md +++ b/packages/kql-lezer/README.md @@ -7,6 +7,7 @@ Pure JavaScript parser with no WASM dependencies, using the Lezer incremental pa ## Features ### Real-time Syntax Highlighting + - CodeMirror 6 language support - Incremental parsing for performance - Semantic token types (keywords, operators, literals, comments) @@ -14,11 +15,13 @@ Pure JavaScript parser with no WASM dependencies, using the Lezer incremental pa ### Full KQL Grammar Support **Query Structure** + - Let statements for variable binding - Pipeline expressions with table sources - Bracketed identifiers (`['column name']`) **Operators** + - `where` - filtering with logical/comparison expressions - `project` - column selection with aliases and expressions - `project-away`, `project-keep`, `project-rename`, `project-reorder` @@ -35,6 +38,7 @@ Pure JavaScript parser with no WASM dependencies, using the Lezer incremental pa - Plus: `parse`, `make-series`, `range`, `as`, `evaluate`, `render`, `partition`, `sample`, `serialize` **Expressions** + - Logical operators: `and`, `or`, `not` - Comparison operators: `==`, `!=`, `>`, `>=`, `<`, `<=` - String operators: `contains`, `startswith`, `endswith`, `has`, `matches`, `regex` (with negations and case-sensitive variants) @@ -44,6 +48,7 @@ Pure JavaScript parser with no WASM dependencies, using the Lezer incremental pa - Unary operators: `-`, `not` **Literals** + - Numbers (integer and decimal) - Strings (regular, verbatim `@"..."`, obfuscated `h"..."`) - Booleans: `true`, `false` @@ -53,11 +58,13 @@ Pure JavaScript parser with no WASM dependencies, using the Lezer incremental pa - GUID literals: `guid(xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx)` **Comments** + - Line comments (`// comment`) ### Complete AST Generation Converts Lezer's CST (Concrete Syntax Tree) to a typed AST compatible with `@fossiq/kql-ast`. Includes: + - Full operator support - Expression trees - Type preservation @@ -67,6 +74,7 @@ Converts Lezer's CST (Concrete Syntax Tree) to a typed AST compatible with `@fos ### Test Coverage **110 tests passing** covering: + - All operators and operator combinations - Expression parsing - Literal types @@ -88,9 +96,9 @@ import { parseKQL } from "@fossiq/kql-lezer"; const result = parseKQL("Events | where Level == 'Error' | take 10"); -console.log(result.ast); // Typed AST (Query object) -console.log(result.errors); // Parse errors (if any) -console.log(result.tokens); // Highlight tokens for syntax coloring +console.log(result.ast); // Typed AST (Query object) +console.log(result.errors); // Parse errors (if any) +console.log(result.tokens); // Highlight tokens for syntax coloring ``` ### CodeMirror Integration @@ -126,12 +134,13 @@ const tokens = extractHighlightTokens("Events | where Level == 'Error'"); ### Commands ```bash -# Build parser from grammar -bun run build:grammar - -# Compile TypeScript +# Full build (generates grammar, parser, and compiles TypeScript) bun run build +# Individual steps (usually not needed) +bun run generate:grammar # Generate .grammar from TypeScript sources +bun run build:parser # Generate parser.ts from .grammar file + # Run tests bun test @@ -141,11 +150,43 @@ bun run test:coverage ### Grammar Development -The grammar is defined in `src/kql.grammar` using Lezer syntax. After modifying: +**IMPORTANT**: The grammar is generated from TypeScript sources in `src/grammar/`, NOT edited directly. + +#### Workflow for Grammar Changes + +1. **Edit TypeScript grammar sources** in `src/grammar/`: -1. Run `bun run build:grammar` to generate `src/parser.ts` -2. Update CST-to-AST mappings in `src/parser/cst-to-ast/` if needed -3. Run tests to verify: `bun test` + - `tokens.ts` - Token definitions (keywords, operators, literals) + - `rules.ts` - Grammar rules and productions + - `precedence.ts` - Operator precedence + - `plugins/` - Modular grammar components + +2. **Build** to regenerate all files: + + ```bash + bun run build + ``` + + This automatically: + + - Generates `src/kql.grammar` from TypeScript sources + - Generates `src/parser.ts` from the grammar + - Compiles TypeScript to `dist/` + +3. **Update CST-to-AST mappings** in `src/parser/cst-to-ast/` if you added new grammar constructs + +4. **Run tests** to verify: + ```bash + bun test + ``` + +#### Generated Files (DO NOT EDIT) + +These files are generated during build and ignored by git: + +- `src/kql.grammar` - Generated Lezer grammar +- `src/parser.ts` - Generated parser +- `src/parser.terms.ts` - Generated term definitions ### Project Structure @@ -170,6 +211,7 @@ src/ ### Two-Stage Parsing 1. **Lezer Parser** → CST (Concrete Syntax Tree) + - Incremental parsing - Error recovery - Position tracking @@ -191,6 +233,7 @@ src/ ### Not Supported Source-modifying commands are out of scope: + - `.create`, `.alter`, `.drop` table/function definitions - `.update`, `.rename` operations - In-place data modifications diff --git a/packages/kql-lezer/package.json b/packages/kql-lezer/package.json index 2d7d3d7..98826dc 100644 --- a/packages/kql-lezer/package.json +++ b/packages/kql-lezer/package.json @@ -67,8 +67,10 @@ "typescript": "catalog:" }, "scripts": { - "build": "tsc", - "build:grammar": "bun x lezer-generator src/kql.grammar -o src/parser.ts", + "build": "bun run generate:grammar && bun run build:parser && bun run fix:parser && tsc", + "build:parser": "bun x lezer-generator src/kql.grammar -o src/parser.ts", + "generate:grammar": "bun scripts/generate-kql-grammar.ts", + "fix:parser": "bun scripts/fix-parser-types.ts", "test": "bun test tests", "test:coverage": "bun test tests --coverage", "ci:publish": "bun x npm@latest publish --ignore-scripts --provenance" diff --git a/packages/kql-lezer/src/grammar/plugins/rules/expressions.ts b/packages/kql-lezer/src/grammar/plugins/rules/expressions.ts index cb32409..1dba634 100644 --- a/packages/kql-lezer/src/grammar/plugins/rules/expressions.ts +++ b/packages/kql-lezer/src/grammar/plugins/rules/expressions.ts @@ -7,6 +7,7 @@ import { optional, separatedList, kw, + many, } from "@fossiq/lezer-grammar-generator"; /** @@ -22,29 +23,33 @@ export const expressionRules: Record = { expression: choice( seq(ref("OrExpression"), kw("or"), ref("AndExpression")), ref("AndExpression") - ) + ), }, AndExpression: { expression: choice( seq(ref("AndExpression"), kw("and"), ref("NotExpression")), ref("NotExpression") - ) + ), }, NotExpression: { expression: choice( seq(kw("not"), ref("NotExpression")), ref("ComparisonExpression") - ) + ), }, RangeExpression: { - expression: seq(ref("AdditiveExpression"), ref("RangeOp"), ref("AdditiveExpression")) + expression: seq( + ref("AdditiveExpression"), + ref("RangeOp"), + ref("AdditiveExpression") + ), }, RangeOp: { - expression: ref("RangeDoubleDot") + expression: ref("RangeDoubleDot"), }, ComparisonExpression: { @@ -55,36 +60,42 @@ export const expressionRules: Record = { }, GeneralComparisonOp: { - expression: choice( - ref("ComparisonOp"), - ref("StringOp"), - ref("BetweenOp") - ) + expression: choice(ref("ComparisonOp"), ref("StringOp"), ref("BetweenOp")), }, StringOp: { - expression: choice( - kw("contains"), ref("NotContains"), - ref("ContainsCs"), ref("NotContainsCs"), - kw("startswith"), ref("NotStartsWith"), - ref("StartsWithCs"), ref("NotStartsWithCs"), - kw("endswith"), ref("NotEndsWith"), - ref("EndsWithCs"), ref("NotEndsWithCs"), - kw("has"), ref("NotHas"), - ref("HasCs"), ref("NotHasCs"), - kw("hasprefix"), ref("NotHasPrefix"), - kw("hassuffix"), ref("NotHasSuffix"), - kw("in"), ref("NotIn"), - ref("InCs"), ref("NotInCs"), - kw("matches"), kw("regex") - ) + expression: choice( + kw("contains"), + ref("NotContains"), + ref("ContainsCs"), + ref("NotContainsCs"), + kw("startswith"), + ref("NotStartsWith"), + ref("StartsWithCs"), + ref("NotStartsWithCs"), + kw("endswith"), + ref("NotEndsWith"), + ref("EndsWithCs"), + ref("NotEndsWithCs"), + kw("has"), + ref("NotHas"), + ref("HasCs"), + ref("NotHasCs"), + kw("hasprefix"), + ref("NotHasPrefix"), + kw("hassuffix"), + ref("NotHasSuffix"), + kw("in"), + ref("NotIn"), + ref("InCs"), + ref("NotInCs"), + kw("matches"), + kw("regex") + ), }, BetweenOp: { - expression: choice( - kw("between"), - ref("NotBetween") - ) + expression: choice(kw("between"), ref("NotBetween")), }, AdditiveExpression: { @@ -111,9 +122,33 @@ export const expressionRules: Record = { PrimaryExpression: { expression: choice( + ref("ArrayLiteral"), seq(ref("OpenParen"), ref("Expression"), ref("CloseParen")), ref("FunctionCall"), - choice(ref("Identifier"), ref("BracketedIdentifier"), ref("Number"), ref("String"), kw("true"), kw("false"), kw("null")) + choice( + ref("Identifier"), + ref("BracketedIdentifier"), + ref("Number"), + ref("String"), + kw("true"), + kw("false"), + kw("null") + ) + ), + }, + + ArrayLiteral: { + expression: choice( + seq( + ref("OpenParen"), + ref("Expression"), + ref("Comma"), + ref("Expression"), + many(seq(ref("Comma"), ref("Expression"))), + ref("CloseParen") + ), + seq(ref("OpenParen"), ref("Expression"), ref("Comma"), ref("CloseParen")), + seq(ref("OpenParen"), ref("CloseParen")) ), }, @@ -129,4 +164,4 @@ export const expressionRules: Record = { ArgumentList: { expression: separatedList(ref("Expression"), ref("Comma"), { min: 1 }), }, -}; \ No newline at end of file +}; diff --git a/packages/kql-lezer/src/kql.grammar b/packages/kql-lezer/src/kql.grammar deleted file mode 100644 index bbbf0d7..0000000 --- a/packages/kql-lezer/src/kql.grammar +++ /dev/null @@ -1,135 +0,0 @@ -@tokens { - CloseBracket { "]" } - CloseParen { ")" } - Colon { ":" } - Comma { "," } - ComparisonOp { "==" | "!=" | ">=" | "<=" | ">" | "<" } - ContainsCs { "contains_cs" } - Dot { "." } - EndsWithCs { "endswith_cs" } - Equals { "=" } - HasCs { "has_cs" } - Identifier { $[A-Za-z_]$[A-Za-z0-9_]* } - InCs { "in~" } - LineComment { "//" ![\n]* } - MakeSeries { "make-series" } - Minus { "-" } - MvExpand { "mv-expand" } - NotBetween { "!between" } - NotContains { "!contains" } - NotContainsCs { "!contains_cs" } - NotEndsWith { "!endswith" } - NotEndsWithCs { "!endswith_cs" } - NotHas { "!has" } - NotHasCs { "!has_cs" } - NotHasPrefix { "!hasprefix" } - NotHasSuffix { "!hassuffix" } - NotIn { "!in" } - NotInCs { "!in~" } - NotStartsWith { "!startswith" } - NotStartsWithCs { "!startswith_cs" } - Number { @digit+("."@digit+)? } - OpenBracket { "[" } - OpenParen { "(" } - Percent { "%" } - Pipe { "|" } - Plus { "+" } - ProjectAway { "project-away" } - ProjectKeep { "project-keep" } - ProjectRename { "project-rename" } - ProjectReorder { "project-reorder" } - RangeDoubleDot { ".." } - Semicolon { ";" } - Slash { "/" } - Star { "*" } - StartsWithCs { "startswith_cs" } - String { "@" "\"" !["]* "\"" | "@" "'" ![']* "'" | "h" "\"" !["\n]* "\"" | "h" "@" "\"" !["]* "\"" | "\"" (!["\\] | "\\" _)* "\"" | "'" (!['\\] | "\\" _)* "'" } - whitespace { $[ \t\n\r]+ } - @precedence { LineComment, Slash, String, MakeSeries, MvExpand, ProjectAway, ProjectKeep, ProjectRename, ProjectReorder, NotBetween, NotContains, NotHas, NotIn, NotStartsWith, NotEndsWith, NotHasPrefix, NotHasSuffix, ContainsCs, NotContainsCs, StartsWithCs, NotStartsWithCs, EndsWithCs, NotEndsWithCs, HasCs, NotHasCs, InCs, NotInCs, Identifier } -} - - -@skip { whitespace | LineComment } - - -@top KQL { Query } - -Query { (LetStatement | SetStatement | DeclareQueryParametersStatement)* QueryExpression } -AdditiveExpression { AdditiveExpression (Plus | Minus) MultiplicativeExpression | MultiplicativeExpression } -AggregationItem { Identifier Equals FunctionCall | FunctionCall } -AggregationList { AggregationItem (Comma AggregationItem)* } -AndExpression { AndExpression @specialize[@name=and] NotExpression | NotExpression } -ArgumentList { Expression (Comma Expression)* } -AsClause { @specialize[@name=as] AsHint? Identifier } -AsHint { @specialize[@name=hint] Dot @specialize[@name=materialized] Equals @specialize[@name=true] | @specialize[@name=false] } -BetweenOp { @specialize[@name=between] | NotBetween } -BracketedIdentifier { OpenBracket String CloseBracket } -ComparisonExpression { AdditiveExpression (GeneralComparisonOp AdditiveExpression)? } -DatatableClause { @specialize[@name=datatable] OpenParen DatatableSchema CloseParen OpenBracket DatatableData CloseBracket } -DatatableColumnDef { Identifier Colon Identifier } -DatatableData { (LiteralValue (Comma LiteralValue)*)? } -DatatableSchema { DatatableColumnDef (Comma DatatableColumnDef)* } -DeclareQueryParametersStatement { @specialize[@name=declare] @specialize[@name=query_parameters] OpenParen QueryParameterList CloseParen Semicolon } -DistinctClause { @specialize[@name=distinct] ProjectExpressionList } -EvaluateClause { @specialize[@name=evaluate] FunctionCall } -Expression { OrExpression | RangeExpression } -ExtendClause { @specialize[@name=extend] ProjectExpressionList } -FindExpression { @specialize[@name=find] (Identifier | String | Pipe | OpenParen | CloseParen | @specialize[@name=in] | @specialize[@name=kind] | Equals)* } -FunctionCall { Identifier OpenParen ArgumentList? CloseParen } -GeneralComparisonOp { ComparisonOp | StringOp | BetweenOp } -GetSchemaClause { @specialize[@name=getschema] } -GroupByList { Expression (Comma Expression)* } -IdentifierList { Identifier (Comma Identifier)* } -JoinClause { @specialize[@name=join] JoinParameters? TableExpression @specialize[@name=on] JoinConditionList } -JoinConditionList { Expression (Comma Expression)* } -JoinKind { @specialize[@name=inner] | @specialize[@name=innerunique] | @specialize[@name=leftouter] | @specialize[@name=rightouter] | @specialize[@name=fullouter] | @specialize[@name=leftanti] | @specialize[@name=leftsemi] | @specialize[@name=rightanti] | @specialize[@name=rightsemi] } -JoinParameters { @specialize[@name=kind] Equals JoinKind } -LetStatement { @specialize[@name=let] Identifier Equals Expression Semicolon } -LimitClause { @specialize[@name=limit] Expression } -LiteralValue { Number | String | @specialize[@name=true] | @specialize[@name=false] | @specialize[@name=null] } -LookupClause { @specialize[@name=lookup] JoinParameters? TableExpression @specialize[@name=on] JoinConditionList } -MakeSeriesClause { MakeSeries AggregationList @specialize[@name=on] Expression (@specialize[@name=step] Expression)? (@specialize[@name=by] GroupByList)? } -MultiplicativeExpression { MultiplicativeExpression (Star | Slash | Percent) PrimaryExpression | PrimaryExpression } -MvExpandClause { MvExpand IdentifierList } -NotExpression { @specialize[@name=not] NotExpression | ComparisonExpression } -OrExpression { OrExpression @specialize[@name=or] AndExpression | AndExpression } -ParseClause { @specialize[@name=parse] ParseKind? Expression @specialize[@name=with] String } -ParseKind { @specialize[@name=kind] Equals @specialize[@name=simple] | @specialize[@name=kind] Equals @specialize[@name=regex] | @specialize[@name=kind] Equals @specialize[@name=relaxed] } -PartitionClause { @specialize[@name=partition] @specialize[@name=by] Identifier OpenParen PipelineExpression CloseParen } -PipelineExpression { TableExpression (Pipe TabularOperator)* } -PrimaryExpression { OpenParen Expression CloseParen | FunctionCall | Identifier | BracketedIdentifier | Number | String | @specialize[@name=true] | @specialize[@name=false] | @specialize[@name=null] } -PrintClause { @specialize[@name=print] ProjectExpressionList } -ProjectAwayClause { ProjectAway IdentifierList } -ProjectClause { @specialize[@name=project] ProjectExpressionList } -ProjectExpressionItem { Identifier Equals Expression | Expression } -ProjectExpressionList { ProjectExpressionItem (Comma ProjectExpressionItem)* } -ProjectKeepClause { ProjectKeep IdentifierList } -ProjectRenameClause { ProjectRename ProjectRenameList } -ProjectRenameItem { Identifier Equals Identifier } -ProjectRenameList { ProjectRenameItem (Comma ProjectRenameItem)* } -ProjectReorderClause { ProjectReorder IdentifierList } -QueryExpression { PipelineExpression | UnionExpression | SearchExpression | FindExpression } -QueryParameter { Identifier Colon Identifier (Equals Expression)? } -QueryParameterList { QueryParameter (Comma QueryParameter)* } -RangeClause { @specialize[@name=range] Identifier @specialize[@name=from] Expression @specialize[@name=to] Expression @specialize[@name=step] Expression } -RangeExpression { AdditiveExpression RangeOp AdditiveExpression } -RangeOp { RangeDoubleDot } -RenderClause { @specialize[@name=render] Identifier } -SampleClause { @specialize[@name=sample] Expression } -SearchExpression { @specialize[@name=search] (Identifier | String | Pipe | OpenParen | CloseParen | @specialize[@name=in] | @specialize[@name=kind] | Equals)* } -SerializeClause { @specialize[@name=serialize] IdentifierList? } -SetStatement { @specialize[@name=set] Identifier (Equals String | Number | Identifier | @specialize[@name=true] | @specialize[@name=false])? Semicolon } -SortClause { @specialize[@name=sort] | @specialize[@name=order] @specialize[@name=by]? SortExpressionList } -SortExpressionItem { Expression (@specialize[@name=asc] | @specialize[@name=desc])? (@specialize[@name=nulls] @specialize[@name=first] | @specialize[@name=last])? } -SortExpressionList { SortExpressionItem (Comma SortExpressionItem)* } -StringOp { @specialize[@name=contains] | NotContains | ContainsCs | NotContainsCs | @specialize[@name=startswith] | NotStartsWith | StartsWithCs | NotStartsWithCs | @specialize[@name=endswith] | NotEndsWith | EndsWithCs | NotEndsWithCs | @specialize[@name=has] | NotHas | HasCs | NotHasCs | @specialize[@name=hasprefix] | NotHasPrefix | @specialize[@name=hassuffix] | NotHasSuffix | @specialize[@name=in] | NotIn | InCs | NotInCs | @specialize[@name=matches] | @specialize[@name=regex] } -SummarizeClause { @specialize[@name=summarize] AggregationList (@specialize[@name=by] GroupByList)? } -TableExpression { RangeClause | Identifier | BracketedIdentifier | OpenParen PipelineExpression CloseParen } -TableList { TableExpression (Comma TableExpression)* } -TabularOperator { WhereClause | ProjectClause | ProjectAwayClause | ProjectKeepClause | ProjectRenameClause | ProjectReorderClause | ExtendClause | SortClause | LimitClause | TakeClause | TopClause | DistinctClause | SummarizeClause | MvExpandClause | UnionClause | JoinClause | LookupClause | ParseClause | DatatableClause | PrintClause | EvaluateClause | AsClause | MakeSeriesClause | PartitionClause | SampleClause | GetSchemaClause | RenderClause | SerializeClause | TableExpression | Number | String } -TakeClause { @specialize[@name=take] Expression } -TopClause { @specialize[@name=top] Expression @specialize[@name=by] SortExpressionList } -UnionClause { @specialize[@name=union] UnionParameters? TableList } -UnionExpression { @specialize[@name=union] UnionParameters? TableList } -UnionParameters { @specialize[@name=kind] Equals @specialize[@name=inner] | @specialize[@name=outer] | @specialize[@name=withsource] Equals Identifier } -WhereClause { @specialize[@name=where] Expression } diff --git a/packages/kql-lezer/src/parser.terms.ts b/packages/kql-lezer/src/parser.terms.ts deleted file mode 100644 index e162c8a..0000000 --- a/packages/kql-lezer/src/parser.terms.ts +++ /dev/null @@ -1,200 +0,0 @@ -// This file was generated by lezer-generator. You probably shouldn't edit it. -export const - LineComment = 1, - KQL = 2, - Query = 3, - LetStatement = 4, - Identifier = 5, - _let = 6, - Equals = 7, - Expression = 8, - OrExpression = 9, - or = 10, - AndExpression = 11, - and = 12, - NotExpression = 13, - not = 14, - ComparisonExpression = 15, - AdditiveExpression = 16, - Plus = 17, - Minus = 18, - MultiplicativeExpression = 19, - Star = 20, - Slash = 21, - Percent = 22, - PrimaryExpression = 23, - OpenParen = 24, - CloseParen = 25, - FunctionCall = 26, - ArgumentList = 27, - Comma = 28, - BracketedIdentifier = 29, - OpenBracket = 30, - String = 31, - CloseBracket = 32, - Number = 33, - _true = 34, - _false = 35, - _null = 36, - GeneralComparisonOp = 37, - ComparisonOp = 38, - StringOp = 39, - contains = 40, - NotContains = 41, - ContainsCs = 42, - NotContainsCs = 43, - startswith = 44, - NotStartsWith = 45, - StartsWithCs = 46, - NotStartsWithCs = 47, - endswith = 48, - NotEndsWith = 49, - EndsWithCs = 50, - NotEndsWithCs = 51, - has = 52, - NotHas = 53, - HasCs = 54, - NotHasCs = 55, - hasprefix = 56, - NotHasPrefix = 57, - hassuffix = 58, - NotHasSuffix = 59, - _in = 60, - NotIn = 61, - InCs = 62, - NotInCs = 63, - matches = 64, - regex = 65, - BetweenOp = 66, - between = 67, - NotBetween = 68, - RangeExpression = 69, - RangeOp = 70, - RangeDoubleDot = 71, - Semicolon = 72, - SetStatement = 73, - set = 74, - DeclareQueryParametersStatement = 75, - declare = 76, - query_parameters = 77, - QueryParameterList = 78, - QueryParameter = 79, - Colon = 80, - QueryExpression = 81, - PipelineExpression = 82, - TableExpression = 83, - RangeClause = 84, - range = 85, - from = 86, - to = 87, - step = 88, - Pipe = 89, - TabularOperator = 90, - WhereClause = 91, - where = 92, - ProjectClause = 93, - project = 94, - ProjectExpressionList = 95, - ProjectExpressionItem = 96, - ProjectAwayClause = 97, - ProjectAway = 98, - IdentifierList = 99, - ProjectKeepClause = 100, - ProjectKeep = 101, - ProjectRenameClause = 102, - ProjectRename = 103, - ProjectRenameList = 104, - ProjectRenameItem = 105, - ProjectReorderClause = 106, - ProjectReorder = 107, - ExtendClause = 108, - extend = 109, - SortClause = 110, - sort = 111, - order = 112, - by = 113, - SortExpressionList = 114, - SortExpressionItem = 115, - asc = 116, - desc = 117, - nulls = 118, - first = 119, - last = 120, - LimitClause = 121, - limit = 122, - TakeClause = 123, - take = 124, - TopClause = 125, - top = 126, - DistinctClause = 127, - distinct = 128, - SummarizeClause = 129, - summarize = 130, - AggregationList = 131, - AggregationItem = 132, - GroupByList = 133, - MvExpandClause = 134, - MvExpand = 135, - UnionClause = 136, - union = 137, - UnionParameters = 138, - kind = 139, - inner = 140, - outer = 141, - withsource = 142, - TableList = 143, - JoinClause = 144, - join = 145, - JoinParameters = 146, - JoinKind = 147, - innerunique = 148, - leftouter = 149, - rightouter = 150, - fullouter = 151, - leftanti = 152, - leftsemi = 153, - rightanti = 154, - rightsemi = 155, - on = 156, - JoinConditionList = 157, - LookupClause = 158, - lookup = 159, - ParseClause = 160, - parse = 161, - ParseKind = 162, - simple = 163, - relaxed = 164, - _with = 165, - DatatableClause = 166, - datatable = 167, - DatatableSchema = 168, - DatatableColumnDef = 169, - DatatableData = 170, - LiteralValue = 171, - PrintClause = 172, - print = 173, - EvaluateClause = 174, - evaluate = 175, - AsClause = 176, - as = 177, - AsHint = 178, - hint = 179, - Dot = 180, - materialized = 181, - MakeSeriesClause = 182, - MakeSeries = 183, - PartitionClause = 184, - partition = 185, - SampleClause = 186, - sample = 187, - GetSchemaClause = 188, - getschema = 189, - RenderClause = 190, - render = 191, - SerializeClause = 192, - serialize = 193, - UnionExpression = 194, - SearchExpression = 195, - search = 196, - FindExpression = 197, - find = 198 diff --git a/packages/kql-lezer/src/parser.ts b/packages/kql-lezer/src/parser.ts deleted file mode 100644 index ded51f0..0000000 --- a/packages/kql-lezer/src/parser.ts +++ /dev/null @@ -1,18 +0,0 @@ -// This file was generated by lezer-generator. You probably shouldn't edit it. -import {LRParser} from "@lezer/lr" -const spec_Identifier: Record = {__proto__:null,let:12, or:20, and:24, not:28, true:68, false:70, null:72, contains:80, startswith:88, endswith:96, has:104, hasprefix:112, hassuffix:116, in:120, matches:128, regex:130, between:134, set:148, declare:152, query_parameters:154, range:170, from:172, to:174, step:176, where:184, project:188, extend:218, sort:222, order:224, by:226, asc:232, desc:234, nulls:236, first:238, last:240, limit:244, take:248, top:252, distinct:256, summarize:260, union:274, kind:278, inner:280, outer:282, withsource:284, join:290, innerunique:296, leftouter:298, rightouter:300, fullouter:302, leftanti:304, leftsemi:306, rightanti:308, rightsemi:310, on:312, lookup:318, parse:322, simple:326, relaxed:328, with:330, datatable:334, print:346, evaluate:350, as:354, hint:358, materialized:362, partition:370, sample:374, getschema:378, render:382, serialize:386, search:392, find:396} -export const parser = LRParser.deserialize({ - version: 14, - states: "GlOYQPOOOzQPO'#CyO!PQPO'#C`O!UQPO'#DwO!ZQPO'#DyOOQO'#Gk'#GkOYQPO'#C_O!`QPO'#ESOOQO'#ER'#ERO!eQPO'#ERO!sQPO'#EQO#OQPO'#GfO$RQPO'#GgO$YQPO'#GiOOQO'#EP'#EPOOQO'#C_'#C_QOQPOOO$aQPO,59eO$fQPO,58zO$kQPO,5:cO%PQPO,5:eOOQO-E:i-E:iOOQO,58y,58yO%UQPO,5:nO%ZQPO,5:mO%`QPO'#GnO'oQPO,5:lO'zQPO'#F[OOQO'#F['#F[O(PQPO'#F[O(UQPO'#FaOOQO,5=Q,5=QO!eQPO,5=QOOQO'#Gw'#GwO(dQPO,5=RO(kQPO,5=TOOQO1G/P1G/PO(rQPO1G.fOOQO1G/}1G/}O)aQPO1G/}O)fQPO1G/}O)kQPO1G0PO(rQPO1G0YOOQO1G0X1G0XO(rQPO'#EZO)pQPO'#E]O*_QPO'#EaO*_QPO'#EdO*dQPO'#EfO*_QPO'#EjO)pQPO'#ElOOQO'#En'#EnO*iQPO'#EnO(rQPO'#EyO(rQPO'#E{O(rQPO'#E}O)pQPO'#FPO*pQPO'#FRO*_QPO'#FWO#OQPO'#FYO*uQPO'#FbO*uQPO'#FpO+WQPO'#FrO+_QPO'#FxO)pQPO'#GOO+dQPO'#GQO+iQPO'#GSO*pQPO'#GYO+tQPO'#G[O(rQPO'#G^OOQO'#G`'#G`O+yQPO'#GbO,OQPO'#GdOOQO'#EY'#EYOOQO,5=Y,5=YOOQO-E:l-E:lO,^QPO,5;vO,cQPO,5;vO!eQPO'#GtO,hQPO,5;{OOQO1G2l1G2lOOQO-E:u-E:uO/lQPO'#CsO(rQPO'#CsOOQO'#Cs'#CsO0`QPO'#ClOOQO'#Co'#CoO6vQPO'#CkO(rQPO'#CiOOQO'#Ci'#CiO6}QPO'#CeOOQO'#Cg'#CgO8RQPO'#CdOOQO'#Cd'#CdO9SQPO7+$QO9XQPO7+%iOOQO7+%i7+%iO9^QPO'#D}O9cQPO'#D|O9kQPO7+%kO9pQPO7+%tOOQO,5:u,5:uO9uQPO'#CsOOQO'#E`'#E`O9|QPO'#E_OOQO,5:w,5:wO:[QPO'#EcOOQO,5:{,5:{OOQO,5;O,5;OO:jQPO'#EiO:oQPO'#EhOOQO,5;Q,5;QOOQO,5;U,5;UOOQO,5;W,5;WO:}QPO'#EsO;iQPO'#ErOOQO,5;Y,5;YO(rQPO,5;YOOQO,5;e,5;eOOQO,5;g,5;gO;wQPO,5;iOOQO,5;k,5;kO;|QPO'#CvOOQO'#FU'#FUOTQPO,59_O>YQPO,59ZO>YQPO,59WOOQO'#DT'#DTOOQO'#Dp'#DpOOQO'#DR'#DRO>YQPO,59VOOQO'#Dt'#DtO>YQPO,5:_O3qQPO'#CkOOQO,59T,59TO(rQPO,59RO(rQPO,59POOQO<tQPO,5:iO)kQPO'#GmO>yQPO,5:hO?RQPO<qAN>qOJtQPOAN>zOOQO1G0f1G0fOOQO,5=Z,5=ZOOQO-E:m-E:mOOQO,5=[,5=[OOQO-E:n-E:nOOQO1G0o1G0oOOQO,5=],5=]OOQO-E:o-E:oOOQO1G0y1G0yOJyQPO1G0yOOQO,5=^,5=^OOQO-E:p-E:pOOQO7+&o7+&oOOQO1G1[1G1[OOQO,5=_,5=_OOQO-E:q-E:qOKOQPO'#FVOOQO7+&s7+&sOOQO'#Fe'#FeOOQO1G1j1G1jOK^QPO'#FoOOQO7+'S7+'SO(rQPO7+'SOOQO7+'b7+'bO(rQPO7+'bOOQO1G1z1G1zOOQO7+'d7+'dOKlQPO7+'dOKqQPO,5Q#T#o4a~>VUT~!Q![4a!c!}4a#R#S4a#T#V4a#V#W>i#W#o4a~>nUT~!Q![4a!c!}4a#R#S4a#T#g4a#g#h?Q#h#o4a~?XS!S~T~!Q![4a!c!}4a#R#S4a#T#o4a~?jVT~rs@P!Q![4a!b!c@l!c!}4a#R#S4a#T#U@r#U#o4a~@SUOY@PZr@Prs-qs;'S@P;'S;=`@f<%lO@P~@iP;=`<%l@P~@oPrs3_~@wUT~!Q![4a!c!}4a#R#S4a#T#g4a#g#hAZ#h#o4a~A`ST~!Q![4a!c!}4a#R#SAl#T#o4a~AqUT~!Q![4a!c!}4a#R#S4a#T#V4a#V#WBT#W#o4a~BYUT~!Q![4a!c!}4a#R#S4a#T#g4a#g#hBl#h#o4a~BsS!W~T~!Q![4a!c!}4a#R#S4a#T#o4a~CUUT~!Q![4a!c!}4a#R#S4a#T#b4a#b#cCh#c#o4a~CmTT~!Q![4a!c!}4a#R#S4a#T#o4a#r#sC|~DRO!`~~DWVT~!Q![4a!c!}4a#R#S4a#T#UDm#U#j4a#j#kF|#k#o4a~DrUT~!Q![4a!c!}4a#R#S4a#T#_4a#_#`EU#`#o4a~EZUT~!Q![4a!c!}4a#R#S4a#T#X4a#X#YEm#Y#o4a~ErTT~}!OFR!Q![4a!c!}4a#R#S4a#T#o4a~FUP#g#hFX~F[P#X#YF_~FbP#f#gFe~FhP#]#^Fk~FnP#X#YFq~FtP#g#hFw~F|O$}~~GRTT~}!OGb!Q![4a!c!}4a#R#S4a#T#o4a~GeP#X#YGh~GkP#l#mGn~GqP#d#eGt~GwP#T#UGz~G}P#b#cHQ~HTP#W#XHW~H]O#{~~HbUT~!Q![4a!c!}4a#R#S4a#T#f4a#f#gHt#g#o4a~HyUT~!Q![4a!c!}4a#R#S4a#T#c4a#c#dI]#d#o4a~IbUT~!Q![4a!c!}4a#R#S4a#T#^4a#^#_It#_#o4a~IyUT~!Q![4a!c!}4a#R#S4a#T#X4a#X#YJ]#Y#o4a~JbUT~!Q![4a!c!}4a#R#S4a#T#V4a#V#WJt#W#o4a~JyUT~!Q![4a!c!}4a#R#S4a#T#h4a#h#iK]#i#o4a~KbTT~}!OKq!Q![4a!c!}4a#R#S4a#T#o4a~KtR#T#UK}#_#`Lf#f#gL}~LQP#k#lLT~LWP#T#ULZ~L^P#m#nLa~LfO#U~~LiP#X#YLl~LoP#X#YLr~LuP#d#eLx~L}O#X~~MQP#X#YMT~MWQ#b#cM^#c#dMu~MaP#T#UMd~MgP#a#bMj~MmP#X#YMp~MuO#Z~~MxP#f#gM{~NOP#W#XNR~NUP#X#YNX~N[P#f#gN_~NdO#_~~NiUT~!Q![4a!c!}4a#R#S4a#T#h4a#h#iN{#i#o4a~! QTT~!Q![4a!c!}4a#R#S4a#T#U! a#U#o4a~! fUT~!Q![4a!c!}4a#R#S4a#T#f4a#f#g! x#g#o4a~! }UT~!Q![4a!c!}4a#R#S4a#T#h4a#h#i!!a#i#o4a~!!fUT~!Q![4a!c!}4a#R#S4a#T#g4a#g#h!!x#h#o4a~!!}UT~!Q![4a!c!}4a#R#S4a#T#k4a#k#l!#a#l#o4a~!#fUT~!Q![4a!c!}4a#R#S4a#T#]4a#]#^!#x#^#o4a~!#}UT~!Q![4a!c!}4a#R#S4a#T#h4a#h#i!$a#i#o4a~!$fUT~!Q![4a!c!}4a#R#S4a#T#[4a#[#]!$x#]#o4a~!$}ST~!Q![4a!c!}4a#R#S!%Z#T#o4a~!%`UT~!Q![4a!c!}4a#R#S4a#T#V4a#V#W!%r#W#o4a~!%wUT~!Q![4a!c!}4a#R#S4a#T#g4a#g#h!&Z#h#o4a~!&bS!O~T~!Q![4a!c!}4a#R#S4a#T#o4a~!&sO!{~", - tokenizers: [0, 1], - topRules: {"KQL":[0,2]}, - specialized: [{term: 5, get: (value: string) => spec_Identifier[value] ?? -1}], - tokenPrec: 2110 -}) diff --git a/packages/kql-lezer/src/parser/cst-to-ast/primitives.ts b/packages/kql-lezer/src/parser/cst-to-ast/primitives.ts index c4366b7..d7d1d7a 100644 --- a/packages/kql-lezer/src/parser/cst-to-ast/primitives.ts +++ b/packages/kql-lezer/src/parser/cst-to-ast/primitives.ts @@ -58,11 +58,36 @@ export function mapPrimitive( end: node.to, }; } + case "ArrayLiteral": + return mapArrayLiteral(node, ctx); default: return createErrorNode(node, `Unknown primitive: ${node.type.name}`); } } +/** + * Map an ArrayLiteral node. + */ +export function mapArrayLiteral( + node: SyntaxNode, + ctx: MapperContext +): AST.ArrayLiteral | AST.ErrorNode { + const { getChildren } = ctx; + + const elements: AST.Expression[] = []; + const exprs = getChildren(node, "Expression"); + for (const expr of exprs) { + elements.push(ctx.mapScalarExpression(expr)); + } + + return { + type: "ArrayLiteral", + elements, + start: node.from, + end: node.to, + }; +} + /** * Map a FunctionCall node. */ diff --git a/packages/kql-to-duckdb/package.json b/packages/kql-to-duckdb/package.json index 7f17854..a1ed2a5 100644 --- a/packages/kql-to-duckdb/package.json +++ b/packages/kql-to-duckdb/package.json @@ -13,8 +13,8 @@ }, "scripts": { "build": "tsc", - "test": "echo \"No tests specified\"", - "test:coverage": "echo \"No tests specified\"", + "test": "bun test", + "test:coverage": "bun test --coverage", "ci:publish": "bun x npm@latest publish --ignore-scripts --provenance" }, "dependencies": { diff --git a/packages/kql-to-duckdb/src/translator/expressions/index.ts b/packages/kql-to-duckdb/src/translator/expressions/index.ts index 1cd5d01..b246af1 100644 --- a/packages/kql-to-duckdb/src/translator/expressions/index.ts +++ b/packages/kql-to-duckdb/src/translator/expressions/index.ts @@ -8,6 +8,7 @@ import type { Literal, ParenthesizedExpression, UnaryExpression, + ArrayLiteral, } from "@fossiq/kql-ast"; export function translateExpression(expr: Expression): string { @@ -32,6 +33,8 @@ export function translateExpression(expr: Expression): string { const unaryExpr = expr as UnaryExpression; return `${unaryExpr.operator} ${translateExpression(unaryExpr.operand)}`; } + case "ArrayLiteral": + return translateArrayLiteral(expr as ArrayLiteral); default: throw new Error( `Unsupported expression type: ${ @@ -41,6 +44,11 @@ export function translateExpression(expr: Expression): string { } } +function translateArrayLiteral(expr: ArrayLiteral): string { + const elements = expr.elements.map(translateExpression).join(", "); + return `(${elements})`; +} + function translateLiteral(expr: Literal): string { if (expr.value === null) return "NULL"; if (typeof expr.value === "boolean") return expr.value ? "TRUE" : "FALSE"; @@ -59,18 +67,36 @@ function translateBinaryExpression(expr: BinaryExpression): string { ? `'%${(expr.right as StringLiteral).value}%'` : `'%' || ${translateExpression(expr.right)} || '%'`; return `${left} LIKE ${right}`; + } else if (operator === "!contains") { + const right = + expr.right.type === "StringLiteral" + ? `'%${(expr.right as StringLiteral).value}%'` + : `'%' || ${translateExpression(expr.right)} || '%'`; + return `${left} NOT LIKE ${right}`; } else if (operator === "startswith") { const right = expr.right.type === "StringLiteral" ? `'${(expr.right as StringLiteral).value}%'` : `${translateExpression(expr.right)} || '%'`; return `${left} LIKE ${right}`; + } else if (operator === "!startswith") { + const right = + expr.right.type === "StringLiteral" + ? `'${(expr.right as StringLiteral).value}%'` + : `${translateExpression(expr.right)} || '%'`; + return `${left} NOT LIKE ${right}`; } else if (operator === "endswith") { const right = expr.right.type === "StringLiteral" ? `'%${(expr.right as StringLiteral).value}'` : `'%' || ${translateExpression(expr.right)}`; return `${left} LIKE ${right}`; + } else if (operator === "!endswith") { + const right = + expr.right.type === "StringLiteral" + ? `'%${(expr.right as StringLiteral).value}'` + : `'%' || ${translateExpression(expr.right)}`; + return `${left} NOT LIKE ${right}`; } else if (operator === "has") { // 'has' in KQL means word boundary match - approximate with REGEXP const right = @@ -78,6 +104,18 @@ function translateBinaryExpression(expr: BinaryExpression): string { ? `'\\b${(expr.right as StringLiteral).value}\\b'` : translateExpression(expr.right); return `${left} REGEXP ${right}`; + } else if (operator === "!has") { + const right = + expr.right.type === "StringLiteral" + ? `'\\b${(expr.right as StringLiteral).value}\\b'` + : translateExpression(expr.right); + return `${left} NOT REGEXP ${right}`; + } else if (operator === "in") { + const right = translateExpression(expr.right); + return `${left} IN ${right}`; + } else if (operator === "!in") { + const right = translateExpression(expr.right); + return `${left} NOT IN ${right}`; } // Default handling for other operators diff --git a/packages/kql-to-duckdb/tests/string-operators.test.ts b/packages/kql-to-duckdb/tests/string-operators.test.ts new file mode 100644 index 0000000..423e064 --- /dev/null +++ b/packages/kql-to-duckdb/tests/string-operators.test.ts @@ -0,0 +1,76 @@ +import { describe, test, expect } from "bun:test"; +import { kqlToDuckDB } from "../src/index"; + +describe("String operators", () => { + test("contains", () => { + expect(kqlToDuckDB('Users | where name contains "test"')).toContain( + "name LIKE '%test%'" + ); + }); + + test("!contains", () => { + expect(kqlToDuckDB('Users | where name !contains "test"')).toContain( + "name NOT LIKE '%test%'" + ); + }); + + test("startswith", () => { + expect(kqlToDuckDB('Users | where name startswith "j"')).toContain( + "name LIKE 'j%'" + ); + }); + + test("!startswith", () => { + expect(kqlToDuckDB('Users | where name !startswith "j"')).toContain( + "name NOT LIKE 'j%'" + ); + }); + + test("endswith", () => { + expect(kqlToDuckDB('Users | where name endswith "com"')).toContain( + "name LIKE '%com'" + ); + }); + + test("!endswith", () => { + expect(kqlToDuckDB('Users | where name !endswith "com"')).toContain( + "name NOT LIKE '%com'" + ); + }); + + test("has", () => { + expect(kqlToDuckDB('Users | where name has "word"')).toContain( + "name REGEXP '\\bword\\b'" + ); + }); + + test("!has", () => { + expect(kqlToDuckDB('Users | where name !has "word"')).toContain( + "name NOT REGEXP '\\bword\\b'" + ); + }); + + test("in", () => { + expect(kqlToDuckDB("Users | where age in (18, 21, 25)")).toContain( + "age IN (18, 21, 25)" + ); + }); + + test("!in", () => { + expect(kqlToDuckDB("Users | where age !in (18, 21, 25)")).toContain( + "age NOT IN (18, 21, 25)" + ); + }); + + test("in with strings", () => { + expect(kqlToDuckDB('Users | where name in ("Alice", "Bob")')).toContain( + "name IN ('Alice', 'Bob')" + ); + }); + + test("!in with strings", () => { + expect(kqlToDuckDB('Users | where name !in ("Alice", "Bob")')).toContain( + "name NOT IN ('Alice', 'Bob')" + ); + }); +}); diff --git a/scripts/publish-github.ts b/scripts/publish-github.ts index 8bb5bfa..c7461c3 100755 --- a/scripts/publish-github.ts +++ b/scripts/publish-github.ts @@ -60,18 +60,29 @@ async function publishToGitHub() { console.log(`✅ @fossiq/${pkg} published to GitHub successfully`); } catch (error) { // Check if error is due to version already existing + // Bun's ShellError has stderr property that contains npm's error output const errorStr = error?.toString() || ""; + const stderr = (error as any)?.stderr?.toString() || ""; + const combinedError = errorStr + " " + stderr; + if ( - errorStr.includes("EPUBLISHCONFLICT") || - errorStr.includes("cannot publish over") || - errorStr.includes("previously published versions") + combinedError.includes("EPUBLISHCONFLICT") || + combinedError.includes("cannot publish over") || + combinedError.includes("previously published versions") || + combinedError.includes( + "You cannot publish over the previously published versions" + ) ) { console.log( `⚠️ @fossiq/${pkg} version already exists in registry, skipping` ); continue; } - console.error(`❌ Failed to publish @fossiq/${pkg} to GitHub:`, error); + console.error(`❌ Failed to publish @fossiq/${pkg} to GitHub:`); + console.error(`Error: ${errorStr}`); + if (stderr) { + console.error(`stderr: ${stderr}`); + } process.exit(1); } } diff --git a/turbo.json b/turbo.json index 5579b01..edf213d 100644 --- a/turbo.json +++ b/turbo.json @@ -10,22 +10,26 @@ "grammar.js", "bindings/**", "prebuilds/**", + "packages/kql-lezer/src/kql.grammar", + "packages/kql-lezer/src/parser.ts", + "packages/kql-lezer/src/parser.terms.ts", "!**/node_modules/**" ], - "inputs": ["src/**", "package.json", "tsconfig.json"] + "inputs": ["src/**", "package.json", "tsconfig.json", "turbo.json"] }, "test": { "dependsOn": [], "outputs": [], - "inputs": ["src/**", "tests/**"] + "inputs": ["src/**", "tests/**", "turbo.json"] }, "test:coverage": { "dependsOn": [], "outputs": ["coverage/**"], - "inputs": ["src/**", "tests/**"] + "inputs": ["src/**", "tests/**", "turbo.json"] }, "lint": { - "outputs": [] + "outputs": [], + "inputs": ["src/**", "turbo.json"] }, "ci:publish": { "outputs": [],