From bed5d1a2ced041272fb93673ee05abead10cb836 Mon Sep 17 00:00:00 2001 From: asrar Date: Wed, 11 Feb 2026 23:54:50 +0100 Subject: [PATCH] feat: add custom Malloy code editor mode with mode=code support --- README.md | 1 + e2e-tests/malloy-editor.spec.ts | 236 ++++++++++++++++++++++++++++++++ package-lock.json | 2 + package.json | 1 + src/SourceExplorer.tsx | 156 ++++++++++++++------- src/helpers.ts | 2 +- src/llms-txt/generator.ts | 143 ++++++++++++++++--- src/monaco-setup.ts | 19 +++ src/routeType.ts | 4 +- src/routing.tsx | 21 ++- 10 files changed, 511 insertions(+), 74 deletions(-) create mode 100644 e2e-tests/malloy-editor.spec.ts create mode 100644 src/monaco-setup.ts diff --git a/README.md b/README.md index 47a7730..944c020 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ Control the UI and behavior using URL query parameters: - **`query`** - Malloy query string (URL-encoded) - **`run=true`** - Auto-execute the query on page load +- **`mode=code`** - Use code editor instead of structured query builder. Auto-inferred for queries with custom expressions, but can be set explicitly to always use the code editor. - **`includeTopValues=true`** - Load top 10 values for field autocomplete (slower) - **`showQueryPanel=true`** - Expand query editor panel - **`showSourcePanel=true`** - Expand source/schema panel diff --git a/e2e-tests/malloy-editor.spec.ts b/e2e-tests/malloy-editor.spec.ts new file mode 100644 index 0000000..9b2f168 --- /dev/null +++ b/e2e-tests/malloy-editor.spec.ts @@ -0,0 +1,236 @@ +import { test, expect } from "@playwright/test"; + +test.describe("Malloy editor mode", () => { + test("Custom malloy query opens in editor mode and shows results", async ({ + page, + }) => { + // A query with a custom aggregate expression can't be represented as a + // structured explorer query, so it falls back to editor mode + const customQuery = + "run: invoices -> { group_by: status; aggregate: double_count is count() * 2 }"; + + await page.goto( + `./#/model/invoices/explorer/invoices?query=${encodeURIComponent(customQuery)}&run=true`, + ); + + // Wait for loader to disappear + await page + .getByTestId("loader") + .waitFor({ state: "hidden", timeout: 15000 }) + .catch(() => {}); + + // Expand the query panel by clicking the filter sliders icon + await page + .getByTestId("icon-primary-filterSliders") + .locator(":visible") + .click(); + + // Assert editor mode is active: "Malloy Editor" heading is visible in the query panel + await expect(page.getByText("Malloy Editor")).toBeVisible({ + timeout: 15000, + }); + + // Assert the query results are shown + await expect( + page.getByRole("tab", { name: "Results", selected: true }), + ).toBeVisible({ timeout: 15000 }); + }); + + test("Parseable malloy query opens in visual query editor mode", async ({ + page, + }) => { + // A query in stable format IS parseable by malloyToQuery + // and should show the visual query editor + await page.goto( + "./#/model/invoices/explorer/invoices?query=run:invoices->by_status&run=true", + ); + + // Wait for loader to disappear + await page + .getByTestId("loader") + .waitFor({ state: "hidden", timeout: 15000 }) + .catch(() => {}); + + // Expand the query panel + await page + .getByTestId("icon-primary-filterSliders") + .locator(":visible") + .click(); + + // The visual query editor should be shown, NOT the Malloy Editor + await expect(page.getByText("Main query")).toBeVisible({ timeout: 15000 }); + await expect(page.getByText("Malloy Editor")).not.toBeVisible(); + + // Results should be shown + await expect( + page.getByRole("tab", { name: "Results", selected: true }), + ).toBeVisible({ timeout: 15000 }); + }); + + test("Custom query is preserved in the URL", async ({ page }) => { + const customQuery = + "run: invoices -> { group_by: status; aggregate: double_count is count() * 2 }"; + + await page.goto( + `./#/model/invoices/explorer/invoices?query=${encodeURIComponent(customQuery)}&run=true`, + ); + + // Wait for loader to disappear + await page + .getByTestId("loader") + .waitFor({ state: "hidden", timeout: 15000 }) + .catch(() => {}); + + // Verify the query is preserved in the URL + const url = page.url(); + expect(url).toContain("query="); + expect(decodeURIComponent(url)).toContain( + "run: invoices -> { group_by: status; aggregate: double_count is count() * 2 }", + ); + }); + + test("Running from editor mode preserves editor mode", async ({ page }) => { + // Start with a parseable query in visual mode + await page.goto( + "./#/model/invoices/explorer/invoices?query=run:invoices->by_status&run=true", + ); + + await page + .getByTestId("loader") + .waitFor({ state: "hidden", timeout: 15000 }) + .catch(() => {}); + + // Expand the query panel + await page + .getByTestId("icon-primary-filterSliders") + .locator(":visible") + .click(); + + await expect(page.getByText("Main query")).toBeVisible({ timeout: 15000 }); + + // Convert to Malloy editor + await page + .getByTestId("icon-primary-meatballs") + .locator(":visible") + .first() + .click(); + await page.getByText("Convert to Malloy").click(); + + await expect(page.getByText("Malloy Editor")).toBeVisible({ + timeout: 15000, + }); + + // Wait for Monaco to load + await expect(page.locator(".monaco-editor").first()).toBeVisible({ + timeout: 30000, + }); + + // Run the query from editor mode + await page.getByRole("button", { name: "Run" }).click(); + + // Wait for query to complete + await page + .getByTestId("loader") + .waitFor({ state: "hidden", timeout: 15000 }) + .catch(() => {}); + + // Should still be in editor mode after running + await expect(page.getByText("Malloy Editor")).toBeVisible({ + timeout: 5000, + }); + + // URL should contain mode=code + expect(page.url()).toContain("mode=code"); + + // Should NOT show the "Query was updated" warning + await expect(page.getByText("Query was updated")).not.toBeVisible(); + }); + + test("mode=code forces editor mode even for simple parseable queries", async ({ + page, + }) => { + // A simple view query that would normally open in the structured builder, + // but mode=code forces it into the code editor + await page.goto( + "./#/model/invoices/explorer/invoices?query=run:invoices->by_status&run=true&mode=code", + ); + + await page + .getByTestId("loader") + .waitFor({ state: "hidden", timeout: 15000 }) + .catch(() => {}); + + // Expand the query panel + await page + .getByTestId("icon-primary-filterSliders") + .locator(":visible") + .click(); + + // Should be in editor mode, NOT the visual query builder + await expect(page.getByText("Malloy Editor")).toBeVisible({ + timeout: 15000, + }); + await expect(page.getByText("Main query")).not.toBeVisible(); + + // Results should still be shown + await expect( + page.getByRole("tab", { name: "Results", selected: true }), + ).toBeVisible({ timeout: 15000 }); + }); + + test("Convert to Malloy editor and back to visual editor", async ({ + page, + }) => { + // Start with a parseable query in visual mode (stable format) + await page.goto( + "./#/model/invoices/explorer/invoices?query=run:invoices->by_status&run=true", + ); + + // Wait for visual editor to be ready + await page + .getByTestId("loader") + .waitFor({ state: "hidden", timeout: 15000 }) + .catch(() => {}); + + // Expand the query panel + await page + .getByTestId("icon-primary-filterSliders") + .locator(":visible") + .click(); + + await expect(page.getByText("Main query")).toBeVisible({ timeout: 15000 }); + + // Click the three-dot menu (meatballs icon) on the "Main query" section + await page + .getByTestId("icon-primary-meatballs") + .locator(":visible") + .first() + .click(); + await page.getByText("Convert to Malloy").click(); + + // Should now be in editor mode + await expect(page.getByText("Malloy Editor")).toBeVisible({ + timeout: 15000, + }); + + // Wait for Monaco editor to initialize (lazy-loaded) + await expect(page.locator(".monaco-editor").first()).toBeVisible({ + timeout: 30000, + }); + + // Wait for Monaco diagnostics to complete so "Use Query Editor" becomes enabled + await page.waitForTimeout(3000); + + // Switch back: click the three-dot menu in editor mode → "Use Query Editor" + await page + .getByTestId("icon-primary-meatballs") + .locator(":visible") + .first() + .click(); + await page.getByText("Use Query Editor").click(); + + // Should be back in visual mode + await expect(page.getByText("Main query")).toBeVisible({ timeout: 5000 }); + await expect(page.getByText("Malloy Editor")).not.toBeVisible(); + }); +}); diff --git a/package-lock.json b/package-lock.json index 08eeafb..a54b520 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@shikijs/core": "^3.21.0", "@shikijs/engine-javascript": "^3.21.0", "@shikijs/themes": "^3.21.0", + "monaco-editor-core": "^0.55.1", "react": "^19.2.4", "react-dom": "^19.2.4", "react-markdown": "^10.1.0", @@ -39,6 +40,7 @@ "eslint-plugin-react-refresh": "^0.4.26", "globals": "^17.2.0", "playwright": "^1.58.0", + "playwright-core": "^1.58.0", "prettier": "^3.8.1", "typescript": "^5.9.3", "typescript-eslint": "^8.54.0", diff --git a/package.json b/package.json index 4cd889f..6cbe4ea 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "@shikijs/core": "^3.21.0", "@shikijs/engine-javascript": "^3.21.0", "@shikijs/themes": "^3.21.0", + "monaco-editor-core": "^0.55.1", "react": "^19.2.4", "react-dom": "^19.2.4", "react-markdown": "^10.1.0", diff --git a/src/SourceExplorer.tsx b/src/SourceExplorer.tsx index 7a23499..a694557 100644 --- a/src/SourceExplorer.tsx +++ b/src/SourceExplorer.tsx @@ -1,10 +1,13 @@ import { useState } from "react"; import * as MalloyInterface from "@malloydata/malloy-interfaces"; +import { malloyToQuery } from "@malloydata/malloy"; +import type * as Monaco from "monaco-editor-core"; import type { Navigation } from "react-router"; import { useLoaderData, useNavigation, useSearchParams } from "react-router"; import * as React from "react"; import type { SubmittedQuery } from "@malloydata/malloy-explorer"; import { + CodeEditorContext, MalloyExplorerProvider, QueryPanel, ResizableCollapsiblePanel, @@ -14,6 +17,7 @@ import { import "@malloydata/malloy-explorer/styles.css"; import type { SourceExplorerLoaderData } from "./routeType"; import { type JSX } from "react/jsx-runtime"; +import { getMonaco } from "./monaco-setup"; export default SourceExplorer; @@ -26,6 +30,13 @@ function SourceExplorer(): JSX.Element { string | MalloyInterface.Query | undefined >(); const [nestViewPath, setNestViewPath] = useState([]); + const [monaco, setMonaco] = useState(); + + React.useEffect(() => { + if (undefined === monaco) { + void getMonaco().then(setMonaco); + } + }, [monaco]); const runQuery = React.useCallback( ( @@ -39,6 +50,17 @@ function SourceExplorer(): JSX.Element { }, [searchParams, setSearchParams], ); + + const runQueryString = React.useCallback( + (_source: MalloyInterface.SourceInfo, query: string): void => { + const newSearchParams = serializeStringQueryToUrl(searchParams, query); + if (null !== newSearchParams) { + setSearchParams(newSearchParams); + } + }, + [searchParams, setSearchParams], + ); + const { state } = useNavigation(); const submittedQueryWithState = React.useMemo( () => updateExecutionState(routeData.submittedQuery, state), @@ -49,66 +71,78 @@ function SourceExplorer(): JSX.Element { setDraftQuery(routeData.parsedQuery); }, [routeData.parsedQuery]); + const codeEditorContextValue = React.useMemo( + () => ({ + ...(monaco ? { monaco } : {}), + modelDef: routeData.modelDef, + modelUri: routeData.modelUri, + malloyToQuery: (src: string) => malloyToQuery(src), + }), + [monaco, routeData.modelDef, routeData.modelUri], + ); + return ( - /* @ts-expect-error Exact Optional Type is wrong from lib */ - -
+ {/* @ts-expect-error Exact Optional Type is wrong from lib */} +
- - { - // if (executedQuery) refreshModel(); - }} - /> - - - - -
- {/* @ts-expect-error Exact optional type is wrong from lib */} - + + { + // if (executedQuery) refreshModel(); + }} + /> + + + + +
+ {/* @ts-expect-error Exact optional type is wrong from lib */} + +
-
- + + ); } @@ -116,7 +150,6 @@ function serializeQueryToUrl( searchParams: URLSearchParams, query: MalloyInterface.Query, ): URLSearchParams | null { - // Update URL with the query being run const queryString = queryToMalloyString(query); const newSearchParams = new URLSearchParams(searchParams); @@ -128,6 +161,29 @@ function serializeQueryToUrl( newSearchParams.set("run", "true"); newSearchParams.delete("load"); + newSearchParams.delete("mode"); + + if (newSearchParams.toString() !== searchParams.toString()) { + return newSearchParams; + } + return null; +} + +function serializeStringQueryToUrl( + searchParams: URLSearchParams, + query: string, +): URLSearchParams | null { + const newSearchParams = new URLSearchParams(searchParams); + + if (query.length > 0) { + newSearchParams.set("query", query); + } else { + newSearchParams.delete("query"); + } + + newSearchParams.set("run", "true"); + newSearchParams.set("mode", "code"); + newSearchParams.delete("load"); if (newSearchParams.toString() !== searchParams.toString()) { return newSearchParams; diff --git a/src/helpers.ts b/src/helpers.ts index b1a4a4d..a17eb5a 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -92,7 +92,7 @@ function setupMalloyRuntime({ async function executeMalloyQuery( modelMaterializer: malloy.ModelMaterializer, query: string, - parsedQuery?: MalloyInterface.Query, + parsedQuery?: MalloyInterface.Query | string, ): Promise { const queryResolutionStartMillis = Date.now(); diff --git a/src/llms-txt/generator.ts b/src/llms-txt/generator.ts index 5a15032..33dfaef 100644 --- a/src/llms-txt/generator.ts +++ b/src/llms-txt/generator.ts @@ -86,7 +86,7 @@ function generateOverview( ); } - // Custom query example + // Custom query example (structured builder) const customQuery = encodeURIComponent( `run: ${source.name} -> { select: * limit: 10 }`, ); @@ -94,6 +94,14 @@ function generateOverview( `\`${fullBase}/#/model/${modelName}/explorer/${sourceName}?query=${customQuery}&run=true\` - Custom query`, ); + // Code editor mode example (custom aggregate) + const codeQuery = encodeURIComponent( + `run: ${source.name} -> { aggregate: row_count is count() }`, + ); + examplesList.push( + `\`${fullBase}/#/model/${modelName}/explorer/${sourceName}?query=${codeQuery}&run=true&mode=code\` - Code editor query (custom aggregate)`, + ); + if (examplesList.length > 0) { examples = `\n\n**Example URLs:**\n${examplesList.join("\n")}`; } @@ -111,7 +119,8 @@ function generateOverview( | \`/#/model/{model}\` | Model schema | | \`/#/model/{model}/preview/{source}\` | Preview data (50 rows) | | \`/#/model/{model}/explorer/{source}\` | Interactive query builder | -| \`/#/model/{model}/explorer/{source}?query={malloy}&run=true\` | Execute query | +| \`/#/model/{model}/explorer/{source}?query={malloy}&run=true\` | Execute query (structured builder) | +| \`/#/model/{model}/explorer/{source}?query={malloy}&run=true&mode=code\` | Execute query (code editor — for custom aggregates) | | \`/#/model/{model}/query/{queryName}\` | Run named query | | \`/#/notebook/{notebook}\` | View notebook | | \`/downloads/models/{model}.malloy\` | Download model file | @@ -202,6 +211,7 @@ Control UI behavior with these query parameters: **Explorer (\`/model/{model}/explorer/{source}\`):** - \`query\` - Malloy query string (URL-encoded) - \`run=true\` - Auto-execute the query +- \`mode=code\` - Use code editor instead of structured query builder (see "Query Modes" below). Auto-inferred for queries with custom expressions, but can be set explicitly to always use the code editor. - \`includeTopValues=true\` - Load field top values - \`showQueryPanel=true\` - Expand query panel - \`showSourcePanel=true\` - Expand source/schema panel @@ -217,7 +227,115 @@ Control UI behavior with these query parameters: } function generateMalloyQueryGuide(): string { - return `## Malloy Query Syntax + return `## Query Modes + +There are two ways to run queries in the explorer: + +### Structured Query Builder (default) + +The default mode uses a visual query builder UI. Queries that only reference **existing** dimensions, measures, and views defined in the source model are parsed into the structured builder automatically. + +**What the structured builder supports:** +- \`group_by:\` using source dimensions +- \`aggregate:\` using source measures (by name only) +- \`where:\` / \`having:\` filters +- \`order_by:\` with asc/desc +- \`limit:\` +- \`nest:\` for nested sub-queries +- Running existing views: \`source -> view_name\` + +**Structured builder URL (no \`mode\` param needed):** +\`\`\` +/#/model/{model}/explorer/{source}?query={url_encoded_malloy}&run=true +\`\`\` + +**Example structured queries:** +\`\`\`malloy +run: flights -> by_carrier + +run: flights -> { + group_by: carrier + aggregate: flight_count + where: distance > 500 + order_by: flight_count desc + limit: 20 +} + +run: orders -> { + group_by: status + aggregate: order_count, total_revenue + nest: by_month is { + group_by: created_date.month + aggregate: order_count + } +} +\`\`\` + +### Code Editor Mode (\`mode=code\`) + +For queries that go beyond referencing existing fields — such as defining **custom aggregate expressions**, **ad-hoc calculated fields**, or **arithmetic on measures** — use code editor mode by adding \`mode=code\` to the URL. This opens a Monaco code editor instead of the structured query builder. + +**Code editor URL (requires \`mode=code\`):** +\`\`\` +/#/model/{model}/explorer/{source}?query={url_encoded_malloy}&run=true&mode=code +\`\`\` + +**What code editor mode enables (not possible in structured builder):** +- Custom aggregate expressions with inline definitions (\`is\` keyword) +- Arithmetic on aggregates (e.g., \`count() * 2\`, \`sum(x) / count()\`) +- Ad-hoc measures not defined in the source model +- Pipelining: chain multiple query stages with \`->\` +- Any valid Malloy query syntax + +**Example code editor queries:** +\`\`\`malloy +# Custom aggregate: define a new measure inline +run: orders -> { + group_by: status + aggregate: double_count is count() * 2 +} + +# Arithmetic on aggregates +run: flights -> { + group_by: carrier + aggregate: + total_flights is count() + avg_distance is sum(distance) / count() +} + +# Ad-hoc calculated fields with complex expressions +run: orders -> { + group_by: category + aggregate: + order_count is count() + total_revenue is sum(amount) + avg_order_value is sum(amount) / count() + order_by: avg_order_value desc + limit: 10 +} + +# Combining filters with custom aggregates +run: invoices -> { + where: status = 'paid' + group_by: region + aggregate: + paid_count is count() + total_paid is sum(amount) + pct_of_total is sum(amount) / all(sum(amount)) * 100 +} + +# Pipelining: chain query stages to transform results +run: orders -> { + group_by: category + aggregate: total_revenue is sum(amount) +} -> { + select: * + where: total_revenue > 1000 + order_by: total_revenue desc +} +\`\`\` + +## Malloy Query Syntax Reference **Basic Structure:** \`\`\`malloy @@ -230,26 +348,17 @@ run: source_name -> { } \`\`\` -**Common Operations:** +**Operations:** - Select all: \`source -> { select: * }\` - Group: \`source -> { group_by: field }\` -- Aggregate: \`source -> { aggregate: measure }\` +- Aggregate existing measure: \`source -> { aggregate: measure_name }\` +- Custom aggregate: \`source -> { aggregate: my_agg is count() * 2 }\` - Filter: \`source -> { where: field > value }\` - Sort: \`source -> { order_by: field desc }\` - Limit: \`source -> { limit: 10 }\` - Run view: \`source -> view_name\` - Nest: \`{ nest: name is { group_by: field; aggregate: measure } }\` +- Pipeline: \`source -> { aggregate: ... } -> { select: * where: ... }\` -**Time Granularity:** \`.year\`, \`.quarter\`, \`.month\`, \`.week\`, \`.day\` (e.g., \`order_date.month\`) - -**Example Query:** -\`\`\`malloy -run: orders -> { - group_by: category - aggregate: order_count, total_revenue - where: status = 'Delivered' - order_by: total_revenue desc - limit: 10 -} -\`\`\``; +**Time Granularity:** \`.year\`, \`.quarter\`, \`.month\`, \`.week\`, \`.day\` (e.g., \`order_date.month\`)`; } diff --git a/src/monaco-setup.ts b/src/monaco-setup.ts new file mode 100644 index 0000000..0716425 --- /dev/null +++ b/src/monaco-setup.ts @@ -0,0 +1,19 @@ +import type * as Monaco from "monaco-editor-core"; + +let monacoInstance: typeof Monaco | undefined; + +export async function getMonaco(): Promise { + if (undefined !== monacoInstance) { + return monacoInstance; + } + const monaco = await import("monaco-editor-core"); + const { default: editorWorker } = + await import("monaco-editor-core/esm/vs/editor/editor.worker.start?worker"); + self.MonacoEnvironment = { + getWorker() { + return new editorWorker(); + }, + }; + monacoInstance = monaco; + return monaco; +} diff --git a/src/routeType.ts b/src/routeType.ts index 114f0de..9367f92 100644 --- a/src/routeType.ts +++ b/src/routeType.ts @@ -15,8 +15,10 @@ export type { type SourceExplorerLoaderData = { sourceInfo: MalloyInterface.SourceInfo; topValues: malloy.SearchValueMapResult[]; - parsedQuery: undefined | MalloyInterface.Query; + parsedQuery: undefined | MalloyInterface.Query | string; submittedQuery: undefined | SubmittedQuery; + modelDef: malloy.ModelDef; + modelUri: URL; }; type ModelHomeLoaderData = RuntimeSetup; diff --git a/src/routing.tsx b/src/routing.tsx index 1e2d4ae..5089e43 100644 --- a/src/routing.tsx +++ b/src/routing.tsx @@ -136,7 +136,7 @@ function createAppRouter({ async function loadAndCacheMalloyQueryResult( source: SourceCache, modelMaterializer: malloy.ModelMaterializer, - parsedQuery: undefined | MalloyInterface.Query, + parsedQuery: undefined | MalloyInterface.Query | string, querySrc: string, ): Promise { return ( @@ -243,23 +243,32 @@ function createAppRouter({ const includeTopValues = urlSearchParams.get("includeTopValues") === "true"; - const { modelMaterializer } = - await loadAndCacheModel(modelName); + const cachedModel = await loadAndCacheModel(modelName); + const { modelMaterializer } = cachedModel; const source = await loadAndCacheSource( modelName, sourceName, includeTopValues, ); + const modeParam = urlSearchParams.get("mode"); + // When mode=code, keep the raw string for the code editor const parsedQuery = null === querySrcParam ? undefined - : loadAndCacheMalloyQuery(source, querySrcParam); + : "code" === modeParam + ? querySrcParam + : (loadAndCacheMalloyQuery(source, querySrcParam) ?? + querySrcParam); + const structuredQuery = + "string" === typeof parsedQuery ? undefined : parsedQuery; const submittedQuery = "true" === runQueryParam && null !== querySrcParam ? await loadAndCacheMalloyQueryResult( source, modelMaterializer, - parsedQuery, + // In code mode, pass the raw string so submittedQuery.query + // matches draftQuery for the "query was updated" comparison + "code" === modeParam ? querySrcParam : structuredQuery, querySrcParam, ) : undefined; @@ -269,6 +278,8 @@ function createAppRouter({ topValues: source.topValues, parsedQuery, submittedQuery, + modelDef: cachedModel.model._modelDef, + modelUri: getModelURL(modelName), }; }, element: ,