diff --git a/.github/workflows/db-bigquery.yaml b/.github/workflows/db-bigquery.yaml index 7b65be1b2..17b854ef3 100644 --- a/.github/workflows/db-bigquery.yaml +++ b/.github/workflows/db-bigquery.yaml @@ -1,4 +1,4 @@ -name: BigQuery DB +name: "DB:BigQuery" on: pull_request: @@ -31,7 +31,7 @@ jobs: run: | npm ci --loglevel error npm run build - npm run test-silent -- -- test packages/malloy-db-bigquery + npx jest --reporters jest-silent-reporter --reporters summary --config jest.bigquery.config.ts env: CI: true MALLOY_DATABASES: bigquery diff --git a/.github/workflows/db-duckdb-wasm.yaml b/.github/workflows/db-duckdb-wasm.yaml new file mode 100644 index 000000000..4acc430a9 --- /dev/null +++ b/.github/workflows/db-duckdb-wasm.yaml @@ -0,0 +1,29 @@ +name: "DB:DuckDB(WASM)" + +on: [pull_request, workflow_call] + +jobs: + test-duckdb-wasm: + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [18.x] + + steps: + - uses: actions/checkout@v4 + with: + submodules: 'true' + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + - name: npm install, build, and test + run: | + npm ci --loglevel error + npm run build + npm run build-duckdb-db + npx jest --reporters jest-silent-reporter --reporters summary --config jest.duckdb-other.config.ts --runInBand + env: + CI: true + MALLOY_DATABASES: duckdb_wasm diff --git a/.github/workflows/db-duckdb.yaml b/.github/workflows/db-duckdb.yaml new file mode 100644 index 000000000..7b4f7a56d --- /dev/null +++ b/.github/workflows/db-duckdb.yaml @@ -0,0 +1,29 @@ +name: "DB:DuckDB" + +on: [pull_request, workflow_call] + +jobs: + test-duckdb: + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [18.x] + + steps: + - uses: actions/checkout@v4 + with: + submodules: 'true' + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + - name: npm install, build, and test + run: | + npm ci --loglevel error + npm run build + npm run build-duckdb-db + npx jest --reporters jest-silent-reporter --reporters summary --config jest.duckdb.config.ts --runInBand + env: + CI: true + MALLOY_DATABASES: duckdb diff --git a/.github/workflows/db-motherduck.yaml b/.github/workflows/db-motherduck.yaml index ce85fafef..916714114 100644 --- a/.github/workflows/db-motherduck.yaml +++ b/.github/workflows/db-motherduck.yaml @@ -1,4 +1,4 @@ -name: MotherDuck DB +name: "DB:MotherDuck" on: pull_request: @@ -27,7 +27,7 @@ jobs: run: | npm ci --loglevel error npm run build - npm run test-silent -- -- test packages/malloy-duckdb + npx jest --reporters jest-silent-reporter --reporters summary --config jest.duckdb-other.config.ts env: CI: true MALLOY_DATABASES: motherduck diff --git a/.github/workflows/db-mysql.yaml b/.github/workflows/db-mysql.yaml index 75fa18512..b23bc31ce 100644 --- a/.github/workflows/db-mysql.yaml +++ b/.github/workflows/db-mysql.yaml @@ -1,4 +1,4 @@ -name: MySQL DB +name: "DB:MySQL" on: [pull_request, workflow_call] @@ -25,7 +25,7 @@ jobs: npm run build npm run build-duckdb-db ./test/mysql/mysql_start.sh - npm run test-silent -- -- test + npx jest --reporters jest-silent-reporter --reporters summary --config jest.mysql.config.ts ./test/mysql/mysql_stop.sh env: MALLOY_DATABASES: mysql diff --git a/.github/workflows/db-postgres.yaml b/.github/workflows/db-postgres.yaml index 9aa12bddf..b7b3db892 100644 --- a/.github/workflows/db-postgres.yaml +++ b/.github/workflows/db-postgres.yaml @@ -1,4 +1,4 @@ -name: Postgres DB +name: "DB:Postgres" on: [pull_request, workflow_call] @@ -39,10 +39,7 @@ jobs: npm run build echo CREATE EXTENSION tsm_system_rows\; | psql gunzip -c test/data/postgres/malloytest-postgres.sql.gz | psql - npm run test-silent -- -- test packages/malloy-db-postgres - export MALLOY_DATABASES=duckdb,postgres - npm run build-duckdb-db - npm run test-silent -- -- test/src/databases/multi-connection/multi_connection.spec.ts + npx jest --reporters jest-silent-reporter --reporters summary --config jest.postgres.config.ts env: MALLOY_DATABASES: postgres PGHOST: localhost diff --git a/.github/workflows/db-presto.yaml b/.github/workflows/db-presto.yaml index cdd2d3138..8baad930d 100644 --- a/.github/workflows/db-presto.yaml +++ b/.github/workflows/db-presto.yaml @@ -1,4 +1,4 @@ -name: Presto DB +name: "DB:Presto" on: pull_request: @@ -30,8 +30,7 @@ jobs: npm run build npm run build-duckdb-db ./test/presto/presto_start.sh - npm run test-silent -- -- packages/malloy-db-trino - npm run test-silent -- -- test + npx jest --reporters jest-silent-reporter --reporters summary --config jest.presto.config.ts ./test/presto/presto_stop.sh env: MALLOY_DATABASES: presto diff --git a/.github/workflows/db-snowflake.yaml b/.github/workflows/db-snowflake.yaml index 3ac9095aa..0accc0f77 100644 --- a/.github/workflows/db-snowflake.yaml +++ b/.github/workflows/db-snowflake.yaml @@ -1,4 +1,4 @@ -name: Snowflake DB +name: "DB:Snowflake" on: pull_request: @@ -28,7 +28,7 @@ jobs: npm ci --loglevel error npm run build ./scripts/gen-snowflake-auth.sh - npm run test-silent -- -- test packages/malloy-db-snowflake + npx jest --reporters jest-silent-reporter --reporters summary --config jest.snowflake.config.ts env: CI: true MALLOY_DATABASES: snowflake diff --git a/.github/workflows/db-trino.yaml b/.github/workflows/db-trino.yaml index 2249e503a..3783bbca7 100644 --- a/.github/workflows/db-trino.yaml +++ b/.github/workflows/db-trino.yaml @@ -1,4 +1,4 @@ -name: Trino DB +name: "DB:Trino" on: pull_request: @@ -30,8 +30,7 @@ jobs: npm run build npm run build-duckdb-db ./test/trino/trino_start.sh - npm run test-silent -- -- packages/malloy-db-trino - npm run test-silent -- -- test + npx jest --reporters jest-silent-reporter --reporters summary --config jest.trino.config.ts ./test/trino/trino_stop.sh env: MALLOY_DATABASES: trino diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 628a8fc8c..b3a4d5aca 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -1,6 +1,15 @@ +# The "Core" job runs through all the tests which are not dialect specific. +# This includes tests which don't touch the database at all, and tests +# which touch some combination of duckdb,bigquery,postgres but do +# not need to be once times for each dialect. name: Core -on: [pull_request, workflow_call] +on: + pull_request: + workflow_call: + secrets: + BIGQUERY_KEY: + required: true jobs: test-all: @@ -10,6 +19,20 @@ jobs: matrix: node-version: [18.x] + services: + postgres: + image: postgres + env: + POSTGRES_USER: root + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - uses: actions/checkout@v4 with: @@ -18,13 +41,23 @@ jobs: uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} + - name: GCloud auth + uses: 'google-github-actions/auth@v2' + with: + credentials_json: '${{ secrets.BIGQUERY_KEY }}' - name: npm install, build, and test run: | npm ci --loglevel error + sh scripts/ci-test-sanity-check.sh npm run lint npm run build npm run build-duckdb-db - npm run test-silent + echo CREATE EXTENSION tsm_system_rows\; | psql + gunzip -c test/data/postgres/malloytest-postgres.sql.gz | psql + npx jest --reporters jest-silent-reporter --reporters summary --config jest.core.config.ts --runInBand env: CI: true - MALLOY_DATABASES: duckdb,duckdb_wasm + PGHOST: localhost + PGPORT: 5432 + PGUSER: root + PGPASSWORD: postgres diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 911eabdb1..3997f9358 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -3,6 +3,20 @@ name: Malloy Tests on: [workflow_dispatch] jobs: + main: + uses: './.github/workflows/main.yaml' + secrets: + BIGQUERY_KEY: ${{ secrets.BIGQUERY_KEY }} + db-trino: + uses: './.github/workflows/db-trino.yaml' + secrets: + BQ_PRESTO_TRINO_KEY: ${{ secrets.BQ_PRESTO_TRINO_KEY }} + db-presto: + uses: './.github/workflows/db-presto.yaml' + secrets: + BQ_PRESTO_TRINO_KEY: ${{ secrets.BQ_PRESTO_TRINO_KEY }} + db-duckdb: + uses: './github/workflows/db-duckdb.yaml' db-bigquery: uses: './.github/workflows/db-bigquery.yaml' secrets: @@ -11,35 +25,33 @@ jobs: uses: './.github/workflows/db-motherduck.yaml' secrets: MOTHERDUCK_TOKEN_10: ${{ secrets.MOTHERDUCK_TOKEN_10 }} - db-mysql: - uses: './.github/workflows/db-mysql.yaml' db-postgres: uses: './.github/workflows/db-postgres.yaml' - db-presto: - uses: './.github/workflows/db-presto.yaml' - secrets: - BQ_PRESTO_TRINO_KEY: ${{ secrets.BQ_PRESTO_TRINO_KEY }} db-snowflake: uses: './.github/workflows/db-snowflake.yaml' secrets: SNOWFLAKE_CONNECTION: ${{ secrets.SNOWFLAKE_CONNECTION }} - db-trino: - uses: './.github/workflows/db-trino.yaml' - secrets: - BQ_PRESTO_TRINO_KEY: ${{ secrets.BQ_PRESTO_TRINO_KEY }} - main: - uses: './.github/workflows/main.yaml' + db-mysql: + uses: './.github/workflows/db-mysql.yaml' + db-duckdb-wasm: + uses: './github/workflows/db-duckdb-wasm.yaml' + + # I think I have the sorted roughly longest to shortest + # so the longer running jobs get wrokers sooner, not sure + # that is the right plan malloy-tests: needs: + - main + - db-snowflake - db-bigquery - - db-motherduck - - db-mysql - - db-postgres + - db-duckdb - db-presto - - db-snowflake + - db-duckdb-wasm - db-trino - - main + - db-postgres + - db-motherduck + - db-mysql runs-on: ubuntu-latest steps: - name: Success diff --git a/jest.bigquery.config.ts b/jest.bigquery.config.ts new file mode 100644 index 000000000..0baba1365 --- /dev/null +++ b/jest.bigquery.config.ts @@ -0,0 +1,8 @@ +module.exports = { + ...require('./jest.config.ts'), + roots: [ + '/packages/malloy-db-bigquery/src/', + '/test/src/databases/all/', + '/test/src/databases/bigquery/', + ], +}; diff --git a/jest.core.config.ts b/jest.core.config.ts new file mode 100644 index 000000000..6fa6456ce --- /dev/null +++ b/jest.core.config.ts @@ -0,0 +1,10 @@ +module.exports = { + ...require('./jest.config.ts'), + roots: [ + '/packages/malloy-malloy-sql/', + '/packages/malloy-syntax-highlight/', + '/packages/malloy/', + '/test/src/core/', + '/test/src/render/', + ], +}; diff --git a/jest.duckdb-other.config.ts b/jest.duckdb-other.config.ts new file mode 100644 index 000000000..05b25d3fc --- /dev/null +++ b/jest.duckdb-other.config.ts @@ -0,0 +1,9 @@ +// For motherduck and duckdb-wasm +module.exports = { + ...require('./jest.config.ts'), + roots: [ + '/packages/malloy-db-duckdb/', + '/test/src/databases/all/', + '/test/src/databases/duckdb-all/', + ], +}; diff --git a/jest.duckdb.config.ts b/jest.duckdb.config.ts new file mode 100644 index 000000000..3b72f9501 --- /dev/null +++ b/jest.duckdb.config.ts @@ -0,0 +1,9 @@ +module.exports = { + ...require('./jest.config.ts'), + roots: [ + '/packages/malloy-db-duckdb/', + '/test/src/databases/all/', + '/test/src/databases/duckdb/', + '/test/src/databases/duckdb-all/', + ], +}; diff --git a/jest.mysql.config.ts b/jest.mysql.config.ts new file mode 100644 index 000000000..f61fb0b6d --- /dev/null +++ b/jest.mysql.config.ts @@ -0,0 +1,4 @@ +module.exports = { + ...require('./jest.config.ts'), + roots: ['/test/src/databases/all/'], +}; diff --git a/jest.postgres.config.ts b/jest.postgres.config.ts new file mode 100644 index 000000000..898cdcc3f --- /dev/null +++ b/jest.postgres.config.ts @@ -0,0 +1,8 @@ +module.exports = { + ...require('./jest.config.ts'), + roots: [ + '/packages/malloy-db-postgres/', + '/test/src/databases/all/', + '/test/src/databases/postgres/', + ], +}; diff --git a/jest.presto.config.ts b/jest.presto.config.ts new file mode 100644 index 000000000..d0431696c --- /dev/null +++ b/jest.presto.config.ts @@ -0,0 +1,7 @@ +module.exports = { + ...require('./jest.config.ts'), + roots: [ + '/test/src/databases/all/', + '/test/src/databases/presto-trino/', + ], +}; diff --git a/jest.snowflake.config.ts b/jest.snowflake.config.ts new file mode 100644 index 000000000..6d38151ea --- /dev/null +++ b/jest.snowflake.config.ts @@ -0,0 +1,7 @@ +module.exports = { + ...require('./jest.config.ts'), + roots: [ + '/packages/malloy-db-snowflake/', + '/test/src/databases/all/', + ], +}; diff --git a/jest.trino.config.ts b/jest.trino.config.ts new file mode 100644 index 000000000..f58de1848 --- /dev/null +++ b/jest.trino.config.ts @@ -0,0 +1,8 @@ +module.exports = { + ...require('./jest.config.ts'), + roots: [ + '/packages/malloy-db-trino/src/', + '/test/src/databases/all/', + '/test/src/databases/presto-trino/', + ], +}; diff --git a/packages/malloy-render/DEVELOPING.md b/packages/malloy-render/DEVELOPING.md index 07e5097e8..ea864ce8a 100644 --- a/packages/malloy-render/DEVELOPING.md +++ b/packages/malloy-render/DEVELOPING.md @@ -8,11 +8,16 @@ The legacy renderer is deprecated but is still available and in use for features ## Viewing the renderer locally -Storybook is used to view the renderer locally. To launch the storybook: +Storybook is used to view the renderer locally. To launch the storybook, run the storybook script in `packages/malloy-render/package.json`: ```bash $ npm run storybook ``` +Or, on a Windows machine, and from the Malloy repo directory: +```bash +$ npm run --prefix packages/malloy-render storybook-windows +``` + Then navigate to the URL provided. In this storybook, you can navigate between different stories that render Malloy queries from the Malloy source code. diff --git a/packages/malloy-render/package.json b/packages/malloy-render/package.json index e8329f400..f489dcdfe 100644 --- a/packages/malloy-render/package.json +++ b/packages/malloy-render/package.json @@ -24,6 +24,7 @@ "clean": "tsc --build --clean", "prepublishOnly": "npm run build", "storybook": "rm -rf ./node_modules/.cache && storybook dev -p 6006", + "storybook-windows": "del /s /q .\\node_modules\\.cache && storybook dev -p 6006", "build-storybook": "storybook build", "build-source": "vite build --outDir 'dist/module' --config vite.config.ts", "build-webcomponent": "vite build --outDir 'dist/webcomponent' --config vite.config.webcomponent.ts && vite build --outDir 'dist/register' --config vite.config.webcomponent-register.ts", diff --git a/packages/malloy-render/src/component/line-chart/generate-line_chart-vega-spec.ts b/packages/malloy-render/src/component/line-chart/generate-line_chart-vega-spec.ts index e88dea82c..9337f3549 100644 --- a/packages/malloy-render/src/component/line-chart/generate-line_chart-vega-spec.ts +++ b/packages/malloy-render/src/component/line-chart/generate-line_chart-vega-spec.ts @@ -80,6 +80,7 @@ export function generateLineChartVegaSpec( const xField = getFieldFromRootPath(explore, xFieldPath); const xIsDateorTime = xField.isAtomicField() && (xField.isDate() || xField.isTimestamp()); + const xIsBoolean = xField.isAtomicField() && xField.isBoolean(); const yField = getFieldFromRootPath(explore, yFieldPath); const seriesField = seriesFieldPath ? getFieldFromRootPath(explore, seriesFieldPath) @@ -576,6 +577,8 @@ export function generateLineChartVegaSpec( domain: shouldShareXDomain ? xIsDateorTime ? [xMeta.min, xMeta.max] + : xIsBoolean + ? [true, false] : [...xMeta.values] : {data: 'values', field: 'x'}, range: 'width', diff --git a/packages/malloy-render/src/stories/line_charts.stories.malloy b/packages/malloy-render/src/stories/line_charts.stories.malloy index e8db5b544..791de34b6 100644 --- a/packages/malloy-render/src/stories/line_charts.stories.malloy +++ b/packages/malloy-render/src/stories/line_charts.stories.malloy @@ -236,6 +236,23 @@ source: products is duckdb.table("static/data/products.parquet") extend { } } + dimension: + is_female is pick true when department = 'Women' else false + measure: + product_count_by_gender is count(id) + + #(story) + view: line_chart_with_boolean is { + nest: product_sales_by_gender is { + group_by: is_female + aggregate: product_count_by_gender + } + # line_chart + nest: product_sales_chart is { + group_by: is_female + aggregate: product_count_by_gender + } + } } run: products -> { group_by: distribution_center_id} \ No newline at end of file diff --git a/packages/malloy/src/lang/grammar/MalloyParser.g4 b/packages/malloy/src/lang/grammar/MalloyParser.g4 index c19a38eba..4672962a3 100644 --- a/packages/malloy/src/lang/grammar/MalloyParser.g4 +++ b/packages/malloy/src/lang/grammar/MalloyParser.g4 @@ -27,13 +27,6 @@ options { tokenVocab=MalloyLexer; } malloyDocument: (malloyStatement | SEMI)* EOF; -closeCurly - : CCURLY - // ANTLR VSCode plugin loses it's tiny mind if { } aren't matched - // even in the error string below - | { this.notifyErrorListeners("'{' missing a '}'"); } - ; - malloyStatement : defineSourceStatement | defineQuery @@ -133,12 +126,7 @@ connectionId : id; queryProperties - : filterShortcut - | OCURLY (queryStatement | SEMI)* closeCurly - ; - -filterShortcut - : OCURLY QMARK fieldExpr closeCurly + : OCURLY (queryStatement | SEMI)* closeCurly ; queryName : id; @@ -169,7 +157,6 @@ sourceNameDef: id; exploreProperties : OCURLY (exploreStatement | SEMI)* closeCurly - | filterShortcut ; exploreStatement @@ -331,7 +318,7 @@ queryExtendStatement ; queryExtendStatementList - : OCURLY (queryExtendStatement | SEMI)* closeCurly + : OCURLY (queryExtendStatement | SEMI)* CCURLY ; joinList @@ -688,7 +675,7 @@ starQualified : OCURLY ( (EXCEPT fieldNameList) | COMMA - )+ closeCurly + )+ CCURLY ; taggedRef @@ -726,3 +713,13 @@ debugPartial: partialAllowedFieldExpr EOF; experimentalStatementForTesting // this only exists to enable tests for the experimental compiler flag : SEMI SEMI OBRACK string CBRACK ; + +// Try to show a nice error for a missing }. Only use this when the next +// legal symbols after the curly are things which would be illegal inside +// the curly brackets. +closeCurly + : CCURLY + // ANTLR VSCode plugin loses it's tiny mind if { } aren't matched + // even in the error string below + | { this.notifyErrorListeners("'{' missing a '}'"); } + ; \ No newline at end of file diff --git a/packages/malloy/src/lang/malloy-to-ast.ts b/packages/malloy/src/lang/malloy-to-ast.ts index 5accfd87f..b26ddf7ef 100644 --- a/packages/malloy/src/lang/malloy-to-ast.ts +++ b/packages/malloy/src/lang/malloy-to-ast.ts @@ -282,16 +282,6 @@ export class MalloyToAST ); } - protected getFilterShortcut(cx: parse.FilterShortcutContext): ast.Filter { - const el = this.getFilterElement(cx.fieldExpr()); - this.m4advisory( - cx, - 'filter-shortcut', - 'Filter shortcut `{? condition }` is deprecated; use `{ where: condition } instead' - ); - return new ast.Filter([el]); - } - protected getPlainStringFrom(cx: HasString): string { const [result, errors] = getPlainString(cx); for (const error of errors) { @@ -444,16 +434,12 @@ export class MalloyToAST } visitExploreProperties(pcx: parse.ExplorePropertiesContext): ast.SourceDesc { - const filterCx = pcx.filterShortcut(); const visited = this.only( pcx.exploreStatement().map(ecx => this.visit(ecx)), x => ast.isSourceProperty(x) && x, 'source property' ); const propList = new ast.SourceDesc(visited); - if (filterCx) { - propList.push(this.getFilterShortcut(filterCx)); - } return propList; } @@ -848,10 +834,6 @@ export class MalloyToAST x => ast.isQueryProperty(x) && x, 'query statement' ); - const fcx = pcx.filterShortcut(); - if (fcx) { - qProps.push(this.getFilterShortcut(fcx)); - } return new ast.QOpDesc(qProps); } diff --git a/packages/malloy/src/lang/parse-log.ts b/packages/malloy/src/lang/parse-log.ts index a8128049b..245617463 100644 --- a/packages/malloy/src/lang/parse-log.ts +++ b/packages/malloy/src/lang/parse-log.ts @@ -336,7 +336,6 @@ type MessageParameterTypes = { 'foreign-key-in-join-cross': string; 'expression-type-error': string; 'unexpected-statement-in-translation': string; - 'filter-shortcut': string; 'illegal-query-interpolation-outside-sql-block': string; 'percent-terminated-query-interpolation': string; 'failed-to-parse-time-literal': string; diff --git a/packages/malloy/src/lang/test/query.spec.ts b/packages/malloy/src/lang/test/query.spec.ts index bce500906..b312fd0f8 100644 --- a/packages/malloy/src/lang/test/query.spec.ts +++ b/packages/malloy/src/lang/test/query.spec.ts @@ -113,7 +113,7 @@ describe('query:', () => { }); test('query with shortcut filtered turtle', () => { expect(`##! -m4warnings - query: allA is ab -> aturtle + {? astr ~ 'a%' }`).toTranslate(); + query: allA is ab -> aturtle + {where: astr ~ 'a%' }`).toTranslate(); }); test('query with filtered turtle', () => { expect( diff --git a/packages/malloy/src/lang/test/source.spec.ts b/packages/malloy/src/lang/test/source.spec.ts index 7f1a941d0..9b3ad5c92 100644 --- a/packages/malloy/src/lang/test/source.spec.ts +++ b/packages/malloy/src/lang/test/source.spec.ts @@ -41,16 +41,6 @@ describe('source:', () => { "source: xA is _db_.table('aTable') extend { where: astr ~ 'a%' }" ).toTranslate(); }); - test('shorcut fitlered table m4warning', () => { - expect(` - ##! m4warnings=warn - source: xA is _db_.table('aTable') extend {? astr ~ 'a%' } - `).toLog( - warningMessage( - 'Filter shortcut `{? condition }` is deprecated; use `{ where: condition } instead' - ) - ); - }); test('fitlered table', () => { expect( "source: testA is _db_.table('aTable') extend { where: astr ~ 'a%' }" @@ -558,18 +548,6 @@ describe('source:', () => { } `).toTranslate(); }); - test('refined explore-query m4warning', () => { - expect(` - ##! m4warnings=warn - source: abNew is ab extend { - view: for1 is aturtle + {? ai = 1 } - } - `).toLog( - warningMessage( - 'Filter shortcut `{? condition }` is deprecated; use `{ where: condition } instead' - ) - ); - }); test('chained explore-query', () => { expect(` source: c is a extend { diff --git a/scripts/ci-test-sanity-check.sh b/scripts/ci-test-sanity-check.sh new file mode 100644 index 000000000..b4223370e --- /dev/null +++ b/scripts/ci-test-sanity-check.sh @@ -0,0 +1,22 @@ +#! /bin/bash + +# This tests to make sure the suite of tests run in CI contains all the +# tests in the source tree. + +MALLOY_ROOT=$(cd $(dirname $0)/..; pwd) +all_test_file=/tmp/mly_all_test.$$ +ci_test_file=/tmp/mly_ci_test.$$ +cd $MALLOY_ROOT +npx jest --listTests | sort > $all_test_file +for ci_test in jest.*.config.ts; do + npx jest --config $ci_test --listTests >> $ci_test_file +done +sort -u $ci_test_file -o $ci_test_file +diff $all_test_file $ci_test_file +status=$? +rm -rf $all_test_file $ci_test_file +if [ $status -ne 0 ]; then + echo "!!!!!!! jest.config.* CI configurations must up dated, some tests are missing !!!!!!!!" + exit 1; +fi +exit 0 diff --git a/test/config/core.config.json b/test/config/core.config.json new file mode 100644 index 000000000..e3413994d --- /dev/null +++ b/test/config/core.config.json @@ -0,0 +1,8 @@ +{ + "roots": [ + "../../packages/malloy-malloy-sql/", + "../../packages/malloy-syntax-highlight/", + "../../packages/malloy/", + "../../test/src/core/" + ] +} diff --git a/test/src/api.spec.ts b/test/src/core/api.spec.ts similarity index 98% rename from test/src/api.spec.ts rename to test/src/core/api.spec.ts index 85a673ac3..e08d4d896 100644 --- a/test/src/api.spec.ts +++ b/test/src/core/api.spec.ts @@ -22,8 +22,8 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -import {runtimeFor} from './runtimes'; -import './util/db-jest-matchers'; +import {runtimeFor} from '../runtimes'; +import '../util/db-jest-matchers'; const runtime = runtimeFor('duckdb'); diff --git a/test/src/dependencies.spec.ts b/test/src/core/dependencies.spec.ts similarity index 100% rename from test/src/dependencies.spec.ts rename to test/src/core/dependencies.spec.ts diff --git a/test/src/events.spec.ts b/test/src/core/events.spec.ts similarity index 98% rename from test/src/events.spec.ts rename to test/src/core/events.spec.ts index b422e1ccf..b1bbfe766 100644 --- a/test/src/events.spec.ts +++ b/test/src/core/events.spec.ts @@ -5,8 +5,8 @@ * LICENSE file in the root directory of this source tree. */ -import {runtimeFor} from './runtimes'; -import './util/db-jest-matchers'; +import {runtimeFor} from '../runtimes'; +import '../util/db-jest-matchers'; const runtime = runtimeFor('duckdb'); diff --git a/test/src/experimental-dialects.spec.ts b/test/src/core/experimental-dialects.spec.ts similarity index 97% rename from test/src/experimental-dialects.spec.ts rename to test/src/core/experimental-dialects.spec.ts index 8459034e1..52e7c34e4 100644 --- a/test/src/experimental-dialects.spec.ts +++ b/test/src/core/experimental-dialects.spec.ts @@ -28,8 +28,8 @@ import { SQLSourceDef, registerDialect, } from '@malloydata/malloy'; -import {testRuntimeFor} from './runtimes'; -import './util/db-jest-matchers'; +import {testRuntimeFor} from '../runtimes'; +import '../util/db-jest-matchers'; import {DuckDBConnection} from '@malloydata/db-duckdb'; const envDatabases = ( diff --git a/test/src/jestMatcher.spec.ts b/test/src/core/jestMatcher.spec.ts similarity index 97% rename from test/src/jestMatcher.spec.ts rename to test/src/core/jestMatcher.spec.ts index ab95eabca..d42844efa 100644 --- a/test/src/jestMatcher.spec.ts +++ b/test/src/core/jestMatcher.spec.ts @@ -22,8 +22,8 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -import {runtimeFor} from './runtimes'; -import './util/db-jest-matchers'; +import {runtimeFor} from '../runtimes'; +import '../util/db-jest-matchers'; const runtime = runtimeFor('duckdb'); diff --git a/test/src/databases/multi-connection/multi_connection.spec.ts b/test/src/core/multi_connection.spec.ts similarity index 96% rename from test/src/databases/multi-connection/multi_connection.spec.ts rename to test/src/core/multi_connection.spec.ts index 72990d3ad..8bf72f089 100644 --- a/test/src/databases/multi-connection/multi_connection.spec.ts +++ b/test/src/core/multi_connection.spec.ts @@ -26,8 +26,8 @@ import * as malloy from '@malloydata/malloy'; import {EmptyURLReader} from '@malloydata/malloy'; -import {DuckDBTestConnection, PostgresTestConnection} from '../../runtimes'; -import {describeIfDatabaseAvailable} from '../../util'; +import {DuckDBTestConnection, PostgresTestConnection} from '../runtimes'; +import {describeIfDatabaseAvailable} from '../util'; const [, databases] = describeIfDatabaseAvailable(['duckdb', 'postgres']); diff --git a/test/src/model/sql_source.spec.ts b/test/src/core/sql_source.spec.ts similarity index 100% rename from test/src/model/sql_source.spec.ts rename to test/src/core/sql_source.spec.ts diff --git a/test/src/databases/streaming/streaming.spec.ts b/test/src/core/streaming.spec.ts similarity index 97% rename from test/src/databases/streaming/streaming.spec.ts rename to test/src/core/streaming.spec.ts index 26b19e7c8..0c815f83a 100644 --- a/test/src/databases/streaming/streaming.spec.ts +++ b/test/src/core/streaming.spec.ts @@ -27,8 +27,8 @@ import { JSONWriter, WriteStream, } from '@malloydata/malloy'; -import {RuntimeList} from '../../runtimes'; -import {describeIfDatabaseAvailable} from '../../util'; +import {RuntimeList} from '../runtimes'; +import {describeIfDatabaseAvailable} from '../util'; class StringAccumulator implements WriteStream { public accumulatedValue = ''; diff --git a/test/src/tags.spec.ts b/test/src/core/tags.spec.ts similarity index 99% rename from test/src/tags.spec.ts rename to test/src/core/tags.spec.ts index 0be09d1da..21f870deb 100644 --- a/test/src/tags.spec.ts +++ b/test/src/core/tags.spec.ts @@ -22,7 +22,7 @@ */ import {TagDict, Tag} from '@malloydata/malloy'; -import {runtimeFor} from './runtimes'; +import {runtimeFor} from '../runtimes'; declare global { // eslint-disable-next-line @typescript-eslint/no-namespace diff --git a/test/src/databases/bigquery/malloy_query.spec.ts b/test/src/databases/bigquery/malloy_query.spec.ts index 50cb83d77..e63ba367b 100644 --- a/test/src/databases/bigquery/malloy_query.spec.ts +++ b/test/src/databases/bigquery/malloy_query.spec.ts @@ -32,7 +32,7 @@ import '../../util/db-jest-matchers'; const runtimeList = new RuntimeList(['bigquery']); const runtime = runtimeList.runtimeMap.get('bigquery'); if (runtime === undefined) { - throw new Error("Couldn't build runtime"); + throw new Error('BigQuery runtime not found'); } const bq = runtime.connection as BigQueryTestConnection; diff --git a/test/src/databases/bigquery-duckdb/nested_source_table.spec.ts b/test/src/databases/bigquery/nested_source_table.spec.ts similarity index 100% rename from test/src/databases/bigquery-duckdb/nested_source_table.spec.ts rename to test/src/databases/bigquery/nested_source_table.spec.ts diff --git a/test/src/databases/duckdb/duckdb.spec.ts b/test/src/databases/duckdb-all/duckdb.spec.ts similarity index 100% rename from test/src/databases/duckdb/duckdb.spec.ts rename to test/src/databases/duckdb-all/duckdb.spec.ts diff --git a/test/src/databases/duckdb/materialization.spec.ts b/test/src/databases/duckdb-all/materialization.spec.ts similarity index 100% rename from test/src/databases/duckdb/materialization.spec.ts rename to test/src/databases/duckdb-all/materialization.spec.ts diff --git a/test/src/databases/duckdb/nested_source_table.spec.ts b/test/src/databases/duckdb/nested_source_table.spec.ts new file mode 100644 index 000000000..605d5c1c4 --- /dev/null +++ b/test/src/databases/duckdb/nested_source_table.spec.ts @@ -0,0 +1,226 @@ +/* + * Copyright 2023 Google LLC + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +/* eslint-disable no-console */ + +import '../../util/db-jest-matchers'; +import {RuntimeList} from '../../runtimes'; +import {describeIfDatabaseAvailable} from '../../util'; + +// No prebuilt shared model, each test is complete. Makes debugging easier. + +const [describe, databases] = describeIfDatabaseAvailable([ + 'bigquery', + 'duckdb', +]); + +function modelText(databaseName: string) { + return ` +source: ga_sessions is ${databaseName}.table('malloytest.ga_sample') extend { + + measure: + user_count is count(fullVisitorId) + session_count is count() + total_visits is totals.visits.sum() + total_hits is totals.hits.sum() + total_page_views is totals.pageviews.sum() + t2 is totals.pageviews.sum() + total_productRevenue is hits.product.productRevenue.sum() + hits_count is hits.count() + sold_count is hits.count() { where: hits.product.productQuantity > 0 } + + view: by_source is { + where: trafficSource.\`source\` != '(direct)' + group_by: trafficSource.\`source\` + aggregate: hits_count + limit: 10 + } + view: by_adContent_bar_chart is { + group_by: device.browser + aggregate: user_count + group_by: device.deviceCategory + } + view: by_region is { + where: geoNetwork.region !~ '%demo%' + group_by: geoNetwork.region + aggregate: user_count + limit: 10 + } + view: by_device is { + group_by: device.browser + aggregate: user_count + group_by: device.deviceCategory + } + view: by_category is { + group_by: category is hits.product.v2ProductCategory + aggregate: total_productRevenue + aggregate: sold_count + limit: 10 + } + view: by_hour_of_day is { + // group_by: gsession_hour is hour(start_time::timestamp) + aggregate: session_count + order_by: 1 + } + + view: page_load_times is { + group_by: hits.page.pageTitle + aggregate: hit_count is hits.count() + nest: load_bar_chart is { + group_by: hit_seconds is floor(hits.latencyTracking.pageLoadTime / 2) * 2 + aggregate: hits_count + } + limit: 10 + } + + view: by_page_title is { where: totals.transactionRevenue > 0 + group_by: hits.page.pageTitle + aggregate: hits_count + aggregate: sold_count + } + + view: by_all is { + nest: by_source + nest: by_adContent_bar_chart + nest: by_region + nest: by_category + } + + view: search_index is { + index: *, hits.*, customDimensions.* totals.*, trafficSource.*, hits.product.* + sample: 1% + } +} + +query: sessions_dashboard is ga_sessions -> { + nest: + by_region + by_device + by_source + by_category is { + group_by: category is hits.product.v2ProductCategory + aggregate: total_productRevenue + aggregate: sold_count + } +} +`; +} + +const runtimes = new RuntimeList(databases); +describe.each(runtimes.runtimeList)( + 'Nested Source Table - %s', + (databaseName, runtime) => { + const gaModel = runtime.loadModel(modelText(databaseName)); + test(`repeated child of record - ${databaseName}`, async () => { + await expect('run: ga_sessions->by_page_title').malloyResultMatches( + gaModel, + {pageTitle: 'Shopping Cart'} + ); + }); + + // Tests intermittently fail and lloyd said + // "I bet it has to do with my sampling test on indexing. Intermittently + // getting different results. I'd comment out the test and I'll take a + // look at it when I get back." and that seems reasonable. + // "search_index" is the test that failed, but a skipped both + // this one and "manual index". Here's the errror: + // * Nested Source Table › search_index - duckdb + // + // expect(received).toBe(expected) // Object.is equality + // + // Expected: "Organic Search" + // Received: "Referral" + test(`search_index - ${databaseName}`, async () => { + await expect(` + run: ga_sessions->search_index -> { + where: fieldName != null + select: * + order_by: fieldName, weight desc + limit: 10 + } + `).malloyResultMatches(gaModel, { + // fieldName: 'channelGrouping', + // fieldValue: 'Organic Search', + // weight: 10, + }); + }); + + test(`manual index - ${databaseName}`, async () => { + let sampleSize = '10'; + if (databaseName === 'bigquery') { + sampleSize = 'false'; + } + await expect(` + run: ${databaseName}.table('malloytest.ga_sample')-> { + index: * + sample: ${sampleSize} + } + -> { + aggregate: field_count is count(fieldName) + nest: top_fields is { + group_by: fieldName + aggregate: row_count is count() + limit: 100 + } + } + `).malloyResultMatches(runtime, { + // 'top_fields.fieldName': 'channelGrouping', + // 'top_fields.fieldValue': 'Organic Search', + // 'top_fields.weight': 18, + }); + }); + + test(`autobin - ${databaseName}`, async () => { + await expect(` + source: airports is ${databaseName}.table('malloytest.airports') extend { + measure: airport_count is count() + view: by_elevation is { + aggregate: bin_size is NULLIF((max(elevation) - min(elevation))/30,0) + nest: data is { + group_by: elevation + aggregate: row_count is count() + } + } + -> { + group_by: elevation is floor(data.elevation/bin_size)*bin_size + bin_size/2 + aggregate: airport_count is data.row_count.sum() + order_by: elevation + } + } + + run: airports -> { + group_by: state is state + aggregate: airport_count + nest: by_elevation_bar_chart is by_elevation + } + `).malloyResultMatches(runtime, { + // don't know what to expect ... + }); + }); + } +); + +afterAll(async () => { + await runtimes.closeAll(); +});