Make CI Faster (#2071)

* shuffle tests around for faster ci maybe

* fix core files

* fix duckdb split

* maybe fix motherduck?

* gteduckdb-other name correct

* add comment back

* get names right

* fix tests on local duckdbs

* changes test-core back to test-all

* tweak file order

* add test for ci sanity

* remove forced failure

* add bigquery and postgres credentials to core tests

* oops

* fix gts lint warning

* start postgres for core

* keep trying to get bigquery auth in core

* Add comment to core job

* spling

* add more googlue auth

* comment on order

* populate postgres in core

Author: mtoy-googly-moogly

.github/workflows/db-bigquery.yaml

@@ -1,4 +1,4 @@
-name: BigQuery DB
+name: "DB:BigQuery"
 
 on:
   pull_request:
@@ -31,7 +31,7 @@ jobs:
         run: |
           npm ci --loglevel error
           npm run build
-          npm run test-silent -- -- test packages/malloy-db-bigquery
+          npx jest --reporters jest-silent-reporter --reporters summary --config jest.bigquery.config.ts
         env:
           CI: true
           MALLOY_DATABASES: bigquery

.github/workflows/db-duckdb-wasm.yaml

@@ -0,0 +1,29 @@
+name: "DB:DuckDB(WASM)"
+
+on: [pull_request, workflow_call]
+
+jobs:
+  test-duckdb-wasm:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [18.x]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'true'
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: npm install, build, and test
+        run: |
+          npm ci --loglevel error
+          npm run build
+          npm run build-duckdb-db
+          npx jest --reporters jest-silent-reporter --reporters summary --config jest.duckdb-other.config.ts --runInBand
+        env:
+          CI: true
+          MALLOY_DATABASES: duckdb_wasm

.github/workflows/db-duckdb.yaml

@@ -0,0 +1,29 @@
+name: "DB:DuckDB"
+
+on: [pull_request, workflow_call]
+
+jobs:
+  test-duckdb:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [18.x]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'true'
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: npm install, build, and test
+        run: |
+          npm ci --loglevel error
+          npm run build
+          npm run build-duckdb-db
+          npx jest --reporters jest-silent-reporter --reporters summary --config jest.duckdb.config.ts --runInBand
+        env:
+          CI: true
+          MALLOY_DATABASES: duckdb

.github/workflows/db-motherduck.yaml

@@ -1,4 +1,4 @@
-name: MotherDuck DB
+name: "DB:MotherDuck"
 
 on:
   pull_request:
@@ -27,7 +27,7 @@ jobs:
         run: |
           npm ci --loglevel error
           npm run build
-          npm run test-silent -- -- test packages/malloy-duckdb
+          npx jest --reporters jest-silent-reporter --reporters summary --config jest.duckdb-other.config.ts
         env:
           CI: true
           MALLOY_DATABASES: motherduck

.github/workflows/db-mysql.yaml

@@ -1,4 +1,4 @@
-name: MySQL DB
+name: "DB:MySQL"
 
 on: [pull_request, workflow_call]
 
@@ -25,7 +25,7 @@ jobs:
           npm run build
           npm run build-duckdb-db
           ./test/mysql/mysql_start.sh
-          npm run test-silent -- -- test
+          npx jest --reporters jest-silent-reporter --reporters summary --config jest.mysql.config.ts
           ./test/mysql/mysql_stop.sh
         env:
           MALLOY_DATABASES: mysql

.github/workflows/db-postgres.yaml

@@ -1,4 +1,4 @@
-name: Postgres DB
+name: "DB:Postgres"
 
 on: [pull_request, workflow_call]
 
@@ -39,10 +39,7 @@ jobs:
           npm run build
           echo CREATE EXTENSION tsm_system_rows\; | psql
           gunzip -c test/data/postgres/malloytest-postgres.sql.gz | psql
-          npm run test-silent -- -- test packages/malloy-db-postgres
-          export MALLOY_DATABASES=duckdb,postgres
-          npm run build-duckdb-db
-          npm run test-silent -- -- test/src/databases/multi-connection/multi_connection.spec.ts
+          npx jest --reporters jest-silent-reporter --reporters summary --config jest.postgres.config.ts
         env:
           MALLOY_DATABASES: postgres
           PGHOST: localhost

.github/workflows/db-presto.yaml

@@ -1,4 +1,4 @@
-name: Presto DB
+name: "DB:Presto"
 
 on:
   pull_request:
@@ -30,8 +30,7 @@ jobs:
           npm run build
           npm run build-duckdb-db
           ./test/presto/presto_start.sh
-          npm run test-silent -- -- packages/malloy-db-trino
-          npm run test-silent -- -- test
+          npx jest --reporters jest-silent-reporter --reporters summary --config jest.presto.config.ts
           ./test/presto/presto_stop.sh
         env:
           MALLOY_DATABASES: presto

.github/workflows/db-snowflake.yaml

@@ -1,4 +1,4 @@
-name: Snowflake DB
+name: "DB:Snowflake"
 
 on:
   pull_request:
@@ -28,7 +28,7 @@ jobs:
           npm ci --loglevel error
           npm run build
           ./scripts/gen-snowflake-auth.sh
-          npm run test-silent -- -- test packages/malloy-db-snowflake
+          npx jest --reporters jest-silent-reporter --reporters summary --config jest.snowflake.config.ts
         env:
           CI: true
           MALLOY_DATABASES: snowflake

.github/workflows/db-trino.yaml

@@ -1,4 +1,4 @@
-name: Trino DB
+name: "DB:Trino"
 
 on:
   pull_request:
@@ -30,8 +30,7 @@ jobs:
           npm run build
           npm run build-duckdb-db
           ./test/trino/trino_start.sh
-          npm run test-silent -- -- packages/malloy-db-trino
-          npm run test-silent -- -- test
+          npx jest --reporters jest-silent-reporter --reporters summary --config jest.trino.config.ts
           ./test/trino/trino_stop.sh
         env:
           MALLOY_DATABASES: trino

.github/workflows/main.yaml

@@ -1,6 +1,15 @@
+# The "Core" job runs through all the tests which are not dialect specific.
+# This includes tests which don't touch the database at all, and tests
+# which touch some combination of duckdb,bigquery,postgres but do
+# not need to be once times for each dialect.
 name: Core
 
-on: [pull_request, workflow_call]
+on:
+  pull_request:
+  workflow_call:
+    secrets:
+      BIGQUERY_KEY:
+        required: true
 
 jobs:
   test-all:
@@ -10,6 +19,20 @@ jobs:
       matrix:
         node-version: [18.x]
 
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: root
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
     steps:
       - uses: actions/checkout@v4
         with:
@@ -18,13 +41,23 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
+      - name: GCloud auth
+        uses: 'google-github-actions/auth@v2'
+        with:
+          credentials_json: '${{ secrets.BIGQUERY_KEY }}'
       - name: npm install, build, and test
         run: |
           npm ci --loglevel error
+          sh scripts/ci-test-sanity-check.sh
           npm run lint
           npm run build
           npm run build-duckdb-db
-          npm run test-silent
+          echo CREATE EXTENSION tsm_system_rows\; | psql
+          gunzip -c test/data/postgres/malloytest-postgres.sql.gz | psql
+          npx jest --reporters jest-silent-reporter --reporters summary --config jest.core.config.ts --runInBand
         env:
           CI: true
-          MALLOY_DATABASES: duckdb,duckdb_wasm
+          PGHOST: localhost
+          PGPORT: 5432
+          PGUSER: root
+          PGPASSWORD: postgres

.github/workflows/run-tests.yaml

@@ -3,6 +3,20 @@ name: Malloy Tests
 on: [workflow_dispatch]
 
 jobs:
+  main:
+    uses: './.github/workflows/main.yaml'
+    secrets:
+      BIGQUERY_KEY: ${{ secrets.BIGQUERY_KEY }}
+  db-trino:
+    uses: './.github/workflows/db-trino.yaml'
+    secrets:
+      BQ_PRESTO_TRINO_KEY: ${{ secrets.BQ_PRESTO_TRINO_KEY }}
+  db-presto:
+    uses: './.github/workflows/db-presto.yaml'
+    secrets:
+      BQ_PRESTO_TRINO_KEY: ${{ secrets.BQ_PRESTO_TRINO_KEY }}
+  db-duckdb:
+     uses: './github/workflows/db-duckdb.yaml'
   db-bigquery:
     uses: './.github/workflows/db-bigquery.yaml'
     secrets:
@@ -11,35 +25,33 @@ jobs:
     uses: './.github/workflows/db-motherduck.yaml'
     secrets:
       MOTHERDUCK_TOKEN_10: ${{ secrets.MOTHERDUCK_TOKEN_10 }}
-  db-mysql:
-    uses: './.github/workflows/db-mysql.yaml'
   db-postgres:
     uses: './.github/workflows/db-postgres.yaml'
-  db-presto:
-    uses: './.github/workflows/db-presto.yaml'
-    secrets:
-      BQ_PRESTO_TRINO_KEY: ${{ secrets.BQ_PRESTO_TRINO_KEY }}
   db-snowflake:
     uses: './.github/workflows/db-snowflake.yaml'
     secrets:
       SNOWFLAKE_CONNECTION: ${{ secrets.SNOWFLAKE_CONNECTION }}
-  db-trino:
-    uses: './.github/workflows/db-trino.yaml'
-    secrets:
-      BQ_PRESTO_TRINO_KEY: ${{ secrets.BQ_PRESTO_TRINO_KEY }}
-  main:
-    uses: './.github/workflows/main.yaml'
+  db-mysql:
+    uses: './.github/workflows/db-mysql.yaml'
+  db-duckdb-wasm:
+     uses: './github/workflows/db-duckdb-wasm.yaml'
 
+
+  # I think I have the sorted roughly longest to shortest
+  # so the longer running jobs get wrokers sooner, not sure
+  # that is the right plan
   malloy-tests:
     needs:
+      - main
+      - db-snowflake
       - db-bigquery
-      - db-motherduck
-      - db-mysql
-      - db-postgres
+      - db-duckdb
       - db-presto
-      - db-snowflake
+      - db-duckdb-wasm
       - db-trino
-      - main
+      - db-postgres
+      - db-motherduck
+      - db-mysql
     runs-on: ubuntu-latest
     steps:
       - name: Success

jest.bigquery.config.ts

@@ -0,0 +1,8 @@
+module.exports = {
+  ...require('./jest.config.ts'),
+  roots: [
+    '<rootDir>/packages/malloy-db-bigquery/src/',
+    '<rootDir>/test/src/databases/all/',
+    '<rootDir>/test/src/databases/bigquery/',
+  ],
+};

jest.core.config.ts

@@ -0,0 +1,10 @@
+module.exports = {
+  ...require('./jest.config.ts'),
+  roots: [
+    '<rootDir>/packages/malloy-malloy-sql/',
+    '<rootDir>/packages/malloy-syntax-highlight/',
+    '<rootDir>/packages/malloy/',
+    '<rootDir>/test/src/core/',
+    '<rootDir>/test/src/render/',
+  ],
+};

jest.duckdb-other.config.ts

@@ -0,0 +1,9 @@
+// For motherduck and duckdb-wasm
+module.exports = {
+  ...require('./jest.config.ts'),
+  roots: [
+    '<rootDir>/packages/malloy-db-duckdb/',
+    '<rootDir>/test/src/databases/all/',
+    '<rootDir>/test/src/databases/duckdb-all/',
+  ],
+};

jest.duckdb.config.ts

@@ -0,0 +1,9 @@
+module.exports = {
+  ...require('./jest.config.ts'),
+  roots: [
+    '<rootDir>/packages/malloy-db-duckdb/',
+    '<rootDir>/test/src/databases/all/',
+    '<rootDir>/test/src/databases/duckdb/',
+    '<rootDir>/test/src/databases/duckdb-all/',
+  ],
+};

jest.mysql.config.ts

@@ -0,0 +1,4 @@
+module.exports = {
+  ...require('./jest.config.ts'),
+  roots: ['<rootDir>/test/src/databases/all/'],
+};

jest.postgres.config.ts

@@ -0,0 +1,8 @@
+module.exports = {
+  ...require('./jest.config.ts'),
+  roots: [
+    '<rootDir>/packages/malloy-db-postgres/',
+    '<rootDir>/test/src/databases/all/',
+    '<rootDir>/test/src/databases/postgres/',
+  ],
+};

jest.presto.config.ts

@@ -0,0 +1,7 @@
+module.exports = {
+  ...require('./jest.config.ts'),
+  roots: [
+    '<rootDir>/test/src/databases/all/',
+    '<rootDir>/test/src/databases/presto-trino/',
+  ],
+};

jest.snowflake.config.ts

@@ -0,0 +1,7 @@
+module.exports = {
+  ...require('./jest.config.ts'),
+  roots: [
+    '<rootDir>/packages/malloy-db-snowflake/',
+    '<rootDir>/test/src/databases/all/',
+  ],
+};

jest.trino.config.ts

@@ -0,0 +1,8 @@
+module.exports = {
+  ...require('./jest.config.ts'),
+  roots: [
+    '<rootDir>/packages/malloy-db-trino/src/',
+    '<rootDir>/test/src/databases/all/',
+    '<rootDir>/test/src/databases/presto-trino/',
+  ],
+};