Pavanka snowflake malloy (#1634)

* [snowflake] add snowflake connector for malloy

* Update tsconfig files and lint fix

* * Explicit upload script
* add a --debug flag in models so you can get the data and query from a test
* Change identifier naming strategy to always quote for case sensitivity
* All SQL tests need to use uppercase variable names to work on all dialect
* make sure we quote identifiers everywhere.

* add support for nesting/unnesting.

* Set up Snowflake CI environment (#1637)

* GHA for Snowflake (#1638)

* Try GHA for Snowflake

* Bump action versions

* change quoting strategy
fix bugs.

* missing commas

* one more test

* one more bug.

* Reset package-lock.json

* Remove unnecessary polyfills

* Lint fix

* Fix merge

---------

Co-authored-by: Pavan Kumar <pavan@datairis.io>
Co-authored-by: lloyd tabb <ltabb@google.com>
Co-authored-by: Will Scullin <willscullin@google.com>

.github/workflows/db-snowflake.yaml

@@ -0,0 +1,34 @@
+name: Snowflake DB
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [18.x]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'true'
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: npm install, build, and test
+        run: |
+          npm ci --loglevel error
+          npm run build
+          npm run test-silent
+        env:
+          CI: true
+          MALLOY_DATABASES: snowflake
+          SNOWFLAKE_ACCOUNT: pt58362-rx92781
+          SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
+          SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+          SNOWFLAKE_WAREHOUSE: LOOKER_WH
+          SNOWFLAKE_DATABASE: MALLOYTEST
+          SNOWFLAKE_SCHEMA: PUBLIC

cloudbuild/build-db/cloudbuild.yaml

@@ -0,0 +1,38 @@
+steps:
+  - id: nix-quiet-install
+    name: gcr.io/cloud-builders/docker
+    args:
+      - pull
+      - '-q'
+      - nixos/nix:2.15.2
+  - id: proxy-install
+    name: 'nixos/nix:2.15.2'
+    entrypoint: sh
+    args:
+      - -c
+      - 'wget -q -O /workspace/cloud_sql_proxy https://dl.google.com/cloudsql/cloud_sql_proxy.linux.386 && chmod +x /workspace/cloud_sql_proxy'
+    waitFor: ['nix-quiet-install']
+  - id: run-script
+    name: 'nixos/nix:2.15.2'
+    env:
+      - 'SNOWFLAKE_ACCOUNT=pt58362-rx92781'
+      - 'SNOWFLAKE_WAREHOUSE=LOOKER_WH'
+      - 'SNOWFLAKE_DATABASE=MALLOYTEST'
+      - 'SNOWFLAKE_SCHEMA=PUBLIC'
+    secretEnv: ['SNOWFLAKE_USER', 'SNOWFLAKE_PASSWORD']
+    entrypoint: sh
+    args:
+      - -c
+      - ./cloudbuild/build-db/$TRIGGER_NAME.sh
+    waitFor: ['proxy-install']
+    timeout: '1800s'
+timeout: '1800s'
+options:
+  machineType: 'E2_HIGHCPU_32'
+  logging: CLOUD_LOGGING_ONLY
+availableSecrets:
+  secretManager:
+    - versionName: projects/malloy-303216/secrets/snowflake-user/versions/latest
+      env: 'SNOWFLAKE_USER'
+    - versionName: projects/malloy-303216/secrets/snowflake-password/versions/latest
+      env: 'SNOWFLAKE_PASSWORD'

cloudbuild/build-db/test-snowflake.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env sh
+set -euxo pipefail
+
+nix-shell \
+  --quiet \
+  --pure \
+  --keep SNOWFLAKE_ACCOUNT \
+  --keep SNOWFLAKE_USER \
+  --keep SNOWFLAKE_PASSWORD \
+  --keep SNOWFLAKE_WAREHOUSE \
+  --keep SNOWFLAKE_DATABASE \
+  --keep SNOWFLAKE_SCHEMA \
+  --command "$(cat <<NIXCMD
+  set -euxo pipefail
+  npm ci --loglevel error
+  npm run build
+  MALLOY_DATABASES=snowflake npm run test-silent
+NIXCMD
+)"

cloudbuild/build-test/test.sh

@@ -1,9 +1,19 @@
 #!/usr/bin/env sh
 set -euxo pipefail
 
-nix-shell --quiet --pure --keep PGHOST --keep PGDATABASE --keep PGUSER --command "$(cat <<NIXCMD
+nix-shell \
+  --quiet \
+  --pure \
+  --keep PGHOST \
+  --keep PGDATABASE \
+  --keep PGUSER \
+  --command "$(cat <<NIXCMD
   set -euxo pipefail
   npm ci --loglevel error
-  npm run lint && npm run build && npm run build-duckdb-db && npm run test-silent
+  export MALLOY_DATABASES=postgres,bigquery,duckdb,duckdb_wasm
+  npm run lint
+  npm run build
+  npm run build-duckdb-db
+  npm run test-silent
 NIXCMD
 )"

cloudbuild/deploy/malloy-npm-publish.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euxo pipefail
 
-export PACKAGES="packages/malloy-interfaces packages/malloy packages/malloy-db-bigquery packages/malloy-db-duckdb packages/malloy-db-postgres packages/malloy-render packages/malloy-malloy-sql test packages/malloy-syntax-highlight"
+export PACKAGES="packages/malloy-interfaces packages/malloy packages/malloy-db-bigquery packages/malloy-db-duckdb packages/malloy-db-postgres packages/malloy-db-snowflake packages/malloy-render packages/malloy-malloy-sql test packages/malloy-syntax-highlight"
 
 nix-shell --pure --keep NPM_TOKEN --keep PACKAGES --command "$(cat <<NIXCMD
   set -euxo pipefail

cloudbuild/deploy/malloy-npm-release-publish.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euxo pipefail
 
-export PACKAGES="packages/malloy-interfaces packages/malloy packages/malloy-db-bigquery packages/malloy-db-duckdb packages/malloy-db-postgres packages/malloy-render packages/malloy-malloy-sql test packages/malloy-syntax-highlight"
+export PACKAGES="packages/malloy-interfaces packages/malloy packages/malloy-db-bigquery packages/malloy-db-duckdb packages/malloy-db-postgres packages/malloy-db-snowflake packages/malloy-render packages/malloy-malloy-sql test packages/malloy-syntax-highlight"
 
 nix-shell --pure --keep NPM_TOKEN --keep PACKAGES --keep BRANCH_NAME --command "$(cat <<NIXCMD
   set -euxo pipefail