Backport Cloud Features

.env.example

@@ -72,5 +72,5 @@ DEKART_STATIC_FILES=
 REACT_APP_CUSTOM_CODE=
 DEKART_DEV_CLAIMS_EMAIL=
 DEKART_DEV_REFRESH_TOKEN=
-DEKART_DOCKER_DEV_TAG= dekart-dev
-DEKART_DOCKER_E2E_TAG= dekart-e2e
+DEKART_DOCKER_DEV_TAG=dekart-dev
+DEKART_DOCKER_E2E_TAG=dekart-e2e

.github/pull_request_template.md

@@ -1,6 +1,3 @@
-resolves #xx
+## Contributor License Agreement
 
-<!---
-  Skip if not applicable.
-  Include the number of the issue addressed by this PR above if applicable.
--->
+- [ ] I hereby grant to the owners of this project repository a perpetual, worldwide, non-exclusive, royalty-free, irrevocable license to use, reproduce, modify, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute my contributions, in whole or in part, under any licenses, including the GNU Affero General Public License (AGPL) v3.0 and a [Commercial License](https://dekart.xyz/legal/dekart-premium-terms/).

.github/workflows/e2e.yaml

@@ -234,6 +234,8 @@ jobs:
             -e DEKART_SNOWFLAKE_ACCOUNT_ID=${{ secrets.SNOWFLAKE_ACCOUNT_ID }} \
             -e DEKART_SNOWFLAKE_USER=${{ secrets.SNOWFLAKE_USER }} \
             -e DEKART_SNOWFLAKE_PASSWORD=${{ secrets.SNOWFLAKE_PASSWORD }} \
+            -e DEKART_DEV_CLAIMS_EMAIL=test@gmail.com \
+            -e DEKART_REQUIRE_AMAZON_OIDC=1 \
             dekartxyz/dekart:${{ env.IMAGE_CACHE_KEY }}
       - name: Upload cypress artifacts
         if: failure()

.gitignore

@@ -45,4 +45,7 @@ cypress/screenshots
 
 # IDE
 .idea
-testtmp*
+testtmp*
+
+# .env
+.env.*

Makefile

@@ -1,10 +1,9 @@
-.PHONY: proto-build proto-docker proto docker docker-compose-up docker-compose-rm version minor patch
+.PHONY: proto-clean proto-build proto-docker proto nodetest docker-compose-up down cloudsql up-and-down
 
 # load .env
 # https://lithic.tech/blog/2020-05/makefile-dot-env
 ifneq (,$(wildcard ./.env))
     include .env
-    export
 endif
 
 UNAME := $(shell uname -m)
@@ -184,8 +183,6 @@ docker: # build docker for local use
 
 up-and-down:
 	docker-compose  --env-file .env --profile local up; docker-compose --env-file .env --profile local down --volumes
-
-
 up:
 	docker-compose  --env-file .env --profile local up
 
@@ -195,12 +192,24 @@ down:
 cloudsql:
 	docker-compose  --env-file .env --profile cloudsql up
 
-server:
+
+define run_server
+	@set -a; \
+	. $(1); \
+	set +a; \
 	go run ./src/server/main.go
+endef
+
+# Pattern rule to match any target starting with ".env."
+server-%:
+	$(call run_server,.env.$*)
+
+# Rule for the default .env file
+server:
+	$(call run_server,.env)
 
 npm:
 	npm i --legacy-peer-deps
-
 prerelease:
 	npm version prerelease --preid=rc
 preminor:

README.md

@@ -68,15 +68,9 @@ Dekart is named after French mathematician René Descartes (French: [ʁəne deka
 
 ## License
 
-## Licensing
+This project is open source under the GNU Affero General Public License Version 3 (AGPLv3) or any later version.
+Commercial licenses are available upon request, please contact us at [support@dekart.xyz](mailto:support@dekart.xyz).
 
-This project contains components licensed under different licenses:
+Copyright (c) 2024 Volodymyr Bilonenko
 
-- The majority of the project is licensed under the [MIT License](./LICENSE), which allows for permissive use, including in proprietary software.
-- Specific components, such as the `./src/server/storage`, `./src/server/snowflakejob/`, `./src/client/actions/grpc.js`, are licensed under the AGPLv3 License.
-
-
-### Commercial License
-
-For companies or users who wish to use the AGPL-licensed components in proprietary environments, a commercial license is available. Please visit [dekart.xyz](https://dekart.xyz) for more information.
 

cypress/e2e/athena/spec.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('basic query flow', () => {
   it('should make simple athena query and get ready status', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_athena_query, { force: true })
@@ -17,7 +17,7 @@ describe('basic query flow', () => {
 describe('cancelling query', () => {
   it('should cancels query', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_athena_query, { force: true })

cypress/e2e/bq/basicFlow.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('basic query flow', () => {
   it('should make simple bigquery query and get ready status', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_sql_query, { force: true })

cypress/e2e/bq/cancelQuery.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('cancelling query', () => {
   it('should cancels query', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_sql_query, { force: true })

cypress/e2e/bq/scripts.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('support bq scripts', () => {
   it('retrieve result for bigquery script', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.bigquery_script, { force: true })

cypress/e2e/bq/updateDataset.cy.js

@@ -5,7 +5,7 @@ describe('update dataset', () => {
   it('should persist kepler config when updating dataset', () => {
     // create report
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
 
     // first query
     cy.get('button:contains("Add data from...")').click()

cypress/e2e/google-oauth/testConnection.cy.js

@@ -6,7 +6,7 @@ describe('basic query flow', () => {
     cy.visit('/')
 
     // create connection
-    cy.get('button:contains("Create connection")').click()
+    cy.get('button:contains("BigQuery")').click()
     const randomConnectionName = `test-${Math.floor(Math.random() * 1000000)}`
     cy.get('div.ant-modal-title').should('contain', 'BigQuery')
     cy.get('input#connectionName').clear()

cypress/e2e/pg/cancelQuery.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('cancelling query', () => {
   it('should cancels query', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_pg_query_long, { force: true })

cypress/e2e/pg/happyPath.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('happy path', () => {
   it('should make simple postgres query and get ready status', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_pg_query, { force: true })

cypress/e2e/snowflake-s3/cancelQuery.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('cancelling query', () => {
   it('should cancels query', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_snowflake_query, { force: true })

cypress/e2e/snowflake-s3/emptyResult.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('empty path', () => {
   it('should complete query returning empty result', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type('SELECT ROUND(uniform(-90::float, 90::float, random()), 6) AS lat, ROUND(uniform(-180::float, 180::float, random()), 6) AS lon FROM TABLE(GENERATOR(ROWCOUNT => 0))', { force: true })

cypress/e2e/snowflake-s3/fork.cy.js

@@ -21,9 +21,8 @@ async function getColorAtMapCenter (win) {
 
 describe('fork', () => {
   it('should have same viz style after fork', () => {
-    let originalColor
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type('select 0 as lat, 0 as lon', { force: true })

cypress/e2e/snowflake-s3/happyPath.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('happy path', () => {
   it('should make simple snowflake query and get ready status', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_snowflake_query, { force: true })

cypress/e2e/snowflake-s3/spec.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('basic query flow', () => {
   it('should make simple snowflake query and get ready status', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_snowflake_query, { force: true })
@@ -18,7 +18,7 @@ describe('basic query flow', () => {
 describe('cancelling query', () => {
   it('should cancels query', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('button:contains("Add data from...")').click()
     cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_snowflake_query, { force: true })

cypress/e2e/snowflake/happyPath.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('happy path', () => {
   it('should make simple snowflake query and get ready status', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('textarea').type(copy.simple_snowflake_query, { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get(`span:contains("${copy.ready}")`, { timeout: 20000 }).should('be.visible')

cypress/e2e/snowflake/runAllQueries.cy.js

@@ -4,7 +4,7 @@ import copy from '../../fixtures/copy.json'
 describe('run all queries', () => {
   it('should run all queries', () => {
     cy.visit('/')
-    cy.get(`button:contains("${copy.create_report}")`).click()
+    cy.get('button#dekart-create-report').click()
     cy.get('textarea').type('SELECT ROUND(uniform(-90::float, 90::float, random()), 6) AS lat, ROUND(uniform(-180::float, 180::float, random()), 6) AS lon FROM TABLE(GENERATOR(ROWCOUNT => 1000))', { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get(`span:contains("${copy.ready}")`, { timeout: 20000 }).should('be.visible')

cypress/fixtures/copy.json

@@ -1,5 +1,4 @@
 {
-  "create_report": "Report",
   "bigquery_query": "BigQuery query",
   "athena_query": "Athena query",
   "snowflake_query": "Snowflake query",

go.mod

@@ -23,6 +23,7 @@ require (
 )
 
 require (
+	cloud.google.com/go/secretmanager v1.10.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/snowflakedb/gosnowflake v1.6.22
 	github.com/stretchr/testify v1.8.1

go.sum

@@ -12,6 +12,8 @@ cloud.google.com/go/datacatalog v1.13.0 h1:4H5IJiyUE0X6ShQBqgFFZvGGcrwGVndTwUSLP
 cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94=
 cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk=
 cloud.google.com/go/longrunning v0.4.2 h1:WDKiiNXFTaQ6qz/G8FCOkuY9kJmOJGY67wPUC1M2RbE=
+cloud.google.com/go/secretmanager v1.10.0 h1:pu03bha7ukxF8otyPKTFdDz+rr9sE3YauS5PliDXK60=
+cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU=
 cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM=
 cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=

migrations/000023__cloud_snowflake_connection.up.sql

@@ -0,0 +1,6 @@
+ALTER TABLE connections
+ADD COLUMN connection_type INT default 1,
+ADD COLUMN snowflake_account_id text default NULL,
+ADD COLUMN snowflake_username text default NULL,
+ADD COLUMN snowflake_password_encrypted text default NULL,
+ADD COLUMN snowflake_warehouse text default NULL;

proto/dekart.proto

@@ -80,8 +80,6 @@ message GetUserStreamResponse {
     StreamOptions stream_options = 1;
     int64 connection_update = 2;
     string email = 3; // user email used to show user icon in UI
-    bool sensitive_scopes_granted = 4; // user has granted sensitive scopes
-    bool sensitive_scopes_granted_once = 5; // user has granted sensitive scopes at least once, now we request all scopes at once
 }
 
 message TestConnectionRequest {
@@ -127,6 +125,22 @@ message Connection {
     int64 updated_at = 8;
     int64 dataset_count = 9;
     bool can_store_files = 10;
+    enum ConnectionType {
+        CONNECTION_TYPE_UNSPECIFIED = 0;
+        CONNECTION_TYPE_BIGQUERY = 1;
+        CONNECTION_TYPE_SNOWFLAKE = 2;
+    }
+    ConnectionType connection_type = 11;
+    string snowflake_account_id = 12;
+    string snowflake_username = 13;
+    Secret snowflake_password = 14;
+    string snowflake_warehouse = 15;
+}
+
+message Secret {
+    string client_encrypted = 1; // encrypted with client key
+    string server_encrypted = 2; // encrypted with server key
+    int32 length = 3; // length of the password for placeholder
 }
 
 message GetUsageRequest {}
@@ -180,6 +194,10 @@ message GetEnvResponse {
             TYPE_UX_ACCESS_ERROR_INFO_HTML = 13;
             TYPE_UX_NOT_FOUND_ERROR_INFO_HTML = 14;
             TYPE_UX_SAMPLE_QUERY_SQL = 15;
+            TYPE_AES_KEY = 16;
+            TYPE_AES_IV = 17;
+            TYPE_AUTH_ENABLED = 18;
+            TYPE_USER_DEFINED_CONNECTION = 19;
         }
         Type type = 1;
         string value = 2;
@@ -191,6 +209,7 @@ message GetEnvResponse {
 message RedirectState {
     string token_json = 1;
     string error  = 2;
+    bool sensitive_scopes_granted = 3; // if true, sensitive scope are granted
 }
 
 // AuthState is used to pass state between UI, auth endpoint and Google OAuth via redirect
@@ -238,6 +257,8 @@ message Report {
     bool is_author = 9; // user is the author of the report
     int64 created_at = 10;
     int64 updated_at = 11;
+    bool is_sharable = 12; // can be shared with other users (depends where result is stored)
+    bool need_sensitive_scope = 13; // need sensitive scope to run queries and read results
 }
 
 message Dataset {
@@ -261,7 +282,7 @@ message Query {
         JOB_STATUS_PENDING = 1;
         JOB_STATUS_RUNNING = 2;
         JOB_STATUS_DONE_LEGACY = 3;  // legacy for backwards compatibility
-        JOB_STATUS_READING_RESULTS = 4; // job is done proccessing results
+        JOB_STATUS_READING_RESULTS = 4; // job is done processing results
         JOB_STATUS_DONE = 5;
     }