Fix upload script, fix time formatting (#1647)

* Fix upload script * badly implemented cross join * ignore the snowflake log * add type for object * .fix time tests. * Rewrite the way we get information schema. * prettier fixes * Pipeline changes * add more snowflake limitations. * Finally passing tests.
malloydata · Mar 9, 2024 · 0467183 · 0467183
1 parent 8c56c0c
commit 0467183
Show file tree

Hide file tree

Showing 10 changed files with 132 additions and 127 deletions.
diff --git a/.gitignore b/.gitignore
@@ -58,3 +58,4 @@ demo/malloy-demo-composer/env
 .env
 test/data/duckdb/duckdb_test.db.wal
 monospace.json
+snowflake.log
diff --git a/packages/malloy-db-snowflake/src/snowflake_connection.ts b/packages/malloy-db-snowflake/src/snowflake_connection.ts
@@ -131,13 +131,9 @@ export class SnowflakeConnection
     await this.executor.done();
   }
 
-  private getTempTableName(sqlCommand: string): string {
+  private getTempViewName(sqlCommand: string): string {
     const hash = crypto.createHash('md5').update(sqlCommand).digest('hex');
-    let tableName = `tt${hash}`;
-    if (this.scratchSpace) {
-      tableName = `${this.scratchSpace.database}.${this.scratchSpace.schema}.${tableName}`;
-    }
-    return tableName;
+    return `tt${hash}`;
   }
 
   public async runSQL(
@@ -179,10 +175,12 @@ export class SnowflakeConnection
   ): Promise<void> {
     const rows = await this.executor.batch(infoQuery);
     for (const row of rows) {
-      const snowflakeDataType = row['DATA_TYPE'] as string;
+      // data types look like `VARCHAR(1234)`
+      let snowflakeDataType = row['type'] as string;
+      snowflakeDataType = snowflakeDataType.toLocaleLowerCase().split('(')[0];
       const s = structDef;
       const malloyType = this.dialect.sqlTypeToMalloyType(snowflakeDataType);
-      const name = row['COLUMN_NAME'] as string;
+      const name = row['name'] as string;
       if (malloyType) {
         s.fields.push({...malloyType, name});
       } else {
@@ -199,15 +197,6 @@ export class SnowflakeConnection
     tableKey: string,
     tablePath: string
   ): Promise<StructDef> {
-    // looks like snowflake:schemaName.tableName
-    tableKey = tableKey.toLowerCase();
-
-    let [schema, tableName] = ['', tablePath];
-    const schema_and_table = tablePath.split('.');
-    if (schema_and_table.length === 2) {
-      [schema, tableName] = schema_and_table;
-    }
-
     const structDef: StructDef = {
       type: 'struct',
       dialect: 'snowflake',
@@ -231,16 +220,7 @@ export class SnowflakeConnection
     //  GROUP BY 1,2
     //  ORDER BY PATH
 
-    const infoQuery = `
-  SELECT
-    column_name, -- LOWER(COLUMN_NAME) AS column_name,
-    LOWER(DATA_TYPE) as data_type
-  FROM
-    INFORMATION_SCHEMA.COLUMNS
-  WHERE
-    table_schema = UPPER('${schema}')
-    AND table_name = UPPER('${tableName}');
-    `;
+    const infoQuery = `DESCRIBE TABLE ${tablePath}`;
 
     await this.schemaFromQuery(infoQuery, structDef);
     return structDef;
@@ -301,24 +281,14 @@ export class SnowflakeConnection
     };
 
     // create temp table with same schema as the query
-    const tempTableName = this.getTempTableName(sqlRef.selectStr);
+    const tempTableName = this.getTempViewName(sqlRef.selectStr);
     this.runSQL(
       `
-      CREATE OR REPLACE TEMP TABLE ${tempTableName} as SELECT * FROM (
-        ${sqlRef.selectStr}
-      ) as x WHERE false;
+      CREATE OR REPLACE TEMP VIEW ${tempTableName} as ${sqlRef.selectStr};
       `
     );
 
-    const infoQuery = `
-  SELECT
-    column_name, -- LOWER(column_name) as column_name,
-    LOWER(data_type) as data_type
-  FROM
-    INFORMATION_SCHEMA.COLUMNS
-  WHERE
-    table_name = UPPER('${tempTableName}');
-  `;
+    const infoQuery = `DESCRIBE TABLE ${tempTableName}`;
     await this.schemaFromQuery(infoQuery, structDef);
     return structDef;
   }
@@ -351,7 +321,7 @@ export class SnowflakeConnection
   }
 
   public async manifestTemporaryTable(sqlCommand: string): Promise<string> {
-    const tableName = this.getTempTableName(sqlCommand);
+    const tableName = this.getTempViewName(sqlCommand);
     const cmd = `CREATE OR REPLACE TEMP TABLE ${tableName} AS (${sqlCommand});`;
     await this.runSQL(cmd);
     return tableName;

diff --git a/packages/malloy-db-snowflake/src/snowflake_executor.ts b/packages/malloy-db-snowflake/src/snowflake_executor.ts
@@ -170,16 +170,15 @@ export class SnowflakeExecutor {
 
   private async _setSessionParams(conn: Connection) {
     // set some default session parameters
-    // this is quite imporant for snowflake because malloy tends to add quotes to all database identifiers
-    // and snowflake is case sensitive by with quotes but matches against all caps identifiers without quotes
-    // await this._execute(
-    //   'ALTER SESSION SET QUOTED_IDENTIFIERS_IGNORE_CASE = true;',
-    //   conn
-    // );
     // set utc as the default timezone which is the malloy convention
     await this._execute("ALTER SESSION SET TIMEZONE = 'UTC';", conn);
     // ensure week starts on Sunday which is the malloy convention
     await this._execute('ALTER SESSION SET WEEK_START = 7;', conn);
+    // so javascript can parse the dates
+    await this._execute(
+      "ALTER SESSION SET TIMESTAMP_NTZ_OUTPUT_FORMAT='YYYY-MM-DDTHH24:MI:SS.FF3TZH:TZM';",
+      conn
+    );
   }
 
   public async batch(sqlText: string): Promise<QueryData> {

diff --git a/packages/malloy/src/dialect/dialect.ts b/packages/malloy/src/dialect/dialect.ts
@@ -110,6 +110,12 @@ export abstract class Dialect {
   // StandardSQL dialects can't partition on expression in window functions
   cantPartitionWindowFunctionsOnExpressions = false;
 
+  // Snowflake can't yet support pipelines in nested views.
+  supportsPipelinesInViews = true;
+
+  // Some dialects don't supporrt arrays.
+  supportsArraysInData = true;
+
   // return the definition of a function with the given name
   abstract getGlobalFunctionDef(
     name: string

diff --git a/packages/malloy/src/dialect/snowflake/snowflake.ts b/packages/malloy/src/dialect/snowflake/snowflake.ts
@@ -108,14 +108,18 @@ export class SnowflakeDialect extends Dialect {
   dontUnionIndex = false;
   supportsQualify = false;
   supportsNesting = true;
+  supportsPipelinesInViews = false;
+  supportsArraysInData = false;
 
   // don't mess with the table pathing.
   quoteTablePath(tablePath: string): string {
     return tablePath;
   }
 
   sqlGroupSetTable(groupSetCount: number): string {
-    return `SELECT index as group_set FROM TABLE(FLATTEN(ARRAY_GENERATE_RANGE(0, ${groupSetCount})))`;
+    return `CROSS JOIN (SELECT index as group_set FROM TABLE(FLATTEN(ARRAY_GENERATE_RANGE(0, ${
+      groupSetCount + 1
+    }))))`;
   }
 
   sqlAnyValue(groupSet: number, fieldName: string): string {
@@ -142,7 +146,7 @@ export class SnowflakeDialect extends Dialect {
   ): string {
     const fields = this.mapFieldsForObjectConstruct(fieldList);
     const orderByClause = orderBy ? ` WITHIN GROUP (${orderBy})` : '';
-    const aggClause = `ARRAY_AGG(CASE WHEN group_set=${groupSet} THEN OBJECT_CONSTRUCT(${fields}) END)${orderByClause}`;
+    const aggClause = `ARRAY_AGG(CASE WHEN group_set=${groupSet} THEN OBJECT_CONSTRUCT_KEEP_NULL(${fields}) END)${orderByClause}`;
     if (limit === undefined) {
       return `COALESCE(${aggClause}, [])`;
     }
@@ -151,7 +155,7 @@ export class SnowflakeDialect extends Dialect {
 
   sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string {
     const fields = this.mapFieldsForObjectConstruct(fieldList);
-    return `(ARRAY_AGG(CASE WHEN group_set=${groupSet} THEN OBJECT_CONSTRUCT(${fields}) END) WITHIN GROUP (ORDER BY 1 ASC NULLS LAST))[0]`;
+    return `(ARRAY_AGG(CASE WHEN group_set=${groupSet} THEN OBJECT_CONSTRUCT_KEEP_NULL(${fields}) END) WITHIN GROUP (ORDER BY 1 ASC NULLS LAST))[0]`;
   }
 
   sqlAnyValueLastTurtle(
@@ -170,7 +174,7 @@ export class SnowflakeDialect extends Dialect {
     const nullValues = fieldList
       .map(f => `'${f.sqlOutputName}', NULL`)
       .join(', ');
-    return `COALESCE(ARRAY_AGG(CASE WHEN group_set=${groupSet} THEN OBJECT_CONSTRUCT(${fields}) END)[0], OBJECT_CONSTRUCT_KEEP_NULL(${nullValues}))`;
+    return `COALESCE(ARRAY_AGG(CASE WHEN group_set=${groupSet} THEN OBJECT_CONSTRUCT_KEEP_NULL(${fields}) END)[0], OBJECT_CONSTRUCT_KEEP_NULL(${nullValues}))`;
   }
 
   sqlUnnestAlias(
@@ -243,6 +247,8 @@ export class SnowflakeDialect extends Dialect {
       let snowflakeType = fieldType;
       if (fieldType === 'string') {
         snowflakeType = 'varchar';
+      } else if (fieldType === 'struct') {
+        snowflakeType = 'variant';
       }
       return `${alias}.value:"${fieldName}"::${snowflakeType}`;
     }
@@ -263,12 +269,13 @@ export class SnowflakeDialect extends Dialect {
     throw new Error('not implemented yet');
   }
 
-  sqlCreateFunctionCombineLastStage(lastStageName: string): string {
-    return `SELECT ARRAY_AGG(OBJECT_CONSTRUCT(*)) FROM ${lastStageName}`;
+  sqlCreateFunctionCombineLastStage(_lastStageName: string): string {
+    throw new Error('not implemented yet');
+    // return `SELECT ARRAY_AGG(OBJECT_CONSTRUCT(*)) FROM ${lastStageName}`;
   }
 
   sqlSelectAliasAsStruct(alias: string): string {
-    return `OBJECT_CONSTRUCT(${alias}.*)`;
+    return `OBJECT_CONSTRUCT_KEEP_NULL(${alias}.*)`;
   }
   sqlMaybeQuoteIdentifier(identifier: string): string {
     return `"${identifier}"`;

diff --git a/packages/malloy/src/malloy.ts b/packages/malloy/src/malloy.ts
@@ -80,7 +80,7 @@ import {
 } from './runtime_types';
 import {DateTime} from 'luxon';
 import {Tag, TagParse, TagParseSpec, Taggable} from './tags';
-import {getDialect} from './dialect';
+import {Dialect, getDialect} from './dialect';
 
 export interface Loggable {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -2477,6 +2477,10 @@ export class SingleConnectionRuntime<
     );
   }
 
+  get dialect(): Dialect {
+    return getDialect(this.connection.dialectName);
+  }
+
   getQuoter(): (arg: TemplateStringsArray) => string {
     return (x: TemplateStringsArray) => this.quote(x.toString());
   }

diff --git a/test/snowflake/uploaddata.sql b/test/snowflake/uploaddata.sql
@@ -4,15 +4,16 @@
 -- snowsql -f uploadddate.sql
 
 
-drop database malloytestdb;
-create database malloytestdb;
+drop database malloytest;
+create database malloytest;
 
-use malloytestdb;
+use malloytest;
 create schema malloytest;
 
 CREATE OR REPLACE FILE FORMAT PARQUET_SCHEMA_DETECTION
   TYPE = PARQUET
-  BINARY_AS_TEXT = FALSE;
+  BINARY_AS_TEXT = FALSE
+  USE_LOGICAL_TYPE = TRUE;
 
 PUT file://../data/duckdb/aircraft.parquet @~/staged;