diff --git a/lerna.json b/lerna.json index 89877afb1..f93c22a57 100644 --- a/lerna.json +++ b/lerna.json @@ -1,7 +1,7 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", "useNx": false, - "version": "0.0.218", + "version": "0.0.222", "command": { "version": { "allowBranch": "main" diff --git a/package-lock.json b/package-lock.json index 951cada81..8db90641e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10796,9 +10796,9 @@ } }, "node_modules/axios": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.4.tgz", - "integrity": "sha512-DukmaFRnY6AzAALSH4J2M3k6PkaC+MfaAGdEERRWcC9q3/TWQwLpHR8ZRLKTdQ3aBDL64EdluRDjJqKw+BPZEw==", + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", @@ -24323,21 +24323,11 @@ } }, "node_modules/trino-client": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/trino-client/-/trino-client-0.2.3.tgz", - "integrity": "sha512-zIHFR+rV6hL1g/dlgFzE1JCeBLNtmHphf+K6hRfu4NK4n3BdBPEcajgKGj/1Z7jyka3hBdcsB83P/ql39CQatQ==", - "dependencies": { - "axios": "1.7.2" - } - }, - "node_modules/trino-client/node_modules/axios": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", - "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==", + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/trino-client/-/trino-client-0.2.6.tgz", + "integrity": "sha512-MK+seOsnxCKASAV528xSE/15+b3tTuGxkzv9VuLfyoQsQsVoX+otT2Fo56xBwDR0iDzPuUwyGbRCd3xQE2776g==", "dependencies": { - "follow-redirects": "^1.15.6", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" + "axios": "1.7.7" } }, "node_modules/triple-beam": { @@ -26608,7 +26598,7 @@ }, "packages/malloy": { "name": "@malloydata/malloy", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "dependencies": { "antlr4ts": "^0.5.0-alpha.4", @@ -26629,13 +26619,13 @@ }, "packages/malloy-db-bigquery": { "name": "@malloydata/db-bigquery", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "dependencies": { "@google-cloud/bigquery": "^7.3.0", "@google-cloud/common": "^5.0.1", "@google-cloud/paginator": "^5.0.0", - "@malloydata/malloy": "^0.0.218", + "@malloydata/malloy": "^0.0.222", "gaxios": "^4.2.0" }, "engines": { @@ -26644,11 +26634,11 @@ }, "packages/malloy-db-duckdb": { "name": "@malloydata/db-duckdb", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "dependencies": { "@duckdb/duckdb-wasm": "1.29.0", - "@malloydata/malloy": "^0.0.218", + "@malloydata/malloy": "^0.0.222", "@motherduck/wasm-client": "^0.6.6", "apache-arrow": "^17.0.0", "duckdb": "1.1.1", @@ -26705,10 +26695,10 @@ }, "packages/malloy-db-mysql": { "name": "@malloydata/db-mysql", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "dependencies": { - "@malloydata/malloy": "^0.0.218", + "@malloydata/malloy": "^0.0.222", "@types/node": "^22.7.4", "fastestsmallesttextencoderdecoder": "^1.0.22", "luxon": "^3.5.0", @@ -26741,10 +26731,10 @@ }, "packages/malloy-db-postgres": { "name": "@malloydata/db-postgres", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "dependencies": { - "@malloydata/malloy": "^0.0.218", + "@malloydata/malloy": "^0.0.222", "@types/pg": "^8.6.1", "pg": "^8.7.1", "pg-query-stream": "4.2.3" @@ -26755,10 +26745,10 @@ }, "packages/malloy-db-snowflake": { "name": "@malloydata/db-snowflake", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "dependencies": { - "@malloydata/malloy": "^0.0.218", + "@malloydata/malloy": "^0.0.222", "generic-pool": "^3.9.0", "snowflake-sdk": "1.14.0", "toml": "^3.0.0" @@ -26769,10 +26759,10 @@ }, "packages/malloy-db-trino": { "name": "@malloydata/db-trino", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "dependencies": { - "@malloydata/malloy": "^0.0.218", + "@malloydata/malloy": "^0.0.222", "@prestodb/presto-js-client": "^1.0.0", "gaxios": "^4.2.0", "trino-client": "^0.2.2" @@ -26783,7 +26773,7 @@ }, "packages/malloy-interfaces": { "name": "@malloydata/malloy-interfaces", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "engines": { "node": ">=18" @@ -26791,10 +26781,10 @@ }, "packages/malloy-malloy-sql": { "name": "@malloydata/malloy-sql", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "dependencies": { - "@malloydata/malloy": "^0.0.218" + "@malloydata/malloy": "^0.0.222" }, "devDependencies": { "peggy": "^3.0.2" @@ -26805,10 +26795,10 @@ }, "packages/malloy-render": { "name": "@malloydata/render", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "dependencies": { - "@malloydata/malloy": "^0.0.218", + "@malloydata/malloy": "^0.0.222", "@tanstack/solid-virtual": "^3.10.4", "component-register": "^0.8.6", "lodash": "^4.17.20", @@ -27242,7 +27232,7 @@ }, "packages/malloy-syntax-highlight": { "name": "@malloydata/syntax-highlight", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "devDependencies": { "@types/jasmine": "^4.3.5", @@ -27292,17 +27282,17 @@ }, "test": { "name": "@malloydata/malloy-tests", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "dependencies": { "@jest/globals": "^29.4.3", - "@malloydata/db-bigquery": "^0.0.218", - "@malloydata/db-duckdb": "^0.0.218", - "@malloydata/db-postgres": "^0.0.218", - "@malloydata/db-snowflake": "^0.0.218", - "@malloydata/db-trino": "^0.0.218", - "@malloydata/malloy": "^0.0.218", - "@malloydata/render": "^0.0.218", + "@malloydata/db-bigquery": "^0.0.222", + "@malloydata/db-duckdb": "^0.0.222", + "@malloydata/db-postgres": "^0.0.222", + "@malloydata/db-snowflake": "^0.0.222", + "@malloydata/db-trino": "^0.0.222", + "@malloydata/malloy": "^0.0.222", + "@malloydata/render": "^0.0.222", "events": "^3.3.0", "jsdom": "^22.1.0", "luxon": "^2.4.0", diff --git a/packages/malloy-db-bigquery/package.json b/packages/malloy-db-bigquery/package.json index 3ba4f5df5..c147ba68f 100644 --- a/packages/malloy-db-bigquery/package.json +++ b/packages/malloy-db-bigquery/package.json @@ -1,6 +1,6 @@ { "name": "@malloydata/db-bigquery", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -25,7 +25,7 @@ "@google-cloud/bigquery": "^7.3.0", "@google-cloud/common": "^5.0.1", "@google-cloud/paginator": "^5.0.0", - "@malloydata/malloy": "^0.0.218", + "@malloydata/malloy": "^0.0.222", "gaxios": "^4.2.0" } } diff --git a/packages/malloy-db-bigquery/src/bigquery_connection.ts b/packages/malloy-db-bigquery/src/bigquery_connection.ts index 5586f82b3..7089c260b 100644 --- a/packages/malloy-db-bigquery/src/bigquery_connection.ts +++ b/packages/malloy-db-bigquery/src/bigquery_connection.ts @@ -35,7 +35,7 @@ import {ResourceStream} from '@google-cloud/paginator'; import * as googleCommon from '@google-cloud/common'; import {GaxiosError} from 'gaxios'; import { - arrayEachFields, + mkArrayDef, Connection, ConnectionConfig, Malloy, @@ -520,14 +520,7 @@ export class BigQueryConnection // Malloy treats repeated values as an array of scalars. const malloyType = this.dialect.sqlTypeToMalloyType(type); if (malloyType) { - const arrayField: StructDef = { - ...structShared, - type: 'array', - elementTypeDef: malloyType, - join: 'many', - fields: arrayEachFields(malloyType), - }; - structDef.fields.push(arrayField); + structDef.fields.push(mkArrayDef(malloyType, name, this.dialectName)); } } else if (isRecord) { const ifRepeatedRecord: StructDef = { diff --git a/packages/malloy-db-duckdb/package.json b/packages/malloy-db-duckdb/package.json index a9c2c30f5..60a39e9d3 100644 --- a/packages/malloy-db-duckdb/package.json +++ b/packages/malloy-db-duckdb/package.json @@ -1,6 +1,6 @@ { "name": "@malloydata/db-duckdb", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -41,7 +41,7 @@ }, "dependencies": { "@duckdb/duckdb-wasm": "1.29.0", - "@malloydata/malloy": "^0.0.218", + "@malloydata/malloy": "^0.0.222", "@motherduck/wasm-client": "^0.6.6", "apache-arrow": "^17.0.0", "duckdb": "1.1.1", diff --git a/packages/malloy-db-duckdb/src/duckdb.spec.ts b/packages/malloy-db-duckdb/src/duckdb.spec.ts index f63956fe4..77c3efc94 100644 --- a/packages/malloy-db-duckdb/src/duckdb.spec.ts +++ b/packages/malloy-db-duckdb/src/duckdb.spec.ts @@ -23,7 +23,7 @@ import {DuckDBCommon} from './duckdb_common'; import {DuckDBConnection} from './duckdb_connection'; -import {arrayEachFields, SQLSourceDef, StructDef} from '@malloydata/malloy'; +import {SQLSourceDef, StructDef, mkArrayDef} from '@malloydata/malloy'; import {describeIfDatabaseAvailable} from '@malloydata/malloy/test'; const [describe] = describeIfDatabaseAvailable(['duckdb']); @@ -132,14 +132,9 @@ describe('DuckDBConnection', () => { it('parses arrays', () => { const structDef = makeStructDef(); connection.fillStructDefFromTypeMap(structDef, {test: ARRAY_SCHEMA}); - expect(structDef.fields[0]).toEqual({ - name: 'test', - type: 'array', - elementTypeDef: intTyp, - join: 'many', - dialect: 'duckdb', - fields: arrayEachFields({type: 'number', numberType: 'integer'}), - }); + expect(structDef.fields[0]).toEqual( + mkArrayDef({type: 'number', numberType: 'integer'}, 'test', 'duckdb') + ); }); it('parses inline', () => { diff --git a/packages/malloy-db-duckdb/src/duckdb_common.ts b/packages/malloy-db-duckdb/src/duckdb_common.ts index e0e1e548a..a64d4a632 100644 --- a/packages/malloy-db-duckdb/src/duckdb_common.ts +++ b/packages/malloy-db-duckdb/src/duckdb_common.ts @@ -35,7 +35,7 @@ import { DuckDBDialect, SQLSourceDef, TableSourceDef, - arrayEachFields, + mkFieldDef, } from '@malloydata/malloy'; import {BaseConnection} from '@malloydata/malloy/connection'; @@ -149,110 +149,16 @@ export abstract class DuckDBCommon return {}; } - /** - * Split's a structs columns declaration into individual columns - * to be fed back into fillStructDefFromTypeMap(). Handles commas - * within nested STRUCT() declarations. - * - * (https://github.com/malloydata/malloy/issues/635) - * - * @param s struct's column declaration - * @return Array of column type declarations - */ - private splitColumns(s: string) { - const columns: string[] = []; - let parens = 0; - let column = ''; - let eatSpaces = true; - for (let idx = 0; idx < s.length; idx++) { - const c = s.charAt(idx); - if (eatSpaces && c === ' ') { - // Eat space - } else { - eatSpaces = false; - if (!parens && c === ',') { - columns.push(column); - column = ''; - eatSpaces = true; - } else { - column += c; - } - if (c === '(') { - parens += 1; - } else if (c === ')') { - parens -= 1; - } - } - } - columns.push(column); - return columns; - } - - private stringToTypeMap(s: string): {[name: string]: string} { - const ret: {[name: string]: string} = {}; - const columns = this.splitColumns(s); - for (const c of columns) { - //const [name, type] = c.split(" ", 1); - const columnMatch = c.match(/^(?[^\s]+) (?.*)$/); - if (columnMatch && columnMatch.groups) { - ret[columnMatch.groups['name']] = columnMatch.groups['type']; - } else { - throw new Error(`Badly form Structure definition ${s}`); - } - } - return ret; - } - fillStructDefFromTypeMap( structDef: StructDef, typeMap: {[name: string]: string} ) { for (const fieldName in typeMap) { - let duckDBType = typeMap[fieldName]; // Remove quotes from field name const name = unquoteName(fieldName); - let malloyType = this.dialect.sqlTypeToMalloyType(duckDBType); - const arrayMatch = duckDBType.match(/(?.*)\[\]$/); - if (arrayMatch && arrayMatch.groups) { - duckDBType = arrayMatch.groups['duckDBType']; - } - const structMatch = duckDBType.match(/^STRUCT\((?.*)\)$/); - if (structMatch && structMatch.groups) { - const newTypeMap = this.stringToTypeMap(structMatch.groups['fields']); - let innerStructDef: StructDef; - const structhead = {name, dialect: this.dialectName, fields: []}; - if (arrayMatch) { - innerStructDef = { - type: 'array', - elementTypeDef: {type: 'record_element'}, - join: 'many', - ...structhead, - }; - } else { - innerStructDef = { - type: 'record', - join: 'one', - ...structhead, - }; - } - this.fillStructDefFromTypeMap(innerStructDef, newTypeMap); - structDef.fields.push(innerStructDef); - } else { - if (arrayMatch) { - malloyType = this.dialect.sqlTypeToMalloyType(duckDBType); - const innerStructDef: StructDef = { - type: 'array', - elementTypeDef: malloyType, - name, - dialect: this.dialectName, - join: 'many', - fields: arrayEachFields(malloyType), - }; - structDef.fields.push(innerStructDef); - } else { - structDef.fields.push({...malloyType, name}); - } - } + const dbType = typeMap[fieldName]; + const malloyType = this.dialect.parseDuckDBType(dbType); + structDef.fields.push(mkFieldDef(malloyType, name, 'duckdb')); } } diff --git a/packages/malloy-db-mysql/package.json b/packages/malloy-db-mysql/package.json index d3393ab5f..25b44c804 100644 --- a/packages/malloy-db-mysql/package.json +++ b/packages/malloy-db-mysql/package.json @@ -1,6 +1,6 @@ { "name": "@malloydata/db-mysql", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -22,7 +22,7 @@ "prepublishOnly": "npm run build" }, "dependencies": { - "@malloydata/malloy": "^0.0.218", + "@malloydata/malloy": "^0.0.222", "@types/node": "^22.7.4", "fastestsmallesttextencoderdecoder": "^1.0.22", "luxon": "^3.5.0", diff --git a/packages/malloy-db-mysql/src/index.ts b/packages/malloy-db-mysql/src/index.ts index a99f6ba49..2fe4aff4d 100644 --- a/packages/malloy-db-mysql/src/index.ts +++ b/packages/malloy-db-mysql/src/index.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ export {MySQLConnection, MySQLExecutor} from './mysql_connection'; diff --git a/packages/malloy-db-mysql/src/mysql.s_p_e_c_dont_run.ts b/packages/malloy-db-mysql/src/mysql.s_p_e_c_dont_run.ts index 7833759c7..f6c75d124 100644 --- a/packages/malloy-db-mysql/src/mysql.s_p_e_c_dont_run.ts +++ b/packages/malloy-db-mysql/src/mysql.s_p_e_c_dont_run.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {MySQLConnection, MySQLExecutor} from '.'; diff --git a/packages/malloy-db-mysql/src/mysql_connection.ts b/packages/malloy-db-mysql/src/mysql_connection.ts index 9ff69e6c0..97f43758b 100644 --- a/packages/malloy-db-mysql/src/mysql_connection.ts +++ b/packages/malloy-db-mysql/src/mysql_connection.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import { diff --git a/packages/malloy-db-postgres/package.json b/packages/malloy-db-postgres/package.json index fbbd19df2..1d66e194f 100644 --- a/packages/malloy-db-postgres/package.json +++ b/packages/malloy-db-postgres/package.json @@ -1,6 +1,6 @@ { "name": "@malloydata/db-postgres", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -22,7 +22,7 @@ "prepublishOnly": "npm run build" }, "dependencies": { - "@malloydata/malloy": "^0.0.218", + "@malloydata/malloy": "^0.0.222", "@types/pg": "^8.6.1", "pg": "^8.7.1", "pg-query-stream": "4.2.3" diff --git a/packages/malloy-db-postgres/src/postgres_connection.ts b/packages/malloy-db-postgres/src/postgres_connection.ts index 20fe59f6b..bf9bf4af1 100644 --- a/packages/malloy-db-postgres/src/postgres_connection.ts +++ b/packages/malloy-db-postgres/src/postgres_connection.ts @@ -42,9 +42,9 @@ import { RunSQLOptions, SQLSourceDef, TableSourceDef, - arrayEachFields, StreamingConnection, StructDef, + mkArrayDef, } from '@malloydata/malloy'; import {BaseConnection} from '@malloydata/malloy/connection'; @@ -237,14 +237,7 @@ export class PostgresConnection const elementType = this.dialect.sqlTypeToMalloyType( row['element_type'] as string ); - structDef.fields.push({ - type: 'array', - elementTypeDef: elementType, - name, - dialect: this.dialectName, - join: 'many', - fields: arrayEachFields(elementType), - }); + structDef.fields.push(mkArrayDef(elementType, name, this.dialectName)); } else { const malloyType = this.dialect.sqlTypeToMalloyType(postgresDataType); structDef.fields.push({...malloyType, name}); diff --git a/packages/malloy-db-snowflake/package.json b/packages/malloy-db-snowflake/package.json index 7dd02bc08..83df91680 100644 --- a/packages/malloy-db-snowflake/package.json +++ b/packages/malloy-db-snowflake/package.json @@ -1,6 +1,6 @@ { "name": "@malloydata/db-snowflake", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -21,7 +21,7 @@ "prepublishOnly": "npm run build" }, "dependencies": { - "@malloydata/malloy": "^0.0.218", + "@malloydata/malloy": "^0.0.222", "generic-pool": "^3.9.0", "snowflake-sdk": "1.14.0", "toml": "^3.0.0" diff --git a/packages/malloy-db-snowflake/src/snowflake_connection.ts b/packages/malloy-db-snowflake/src/snowflake_connection.ts index 5dde6b40e..bd7b5d308 100644 --- a/packages/malloy-db-snowflake/src/snowflake_connection.ts +++ b/packages/malloy-db-snowflake/src/snowflake_connection.ts @@ -36,8 +36,12 @@ import { QueryDataRow, SnowflakeDialect, TestableConnection, - arrayEachFields, - LeafAtomicTypeDef, + TinyParser, + Dialect, + RecordDef, + mkArrayDef, + AtomicFieldDef, + ArrayDef, } from '@malloydata/malloy'; import {BaseConnection} from '@malloydata/malloy/connection'; @@ -61,20 +65,152 @@ export interface SnowflakeConnectionOptions { queryOptions?: RunSQLOptions; } -class StructMap { - fieldMap = new Map(); - type = 'record'; - isArray = false; +type PathChain = + | {arrayRef: true; next?: PathChain} + | {name: string; next?: PathChain}; - constructor(type: string, isArray: boolean) { - this.type = type; - this.isArray = isArray; +class SnowField { + constructor( + readonly name: string, + readonly type: string, + readonly dialect: Dialect + ) {} + fieldDef(): AtomicFieldDef { + return { + ...this.dialect.sqlTypeToMalloyType(this.type), + name: this.name, + }; + } + walk(_path: PathChain, _fieldType: string): void { + throw new Error( + 'SNOWWFLAKE SCHEMA PARSE ERROR: Should not walk through fields' + ); + } + static make(name: string, fieldType: string, d: Dialect) { + if (fieldType === 'array') { + return new SnowArray(name, d); + } else if (fieldType === 'object') { + return new SnowObject(name, d); + } + return new SnowField(name, fieldType, d); + } +} + +class SnowObject extends SnowField { + fieldMap = new Map(); + constructor(name: string, d: Dialect) { + super(name, 'object', d); + } + + get fields(): AtomicFieldDef[] { + const fields: AtomicFieldDef[] = []; + for (const [_, fieldObj] of this.fieldMap) { + fields.push(fieldObj.fieldDef()); + } + return fields; } - addChild(name: string, type: string): StructMap { - const s = new StructMap(type, false); - this.fieldMap.set(name, s); - return s; + fieldDef(): RecordDef { + const rec: RecordDef = { + type: 'record', + name: this.name, + fields: this.fields, + join: 'one', + dialect: this.dialect.name, + }; + return rec; + } + + walk(path: PathChain, fieldType: string) { + if ('name' in path) { + const field = this.fieldMap.get(path.name); + if (path.next) { + if (field) { + field.walk(path.next, fieldType); + return; + } + throw new Error( + 'SNOWFLAKE SCHEMA PARSER ERROR: Walk through undefined' + ); + } else { + // If we get multiple type for a field, ignore them, should + // which will do until we support viarant data + if (!field) { + this.fieldMap.set( + path.name, + SnowField.make(path.name, fieldType, this.dialect) + ); + return; + } + } + } + throw new Error( + 'SNOWFLAKE SCHEMA PARSER ERROR: Walk object reference through array reference' + ); + } +} + +class SnowArray extends SnowField { + arrayOf = 'unknown'; + objectChild?: SnowObject; + arrayChild?: SnowArray; + constructor(name: string, d: Dialect) { + super(name, 'array', d); + } + + isArrayOf(type: string) { + if (this.arrayOf !== 'unknown') { + this.arrayOf = 'variant'; + return; + } + this.arrayOf = type; + if (type === 'object') { + this.objectChild = new SnowObject('', this.dialect); + } else if (type === 'array') { + this.arrayChild = new SnowArray('', this.dialect); + } + } + + fieldDef(): ArrayDef { + if (this.objectChild) { + const t = mkArrayDef( + {type: 'record', fields: this.objectChild.fields}, + this.name, + this.dialect.name + ); + return t; + } + if (this.arrayChild) { + return mkArrayDef( + this.arrayChild.fieldDef(), + this.name, + this.dialect.name + ); + } + return mkArrayDef( + this.dialect.sqlTypeToMalloyType(this.arrayOf), + this.name, + this.dialect.name + ); + } + + walk(path: PathChain, fieldType: string) { + if ('arrayRef' in path) { + if (path.next) { + const next = this.arrayChild || this.objectChild; + if (next) { + next.walk(path.next, fieldType); + return; + } + throw new Error( + 'SNOWFLAKE SCHEMA PARSER ERROR: Array walk through leaf' + ); + } else { + this.isArrayOf(fieldType); + return; + } + } + throw new Error('SNOWFLAKE SCHEMA PARSER ERROR: Array walk through name'); } } @@ -175,57 +311,6 @@ export class SnowflakeConnection await this.executor.batch('SELECT 1 as one'); } - private addFieldsToStructDef( - structDef: StructDef, - structMap: StructMap - ): void { - if (structMap.fieldMap.size === 0) return; - for (const [field, value] of structMap.fieldMap) { - const type = value.type; - const name = field; - - // check for an array - if (value.isArray && type !== 'object') { - // Apparently there can only be arrays of integers, strings, or unknowns? - // TODO is this true or is this just all that got implemented? - const malloyType: LeafAtomicTypeDef = - type === 'integer' - ? {type: 'number', numberType: 'integer'} - : type === 'varchar' - ? {type: 'string'} - : {type: 'sql native', rawType: type}; - const innerStructDef: StructDef = { - type: 'array', - name, - dialect: this.dialectName, - join: 'many', - elementTypeDef: malloyType, - fields: arrayEachFields(malloyType), - }; - structDef.fields.push(innerStructDef); - } else if (type === 'object') { - const structParts = {name, dialect: this.dialectName, fields: []}; - const innerStructDef: StructDef = value.isArray - ? { - ...structParts, - type: 'array', - elementTypeDef: {type: 'record_element'}, - join: 'many', - } - : { - ...structParts, - type: 'record', - join: 'one', - }; - this.addFieldsToStructDef(innerStructDef, value); - structDef.fields.push(innerStructDef); - } else { - const malloyType = this.dialect.sqlTypeToMalloyType(type); - structDef.fields.push({...malloyType, name}); - } - } - } - private async schemaFromTablePath( tablePath: string, structDef: StructDef @@ -236,70 +321,48 @@ export class SnowflakeConnection const notVariant = new Map(); for (const row of rows) { // data types look like `VARCHAR(1234)` - let snowflakeDataType = row['type'] as string; - snowflakeDataType = snowflakeDataType.toLocaleLowerCase().split('(')[0]; - const s = structDef; - const malloyType = this.dialect.sqlTypeToMalloyType(snowflakeDataType); + const snowflakeDataType = (row['type'] as string) + .toLocaleLowerCase() + .split('(')[0]; const name = row['name'] as string; - if (snowflakeDataType === 'variant' || snowflakeDataType === 'array') { + if (['variant', 'array', 'object'].includes(snowflakeDataType)) { variants.push(name); - continue; - } - - notVariant.set(name, true); - if (malloyType) { - s.fields.push({...malloyType, name}); } else { - s.fields.push({ - type: 'sql native', - rawType: snowflakeDataType, - name, - }); + notVariant.set(name, true); + const malloyType = this.dialect.sqlTypeToMalloyType(snowflakeDataType); + structDef.fields.push({...malloyType, name}); } } - // if we have variants, sample the data + // For these things, we need to sample the data to know the schema if (variants.length > 0) { const sampleQuery = ` - SELECT regexp_replace(PATH, '\\\\[[0-9]*\\\\]', '') as PATH, lower(TYPEOF(value)) as type - FROM (select object_construct(*) o from ${tablePath} limit 100) + SELECT regexp_replace(PATH, '\\[[0-9]+\\]', '[*]') as PATH, lower(TYPEOF(value)) as type + FROM (select object_construct(*) o from ${tablePath} limit 100) ,table(flatten(input => o, recursive => true)) as meta - GROUP BY 1,2 - ORDER BY PATH; + GROUP BY 1,2 + ORDER BY PATH; `; const fieldPathRows = await this.executor.batch(sampleQuery); // take the schema in list form an convert it into a tree. - const structMap = new StructMap('object', true); + const rootObject = new SnowObject('__root__', this.dialect); for (const f of fieldPathRows) { const pathString = f['PATH']?.valueOf().toString(); const fieldType = f['TYPE']?.valueOf().toString(); if (pathString === undefined || fieldType === undefined) continue; - const path = pathString.split('.'); - let parent = structMap; - - // ignore the fields we've already added. - if (path.length === 1 && notVariant.get(pathString)) continue; - - let index = 0; - for (const segment of path) { - let thisNode = parent.fieldMap.get(segment); - if (thisNode === undefined) { - thisNode = parent.addChild(segment, fieldType); - } - if (fieldType === 'array') { - thisNode.isArray = true; - // if this is the last - } else if (index === path.length - 1) { - thisNode.type = fieldType; - } - parent = thisNode; - index += 1; + const pathParser = new PathParser(pathString); + const path = pathParser.pathChain(); + if ('name' in path && notVariant.get(path.name)) { + // Name will already be in the structdef + continue; } + // Walk the path and mark the type at the end + rootObject.walk(path, fieldType); } - this.addFieldsToStructDef(structDef, structMap); + structDef.fields.push(...rootObject.fields); } } @@ -338,3 +401,54 @@ export class SnowflakeConnection return tableName; } } + +export class PathParser extends TinyParser { + constructor(pathName: string) { + super(pathName, { + quoted: /^'(\\'|[^'])*'/, + array_of: /^\[\*]/, + char: /^[[.\]]/, + number: /^\d+/, + word: /^\w+/, + }); + } + + getName() { + const nameStart = this.next(); + if (nameStart.type === 'word') { + return nameStart.text; + } + if (nameStart.type === '[') { + const quotedName = this.next('quoted'); + this.next(']'); + return quotedName.text; + } + throw this.parseError('Expected column name'); + } + + pathChain(): PathChain { + const chain: PathChain = {name: this.getName()}; + let node: PathChain = chain; + for (;;) { + const sep = this.next(); + if (sep.type === 'eof') { + return chain; + } + if (sep.type === '.') { + node.next = {name: this.next('word').text}; + node = node.next; + } else if (sep.type === 'array_of') { + node.next = {arrayRef: true}; + node = node.next; + } else if (sep.type === '[') { + // Actually a dot access through a quoted name + const quoted = this.next('quoted'); + node.next = {name: quoted.text}; + node = node.next; + this.next(']'); + } else { + throw this.parseError(`Unexpected ${sep.type}`); + } + } + } +} diff --git a/packages/malloy-db-trino/package.json b/packages/malloy-db-trino/package.json index fe64ed84c..cee0f19bd 100644 --- a/packages/malloy-db-trino/package.json +++ b/packages/malloy-db-trino/package.json @@ -1,6 +1,6 @@ { "name": "@malloydata/db-trino", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -22,7 +22,7 @@ "prepublishOnly": "npm run build" }, "dependencies": { - "@malloydata/malloy": "^0.0.218", + "@malloydata/malloy": "^0.0.222", "@prestodb/presto-js-client": "^1.0.0", "gaxios": "^4.2.0", "trino-client": "^0.2.2" diff --git a/packages/malloy-db-trino/src/index.ts b/packages/malloy-db-trino/src/index.ts index 87fe09525..f8e6cf23d 100644 --- a/packages/malloy-db-trino/src/index.ts +++ b/packages/malloy-db-trino/src/index.ts @@ -21,6 +21,11 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -export {TrinoConnection} from './trino_connection'; -export {PrestoConnection} from './trino_connection'; +export type {BaseRunner} from './trino_connection'; +export { + PrestoConnection, + PrestoExplainParser, + TrinoConnection, + TrinoPrestoConnection, +} from './trino_connection'; export {TrinoExecutor} from './trino_executor'; diff --git a/packages/malloy-db-trino/src/trino_connection.spec.ts b/packages/malloy-db-trino/src/trino_connection.spec.ts index 38410255f..c3c6a8e77 100644 --- a/packages/malloy-db-trino/src/trino_connection.spec.ts +++ b/packages/malloy-db-trino/src/trino_connection.spec.ts @@ -21,7 +21,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -import {arrayEachFields, AtomicTypeDef, FieldDef} from '@malloydata/malloy'; +import {AtomicTypeDef, FieldDef} from '@malloydata/malloy'; import {TrinoConnection, TrinoExecutor} from '.'; // array(varchar) is array @@ -64,22 +64,15 @@ describe('Trino connection', () => { describe('schema parser', () => { it('parses arrays', () => { expect(connection.malloyTypeFromTrinoType('test', ARRAY_SCHEMA)).toEqual({ - 'name': 'test', - 'type': 'array', - 'dialect': 'trino', - 'elementTypeDef': intType, - 'join': 'many', - 'fields': arrayEachFields(intType), + type: 'array', + elementTypeDef: intType, }); }); it('parses inline', () => { expect(connection.malloyTypeFromTrinoType('test', INLINE_SCHEMA)).toEqual( { - 'name': 'test', 'type': 'record', - 'dialect': 'trino', - 'join': 'one', 'fields': recordSchema, } ); @@ -88,11 +81,8 @@ describe('Trino connection', () => { it('parses nested', () => { expect(connection.malloyTypeFromTrinoType('test', NESTED_SCHEMA)).toEqual( { - 'name': 'test', 'type': 'array', 'elementTypeDef': {type: 'record_element'}, - 'dialect': 'trino', - 'join': 'many', 'fields': recordSchema, } ); @@ -106,11 +96,8 @@ describe('Trino connection', () => { it('parses deep nesting', () => { expect(connection.malloyTypeFromTrinoType('test', DEEP_SCHEMA)).toEqual({ - 'name': 'test', 'type': 'array', - 'dialect': 'trino', 'elementTypeDef': {type: 'record_element'}, - 'join': 'many', 'fields': [ {'name': 'a', ...doubleType}, { diff --git a/packages/malloy-db-trino/src/trino_connection.ts b/packages/malloy-db-trino/src/trino_connection.ts index 0fc3f38c1..b12fe5018 100644 --- a/packages/malloy-db-trino/src/trino_connection.ts +++ b/packages/malloy-db-trino/src/trino_connection.ts @@ -37,10 +37,14 @@ import { TableSourceDef, SQLSourceDef, AtomicTypeDef, - ArrayDef, + mkFieldDef, + isScalarArray, RepeatedRecordTypeDef, RecordTypeDef, - arrayEachFields, + Dialect, + ArrayTypeDef, + FieldDef, + TinyParser, isRepeatedRecord, } from '@malloydata/malloy'; @@ -163,33 +167,19 @@ export abstract class TrinoPrestoConnection extends BaseConnection implements Connection, PersistSQLResults { - public name: string; - private readonly dialect = new TrinoDialect(); + protected readonly dialect = new TrinoDialect(); static DEFAULT_QUERY_OPTIONS: RunSQLOptions = { rowLimit: 10, }; - private queryOptions?: QueryOptionsReader; - - //private config: TrinoConnectionConfiguration; - - private client: BaseRunner; - constructor( - name: string, - queryOptions?: QueryOptionsReader, - pConfig?: TrinoConnectionConfiguration + public name: string, + private client: BaseRunner, + private queryOptions?: QueryOptionsReader ) { super(); - const config = pConfig || {}; this.name = name; - if (name === 'trino') { - this.client = new TrinoRunner(config); - } else { - this.client = new PrestoRunner(config); - } this.queryOptions = queryOptions; - //this.config = config; } get dialectName(): string { @@ -225,18 +215,19 @@ export abstract class TrinoPrestoConnection return data as unknown[]; } - convertRow(structDef: StructDef, rawRow: unknown) { + convertRow(fields: FieldDef[], rawRow: unknown) { const retRow = {}; const row = this.unpackArray(rawRow); - for (let i = 0; i < structDef.fields.length; i++) { - const field = structDef.fields[i]; + for (let i = 0; i < fields.length; i++) { + const field = fields[i]; if (field.type === 'record') { - retRow[field.name] = this.convertRow(field, row[i]); + retRow[field.name] = this.convertRow(field.fields, row[i]); } else if (isRepeatedRecord(field)) { - retRow[field.name] = this.convertNest(field, row[i]); + retRow[field.name] = this.convertNest(field.fields, row[i]); } else if (field.type === 'array') { - retRow[field.name] = this.convertNest(field, row[i]); + // mtoy todo don't understand this line actually + retRow[field.name] = this.convertNest(field.fields.slice(0, 1), row[i]); } else { retRow[field.name] = row[i] ?? null; } @@ -245,12 +236,12 @@ export abstract class TrinoPrestoConnection return retRow; } - convertNest(structDef: StructDef, _data: unknown) { + convertNest(fields: FieldDef[], _data: unknown) { const data = this.unpackArray(_data); const ret: unknown[] = []; const rows = (data === null || data === undefined ? [] : data) as unknown[]; for (const row of rows) { - ret.push(this.convertRow(structDef, row)); + ret.push(this.convertRow(fields, row)); } return ret; } @@ -280,28 +271,7 @@ export abstract class TrinoPrestoConnection for (let i = 0; i < columns.length; i++) { const column = columns[i]; const schemaColumn = malloyColumns[i]; - if (schemaColumn.type === 'record') { - malloyRow[column.name] = this.convertRow(schemaColumn, row[i]); - } else if (schemaColumn.type === 'array') { - malloyRow[column.name] = this.convertNest( - schemaColumn, - row[i] - ) as QueryValue; - } else if ( - schemaColumn.type === 'number' && - typeof row[i] === 'string' - ) { - // decimal numbers come back as strings - malloyRow[column.name] = Number(row[i]); - } else if ( - schemaColumn.type === 'timestamp' && - typeof row[i] === 'string' - ) { - // timestamps come back as strings - malloyRow[column.name] = new Date(row[i] as string); - } else { - malloyRow[column.name] = row[i] as QueryValue; - } + malloyRow[column.name] = this.resultRow(schemaColumn, row[i]); } malloyRows.push(malloyRow); @@ -310,6 +280,29 @@ export abstract class TrinoPrestoConnection return {rows: malloyRows, totalRows: malloyRows.length}; } + private resultRow(colSchema: AtomicTypeDef, rawRow: unknown) { + if (colSchema.type === 'record') { + return this.convertRow(colSchema.fields, rawRow); + } else if (isRepeatedRecord(colSchema)) { + return this.convertNest(colSchema.fields, rawRow) as QueryValue; + } else if (isScalarArray(colSchema)) { + const elType = colSchema.elementTypeDef; + let theArray = this.unpackArray(rawRow); + if (elType.type === 'array') { + theArray = theArray.map(el => this.resultRow(elType, el)); + } + return theArray as QueryData; + } else if (colSchema.type === 'number' && typeof rawRow === 'string') { + // decimal numbers come back as strings + return Number(rawRow); + } else if (colSchema.type === 'timestamp' && typeof rawRow === 'string') { + // timestamps come back as strings + return new Date(rawRow as string); + } else { + return rawRow as QueryValue; + } + } + public async runSQLBlockAndFetchResultSchema( _sqlBlock: SQLSourceDef, _options?: RunSQLOptions @@ -401,21 +394,14 @@ export abstract class TrinoPrestoConnection if (innerType.type === 'record') { const complexStruct: RepeatedRecordTypeDef = { type: 'array', - name, elementTypeDef: {type: 'record_element'}, - dialect: this.dialectName, - join: 'many', fields: innerType.fields, }; return complexStruct; } else { - const arrayStruct: ArrayDef = { + const arrayStruct: ArrayTypeDef = { type: 'array', - name, elementTypeDef: innerType, - dialect: this.dialectName, - join: 'many', - fields: arrayEachFields(innerType), }; return arrayStruct; } @@ -426,9 +412,6 @@ export abstract class TrinoPrestoConnection const innerTypes = this.splitColumns(structMatch[3]); const recordType: RecordTypeDef = { type: 'record', - name, - dialect: this.dialectName, - join: 'one', fields: [], }; for (let innerType of innerTypes) { @@ -447,7 +430,9 @@ export abstract class TrinoPrestoConnection innerName, innerTrinoType ); - recordType.fields.push({...innerMalloyType, name: innerName}); + recordType.fields.push( + mkFieldDef(innerMalloyType, innerName, this.dialectName) + ); } } return recordType; @@ -461,7 +446,7 @@ export abstract class TrinoPrestoConnection const type = row[4] || row[1]; const malloyType = this.malloyTypeFromTrinoType(name, type); // console.log('>', row, '\n<', malloyType); - structDef.fields.push({name, ...malloyType}); + structDef.fields.push(mkFieldDef(malloyType, name, this.dialectName)); } } @@ -536,7 +521,11 @@ export class PrestoConnection extends TrinoPrestoConnection { queryOptions?: QueryOptionsReader, config: TrinoConnectionConfiguration = {} ) { - super('presto', queryOptions, config); + super( + typeof arg === 'string' ? arg : arg.name, + new PrestoRunner(config), + queryOptions + ); } protected async fillStructDefForSqlBlockSchema( @@ -574,41 +563,8 @@ export class PrestoConnection extends TrinoPrestoConnection { ); } - let outputLine = lines[0]; - - const namesIndex = outputLine.indexOf(']['); - outputLine = outputLine.substring(namesIndex + 2); - - const lineParts = outputLine.split('] => ['); - - if (lineParts.length !== 2) { - throw new Error('There was a problem parsing schema from Explain.'); - } - - const fieldNamesPart = lineParts[0]; - const fieldNames = fieldNamesPart.split(',').map(e => e.trim()); - - let schemaData = lineParts[1]; - schemaData = schemaData.substring(0, schemaData.length - 1); - const rawFieldsTarget = schemaData - .split(',') - .map(e => e.trim()) - .map(e => e.split(':')); - - if (rawFieldsTarget.length !== fieldNames.length) { - throw new Error( - 'There was a problem parsing schema from Explain. Field names size do not match target fields with types.' - ); - } - - for (let index = 0; index < fieldNames.length; index++) { - const name = fieldNames[index]; - const type = rawFieldsTarget[index][1]; - structDef.fields.push({ - name, - ...this.malloyTypeFromTrinoType(name, type), - }); - } + const schemaDesc = new PrestoExplainParser(lines[0], this.dialect); + structDef.fields = schemaDesc.parseExplain(); } unpackArray(data: unknown): unknown[] { @@ -631,7 +587,11 @@ export class TrinoConnection extends TrinoPrestoConnection { queryOptions?: QueryOptionsReader, config: TrinoConnectionConfiguration = {} ) { - super('trino', queryOptions, config); + super( + typeof arg === 'string' ? arg : arg.name, + new TrinoRunner(config), + queryOptions + ); } protected async fillStructDefForSqlBlockSchema( @@ -647,3 +607,128 @@ export class TrinoConnection extends TrinoPrestoConnection { ); } } + +/** + * A hand built parser for schema lines, roughly this grammar + * SCHEMA_LINE: - Output [PlanName N] [NAME_LIST] => [TYPE_LIST] + * NAME_LIST: NAME (, NAME)* + * TYPE_LIST: TYPE_SPEC (, TYPE_SPEC)* + * TYPE_SPEC: exprN ':' TYPE + * TYPE: REC_TYPE | ARRAY_TYPE | SQL_TYPE + * ARRAY_TYPE: ARRAY '(' TYPE ')' + * REC_TYPE: REC '(' "name" TYPE (, "name" TYPE)* ')' + */ +export class PrestoExplainParser extends TinyParser { + constructor( + readonly input: string, + readonly dialect: Dialect + ) { + super(input, { + space: /^\s+/, + arrow: /^=>/, + char: /^[,:[\]()-]/, + id: /^\w+/, + quoted_name: /^"\w+"/, + }); + } + + fieldNameList(): string[] { + this.skipTo(']'); // Skip to end of plan + this.next('['); // Expect start of name list + const fieldNames: string[] = []; + for (;;) { + const nmToken = this.next('id'); + fieldNames.push(nmToken.text); + const sep = this.next(); + if (sep.type === ',') { + continue; + } + if (sep.type !== ']') { + throw this.parseError( + `Unexpected '${sep.text}' while getting field name list` + ); + } + break; + } + return fieldNames; + } + + parseExplain(): FieldDef[] { + const fieldNames = this.fieldNameList(); + const fields: FieldDef[] = []; + this.next('arrow', '['); + for (let nameIndex = 0; ; nameIndex += 1) { + const name = fieldNames[nameIndex]; + this.next('id', ':'); + const nextType = this.typeDef(); + fields.push(mkFieldDef(nextType, name, this.dialect.name)); + const sep = this.next(); + if (sep.text === ',') { + continue; + } + if (sep.text !== ']') { + throw this.parseError(`Unexpected '${sep.text}' between field types`); + } + break; + } + if (fields.length !== fieldNames.length) { + throw new Error( + `Presto schema error mismatched ${fields.length} types and ${fieldNames.length} fields` + ); + } + return fields; + } + + typeDef(): AtomicTypeDef { + const typToken = this.next(); + if (typToken.type === 'eof') { + throw this.parseError( + 'Unexpected EOF parsing type, expected a type name' + ); + } else if (typToken.text === 'row' && this.next('(')) { + const fields: FieldDef[] = []; + for (;;) { + const name = this.next('quoted_name'); + const getDef = this.typeDef(); + fields.push(mkFieldDef(getDef, name.text, this.dialect.name)); + const sep = this.next(); + if (sep.text === ')') { + break; + } + if (sep.text === ',') { + continue; + } + } + const def: RecordTypeDef = { + type: 'record', + fields, + }; + return def; + } else if (typToken.text === 'array' && this.next('(')) { + const elType = this.typeDef(); + this.next(')'); + return elType.type === 'record' + ? { + type: 'array', + elementTypeDef: {type: 'record_element'}, + fields: elType.fields, + } + : {type: 'array', elementTypeDef: elType}; + } else if (typToken.type === 'id') { + const sqlType = typToken.text; + const def = this.dialect.sqlTypeToMalloyType(sqlType); + if (def === undefined) { + throw this.parseError(`Can't parse presto type ${sqlType}`); + } + if (sqlType === 'varchar') { + if (this.peek().type === '(') { + this.next('(', 'id', ')'); + } + } + return def; + } + throw this.parseError( + `'${typToken.text}' unexpected while looking for a type` + ); + } +} diff --git a/packages/malloy-interfaces/package.json b/packages/malloy-interfaces/package.json index dfb01e1ec..fab4a7bff 100644 --- a/packages/malloy-interfaces/package.json +++ b/packages/malloy-interfaces/package.json @@ -1,6 +1,6 @@ { "name": "@malloydata/malloy-interfaces", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/malloy-malloy-sql/package.json b/packages/malloy-malloy-sql/package.json index 38ed24e1e..e8eaab858 100644 --- a/packages/malloy-malloy-sql/package.json +++ b/packages/malloy-malloy-sql/package.json @@ -1,6 +1,6 @@ { "name": "@malloydata/malloy-sql", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -24,7 +24,7 @@ "prepublishOnly": "npm run build" }, "dependencies": { - "@malloydata/malloy": "^0.0.218" + "@malloydata/malloy": "^0.0.222" }, "devDependencies": { "peggy": "^3.0.2" diff --git a/packages/malloy-render/.storybook/malloy-stories-indexer.ts b/packages/malloy-render/.storybook/malloy-stories-indexer.ts index a68d4c646..80670a91c 100644 --- a/packages/malloy-render/.storybook/malloy-stories-indexer.ts +++ b/packages/malloy-render/.storybook/malloy-stories-indexer.ts @@ -153,6 +153,9 @@ export function viteMalloyStoriesPlugin(): PluginOption { const el = document.createElement('malloy-render'); if (classes) el.classList.add(classes); el.result = context.loaded['result']; + el.tableConfig = { + enableDrill: true + }; const button = document.createElement('button'); button.innerHTML = "Copy HTML"; diff --git a/packages/malloy-render/package.json b/packages/malloy-render/package.json index caaf6ec59..84f4a30cb 100644 --- a/packages/malloy-render/package.json +++ b/packages/malloy-render/package.json @@ -1,6 +1,6 @@ { "name": "@malloydata/render", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "main": "dist/module/index.umd.js", "types": "dist/index.d.ts", @@ -30,7 +30,7 @@ "build-types": "tsc --build --declaration --emitDeclarationOnly" }, "dependencies": { - "@malloydata/malloy": "^0.0.218", + "@malloydata/malloy": "^0.0.222", "@tanstack/solid-virtual": "^3.10.4", "component-register": "^0.8.6", "lodash": "^4.17.20", diff --git a/packages/malloy-render/src/component/apply-renderer.tsx b/packages/malloy-render/src/component/apply-renderer.tsx index 9bd1bd4df..3a27d9b33 100644 --- a/packages/malloy-render/src/component/apply-renderer.tsx +++ b/packages/malloy-render/src/component/apply-renderer.tsx @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import { @@ -25,7 +25,6 @@ import {renderList} from './render-list'; import {renderImage} from './render-image'; import {Dashboard} from './dashboard/dashboard'; import {LegacyChart} from './legacy-charts/legacy_chart'; -import {hasAny} from './tag-utils'; import {renderTime} from './render-time'; export type RendererProps = { @@ -36,27 +35,47 @@ export type RendererProps = { customProps?: Record>; }; +const RENDER_TAG_LIST = [ + 'link', + 'image', + 'cell', + 'list', + 'list_detail', + 'bar_chart', + 'line_chart', + 'dashboard', + 'scatter_chart', + 'shape_map', + 'segment_map', +]; + +const CHART_TAG_LIST = ['bar_chart', 'line_chart']; + +export function shouldRenderChartAs(tag: Tag) { + const tagNamesInOrder = Object.keys(tag.properties ?? {}).reverse(); + return tagNamesInOrder.find(name => CHART_TAG_LIST.includes(name)); +} + export function shouldRenderAs(f: Field | Explore, tagOverride?: Tag) { const tag = tagOverride ?? f.tagParse().tag; - if (!f.isExplore() && f.isAtomicField()) { - if (tag.has('link')) return 'link'; - if (tag.has('image')) return 'image'; - return 'cell'; + const tagNamesInOrder = Object.keys(tag.properties ?? {}).reverse(); + for (const tagName of tagNamesInOrder) { + if (RENDER_TAG_LIST.includes(tagName)) { + if (['list', 'list_detail'].includes(tagName)) return 'list'; + if (['bar_chart', 'line_chart'].includes(tagName)) return 'chart'; + return tagName; + } } - if (hasAny(tag, 'list', 'list_detail')) return 'list'; - if (hasAny(tag, 'bar_chart', 'line_chart')) return 'chart'; - if (tag.has('dashboard')) return 'dashboard'; - if (hasAny(tag, 'scatter_chart')) return 'scatter_chart'; - if (hasAny(tag, 'shape_map')) return 'shape_map'; - if (hasAny(tag, 'segment_map')) return 'segment_map'; - else return 'table'; + + if (!f.isExplore() && f.isAtomicField()) return 'cell'; + return 'table'; } export const NULL_SYMBOL = '∅'; export function applyRenderer(props: RendererProps) { - const {field, dataColumn, resultMetadata, tag, customProps = {}} = props; - const renderAs = shouldRenderAs(field, tag); + const {field, dataColumn, resultMetadata, customProps = {}} = props; + const renderAs = resultMetadata.field(field).renderAs; let renderValue: JSXElement = ''; const propsToPass = customProps[renderAs] || {}; if (dataColumn.isNull()) { diff --git a/packages/malloy-render/src/component/bar-chart/generate-bar_chart-vega-spec.ts b/packages/malloy-render/src/component/bar-chart/generate-bar_chart-vega-spec.ts index d66c9bad7..b284d855d 100644 --- a/packages/malloy-render/src/component/bar-chart/generate-bar_chart-vega-spec.ts +++ b/packages/malloy-render/src/component/bar-chart/generate-bar_chart-vega-spec.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {Explore, QueryDataRow, QueryValue} from '@malloydata/malloy'; @@ -708,6 +708,7 @@ export function generateBarChartVegaSpec( fill: 'color', // No title for measure list legends title: seriesField ? seriesField.name : '', + orient: 'right', ...legendSettings, encode: { entries: { diff --git a/packages/malloy-render/src/component/bar-chart/get-custom-tooltips-entries.ts b/packages/malloy-render/src/component/bar-chart/get-custom-tooltips-entries.ts index a36d2182b..ca12b52c6 100644 --- a/packages/malloy-render/src/component/bar-chart/get-custom-tooltips-entries.ts +++ b/packages/malloy-render/src/component/bar-chart/get-custom-tooltips-entries.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {Explore} from '@malloydata/malloy'; diff --git a/packages/malloy-render/src/component/chart/chart.tsx b/packages/malloy-render/src/component/chart/chart.tsx index 5fcf79535..5a184e716 100644 --- a/packages/malloy-render/src/component/chart/chart.tsx +++ b/packages/malloy-render/src/component/chart/chart.tsx @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {Explore, ExploreField, QueryData} from '@malloydata/malloy'; diff --git a/packages/malloy-render/src/component/chart/default-chart-tooltip.tsx b/packages/malloy-render/src/component/chart/default-chart-tooltip.tsx index 2e4cd9c0f..5f6aba919 100644 --- a/packages/malloy-render/src/component/chart/default-chart-tooltip.tsx +++ b/packages/malloy-render/src/component/chart/default-chart-tooltip.tsx @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {For, Match, Show, Switch, createEffect, createSignal} from 'solid-js'; diff --git a/packages/malloy-render/src/component/dashboard/dashboard.tsx b/packages/malloy-render/src/component/dashboard/dashboard.tsx index d7c197cbe..40ab1b3db 100644 --- a/packages/malloy-render/src/component/dashboard/dashboard.tsx +++ b/packages/malloy-render/src/component/dashboard/dashboard.tsx @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {DataArray, DataRecord, Field} from '@malloydata/malloy'; diff --git a/packages/malloy-render/src/component/line-chart/generate-line_chart-vega-spec.ts b/packages/malloy-render/src/component/line-chart/generate-line_chart-vega-spec.ts index 4a5f0ee6c..e88dea82c 100644 --- a/packages/malloy-render/src/component/line-chart/generate-line_chart-vega-spec.ts +++ b/packages/malloy-render/src/component/line-chart/generate-line_chart-vega-spec.ts @@ -669,6 +669,7 @@ export function generateLineChartVegaSpec( fill: 'color', // No title for measure list legends title: seriesField ? seriesField.name : '', + orient: 'right', ...legendSettings, encode: { entries: { diff --git a/packages/malloy-render/src/component/register-webcomponent.ts b/packages/malloy-render/src/component/register-webcomponent.ts index 77a17ff2b..e1468d1ae 100644 --- a/packages/malloy-render/src/component/register-webcomponent.ts +++ b/packages/malloy-render/src/component/register-webcomponent.ts @@ -20,6 +20,7 @@ export default function registerWebComponent({ modelDef: undefined, scrollEl: undefined, onClick: undefined, + onDrill: undefined, vegaConfigOverride: undefined, tableConfig: undefined, dashboardConfig: undefined, diff --git a/packages/malloy-render/src/component/render-result-metadata.ts b/packages/malloy-render/src/component/render-result-metadata.ts index 1ccf098a9..89b31ee9e 100644 --- a/packages/malloy-render/src/component/render-result-metadata.ts +++ b/packages/malloy-render/src/component/render-result-metadata.ts @@ -37,21 +37,24 @@ import { valueIsNumber, valueIsString, } from './util'; -import {hasAny} from './tag-utils'; import { DataRowWithRecord, RenderResultMetadata, VegaChartProps, VegaConfigHandler, } from './types'; -import {NULL_SYMBOL, shouldRenderAs} from './apply-renderer'; +import { + NULL_SYMBOL, + shouldRenderAs, + shouldRenderChartAs, +} from './apply-renderer'; import {mergeVegaConfigs} from './vega/merge-vega-configs'; import {baseVegaConfig} from './vega/base-vega-config'; import {renderTimeString} from './render-time'; import {generateBarChartVegaSpec} from './bar-chart/generate-bar_chart-vega-spec'; import {createResultStore} from './result-store/result-store'; import {generateLineChartVegaSpec} from './line-chart/generate-line_chart-vega-spec'; -import {parse} from 'vega'; +import {parse, Config} from 'vega'; function createDataCache() { const dataCache = new WeakMap(); @@ -59,11 +62,16 @@ function createDataCache() { get: (cell: DataColumn) => { if (!dataCache.has(cell) && cell.isArray()) { const data: DataRowWithRecord[] = []; + const fields = cell.field.allFields; for (const row of cell) { - const record = Object.assign({}, row.toObject(), { - __malloyDataRecord: row, - }); - data.push(record); + const record = {__malloyDataRecord: row}; + for (const field of fields) { + const value = row.cell(field).value; + // TODO: can we store Date objects as is? Downstream chart code would need to be updated + record[field.name] = + value instanceof Date ? value.valueOf() : row.cell(field).value; + } + data.push(record as DataRowWithRecord); } dataCache.set(cell, data); } @@ -245,17 +253,32 @@ function populateExploreMeta( ) { const fieldMeta = metadata.field(f); let vegaChartProps: VegaChartProps | null = null; - if (hasAny(tag, 'bar', 'bar_chart')) { + const chartType = shouldRenderChartAs(tag); + if (chartType === 'bar_chart') { vegaChartProps = generateBarChartVegaSpec(f, metadata); - } else if (tag.has('line_chart')) { + } else if (chartType === 'line_chart') { vegaChartProps = generateLineChartVegaSpec(f, metadata); } if (vegaChartProps) { - const vegaConfig = mergeVegaConfigs( + const vegaConfigOverride = + options.getVegaConfigOverride?.(vegaChartProps.chartType) ?? {}; + + const vegaConfig: Config = mergeVegaConfigs( baseVegaConfig(), options.getVegaConfigOverride?.(vegaChartProps.chartType) ?? {} ); + + const maybeAxisYLabelFont = vegaConfigOverride['axisY']?.['labelFont']; + const maybeAxisLabelFont = vegaConfigOverride['axis']?.['labelFont']; + if (maybeAxisYLabelFont || maybeAxisLabelFont) { + const refLineFontSignal = vegaConfig.signals?.find( + signal => signal.name === 'referenceLineFont' + ); + if (refLineFontSignal) + refLineFontSignal.value = maybeAxisYLabelFont ?? maybeAxisLabelFont; + } + fieldMeta.vegaChartProps = { ...vegaChartProps, spec: { diff --git a/packages/malloy-render/src/component/render.css b/packages/malloy-render/src/component/render.css index d430704c1..2f8b7f589 100644 --- a/packages/malloy-render/src/component/render.css +++ b/packages/malloy-render/src/component/render.css @@ -28,3 +28,19 @@ 'calt' 1; } } + +.malloy-copied-modal { + position: fixed; + background: #333; + font-size: 13px; + padding: 6px 12px; + border-radius: 4px; + box-shadow: rgba(100, 100, 111, 0.2) 0px 7px 29px 0px; + color: white; + bottom: 24px; + left: 100%; + text-wrap: nowrap; + + transition: all 0s; + animation: modal-slide-in 2s forwards; +} diff --git a/packages/malloy-render/src/component/render.tsx b/packages/malloy-render/src/component/render.tsx index cc174588a..8e3d05d9f 100644 --- a/packages/malloy-render/src/component/render.tsx +++ b/packages/malloy-render/src/component/render.tsx @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import { @@ -27,10 +27,12 @@ import {ComponentOptions, ICustomElement} from 'component-register'; import {applyRenderer} from './apply-renderer'; import { DashboardConfig, + DrillData, MalloyClickEventPayload, TableConfig, VegaConfigHandler, } from './types'; +export type {DimensionContextEntry, DrillData} from './types'; import css from './render.css?raw'; export type MalloyRenderProps = { @@ -40,6 +42,7 @@ export type MalloyRenderProps = { scrollEl?: HTMLElement; modalElement?: HTMLElement; onClick?: (payload: MalloyClickEventPayload) => void; + onDrill?: (drillData: DrillData) => void; vegaConfigOverride?: VegaConfigHandler; tableConfig?: Partial; dashboardConfig?: Partial; @@ -53,6 +56,7 @@ const ConfigContext = createContext<{ addCSSToShadowRoot: (css: string) => void; addCSSToDocument: (id: string, css: string) => void; onClick?: (payload: MalloyClickEventPayload) => void; + onDrill?: (drillData: DrillData) => void; vegaConfigOverride?: VegaConfigHandler; modalElement?: HTMLElement; }>(); @@ -122,6 +126,7 @@ export function MalloyRender( disableVirtualization: false, rowLimit: Infinity, shouldFillWidth: false, + enableDrill: false, }, props.tableConfig ); @@ -139,6 +144,7 @@ export function MalloyRender( - {rendering().renderValue} - + <> + + {rendering().renderValue} + + +
Copied query to clipboard!
+ + ); } diff --git a/packages/malloy-render/src/component/result-store/result-store.ts b/packages/malloy-render/src/component/result-store/result-store.ts index 09ee7fe43..b2de11cf4 100644 --- a/packages/malloy-render/src/component/result-store/result-store.ts +++ b/packages/malloy-render/src/component/result-store/result-store.ts @@ -1,5 +1,7 @@ import {createStore, produce, unwrap} from 'solid-js/store'; import {useResultContext} from '../result-context'; +import {DrillData, RenderResultMetadata, DimensionContextEntry} from '../types'; +import {Explore, Field} from '@malloydata/malloy'; interface BrushDataBase { fieldRefId: string; @@ -45,11 +47,13 @@ export type VegaBrushOutput = { export interface ResultStoreData { brushes: BrushData[]; + showCopiedModal: boolean; } export function createResultStore() { const [store, setStore] = createStore({ brushes: [], + showCopiedModal: false, }); const getFieldBrushBySourceId = ( @@ -108,6 +112,20 @@ export function createResultStore() { return { store, applyBrushOps, + triggerCopiedModal: (time = 2000) => { + setStore( + produce(state => { + state.showCopiedModal = true; + }) + ); + setTimeout(() => { + setStore( + produce(state => { + state.showCopiedModal = false; + }) + ); + }, time); + }, }; } @@ -117,3 +135,48 @@ export function useResultStore() { const metadata = useResultContext(); return metadata.store; } + +export async function copyExplorePathQueryToClipboard({ + metadata, + field, + dimensionContext, + onDrill, +}: { + metadata: RenderResultMetadata; + field: Field; + dimensionContext: DimensionContextEntry[]; + onDrill?: (drillData: DrillData) => void; +}) { + const dimensionContextEntries = dimensionContext; + let explore: Field | Explore = field; + while (explore.parentExplore) { + explore = explore.parentExplore; + } + + const whereClause = dimensionContextEntries + .map(entry => `\t\t${entry.fieldDef} = ${JSON.stringify(entry.value)}`) + .join(',\n'); + + const query = ` +run: ${explore.name} -> { +where: +${whereClause} +} + { select: * }`.trim(); + + const drillData: DrillData = { + dimensionFilters: dimensionContextEntries, + copyQueryToClipboard: async () => { + try { + await navigator.clipboard.writeText(query); + metadata.store.triggerCopiedModal(); + } catch (error) { + // eslint-disable-next-line no-console + console.error('Failed to copy text: ', error); + } + }, + query, + whereClause, + }; + if (onDrill) onDrill(drillData); + else await drillData.copyQueryToClipboard(); +} diff --git a/packages/malloy-render/src/component/table/table-context.ts b/packages/malloy-render/src/component/table/table-context.ts index c4198e727..965a9c953 100644 --- a/packages/malloy-render/src/component/table/table-context.ts +++ b/packages/malloy-render/src/component/table/table-context.ts @@ -1,18 +1,25 @@ import {createContext, useContext} from 'solid-js'; import {createStore, SetStoreFunction, Store} from 'solid-js/store'; import {TableLayout} from './table-layout'; +import {DimensionContextEntry} from '../types'; type TableStore = { headerSizes: Record; columnWidths: Record; + highlightedRow: number[] | null; + highlightedExplore: string[] | null; + showCopiedModal: boolean; }; -type TableContext = { +export type TableContext = { root: boolean; layout: TableLayout; store: Store; setStore: SetStoreFunction; headerSizeStore: ReturnType>>; + currentRow: number[]; + currentExplore: string[]; + dimensionContext: DimensionContextEntry[]; }; export const TableContext = createContext(); @@ -21,5 +28,8 @@ export function createTableStore() { return createStore({ headerSizes: {}, columnWidths: {}, + highlightedRow: null, + highlightedExplore: null, + showCopiedModal: false, }); } diff --git a/packages/malloy-render/src/component/table/table.css b/packages/malloy-render/src/component/table/table.css index 323be1df5..ee559b26d 100644 --- a/packages/malloy-render/src/component/table/table.css +++ b/packages/malloy-render/src/component/table/table.css @@ -217,3 +217,23 @@ .malloy-table .malloy-list { line-height: calc(var(--malloy-render--table-row-height) * 5 / 7 - 1px); } + +.malloy-table .td.highlight { + color: #547ce4; + cursor: pointer; +} + +@keyframes modal-slide-in { + 0% { + transform: translateX(0); + } + 10% { + transform: translateX(calc(-100% - 24px)); + } + 90% { + transform: translateX(calc(-100% - 24px)); + } + 100% { + transform: translateX(0); + } +} diff --git a/packages/malloy-render/src/component/table/table.tsx b/packages/malloy-render/src/component/table/table.tsx index 96d5e1925..9773104c5 100644 --- a/packages/malloy-render/src/component/table/table.tsx +++ b/packages/malloy-render/src/component/table/table.tsx @@ -11,17 +11,24 @@ import { JSX, onMount, } from 'solid-js'; -import {DataArrayOrRecord, DataRecord, Field} from '@malloydata/malloy'; +import { + AtomicField, + DataArrayOrRecord, + DataRecord, + Field, +} from '@malloydata/malloy'; import {getRangeSize, isFirstChild, isLastChild} from '../util'; import {getTableLayout} from './table-layout'; import {useResultContext} from '../result-context'; import {createTableStore, TableContext, useTableContext} from './table-context'; import tableCss from './table.css?raw'; -import {applyRenderer, shouldRenderAs} from '../apply-renderer'; +import {applyRenderer} from '../apply-renderer'; import {isFieldHidden} from '../../tags_utils'; import {createStore, produce} from 'solid-js/store'; import {createVirtualizer, Virtualizer} from '@tanstack/solid-virtual'; import {useConfig} from '../render'; +import {DimensionContextEntry} from '../types'; +import {copyExplorePathQueryToClipboard} from '../result-store/result-store'; const IS_CHROMIUM = navigator.userAgent.toLowerCase().indexOf('chrome') >= 0; // CSS Subgrid + Sticky Positioning only seems to work reliably in Chrome @@ -159,7 +166,13 @@ const HeaderField = (props: {field: Field; isPinned?: boolean}) => { ); }; -const TableField = (props: {field: Field; row: DataRecord}) => { +const DRILL_RENDERER_IGNORE_LIST = ['chart', 'link']; +const TableField = (props: { + field: Field; + row: DataRecord; + rowPath: number[]; + dimensionContext: DimensionContextEntry[]; +}) => { let renderValue: JSXElement = ''; let renderAs = ''; ({renderValue, renderAs} = applyRenderer({ @@ -170,12 +183,23 @@ const TableField = (props: {field: Field; row: DataRecord}) => { customProps: { table: { rowLimit: 100, // Limit nested tables to 100 records + currentRow: [...props.rowPath], + // dimensionContext, }, }, })); const tableLayout = useTableContext()!.layout; const fieldLayout = tableLayout.fieldLayout(props.field); const columnRange = fieldLayout.relativeColumnRange; + const tableCtx = useTableContext(); + const isHighlightedRow = () => { + return ( + JSON.stringify(props.rowPath) === + JSON.stringify(tableCtx?.store.highlightedRow) && + JSON.stringify(tableCtx?.currentExplore) === + JSON.stringify(tableCtx?.store.highlightedExplore) + ); + }; const style: JSX.CSSProperties = { 'grid-column': `${columnRange[0] + 1} / span ${getRangeSize(columnRange)}`, 'height': 'fit-content', @@ -194,12 +218,54 @@ const TableField = (props: {field: Field; row: DataRecord}) => { const tableGutterLeft = fieldLayout.depth > 0 && isFirstChild(props.field); const tableGutterRight = fieldLayout.depth > 0 && isLastChild(props.field); + const handleMouseEnter = () => { + if (tableCtx && !DRILL_RENDERER_IGNORE_LIST.includes(renderAs)) { + // TODO: only update if changed; need to check change via stringify + tableCtx.setStore(s => ({ + ...s, + highlightedRow: props.rowPath, + })); + } + }; + + const handleMouseLeave = () => { + tableCtx!.setStore(s => ({ + ...s, + highlightedRow: null, + })); + }; + + const config = useConfig(); + const isDrillingEnabled = config.tableConfig().enableDrill; + const metadata = useResultContext(); + const handleClick = async evt => { + evt.stopPropagation(); + if (isDrillingEnabled && !DRILL_RENDERER_IGNORE_LIST.includes(renderAs)) { + copyExplorePathQueryToClipboard({ + metadata, + field: props.field, + dimensionContext: [ + ...tableCtx!.dimensionContext, + ...props.dimensionContext, + ], + onDrill: config.onDrill, + }); + } + }; + return (
@@ -251,11 +317,12 @@ const MalloyTableRoot = (_props: { else return 0; }) .filter(([key, value]) => { + const field = resultMetadata.fields[key].field; + const parentFieldRenderer = field.parentExplore + ? resultMetadata.field(field.parentExplore)?.renderAs + : null; const isNotRoot = value.depth >= 0; - const isPartOfTable = - isNotRoot && - shouldRenderAs(resultMetadata.fields[key].field.parentExplore!) === - 'table'; + const isPartOfTable = isNotRoot && parentFieldRenderer === 'table'; return isPartOfTable; }) .map(([key, value]) => ({ @@ -495,6 +562,35 @@ const MalloyTableRoot = (_props: { 'margin-bottom': `calc(-1 * (var(--malloy-render--table-header-cumulative-height-${maxPinnedHeaderDepth()}) - var(--malloy-render--table-header-height-0)))`, }); + const getRowPath = (rowIndex?: number) => { + if (typeof rowIndex === 'number') return [...tableCtx.currentRow, rowIndex]; + return tableCtx.currentRow; + }; + + const handleTableMouseOver = (evt: MouseEvent) => { + evt.stopPropagation(); + + tableCtx.setStore(s => ({ + ...s, + highlightedExplore: props.data.field.fieldPath, + })); + }; + + const getRowDimensionContext = (row: DataRecord) => { + const dimensionContext: DimensionContextEntry[] = []; + + const dimensions = row.field.allFields.filter( + f => f.isAtomicField() && f.sourceWasDimension() + ) as AtomicField[]; + dimensions.forEach(field => { + dimensionContext.push({ + fieldDef: field.expression, + value: row.cell(field).value as string | number | boolean | Date, + }); + }); + return dimensionContext; + }; + return (
{/* pinned header */} @@ -566,6 +665,7 @@ const MalloyTableRoot = (_props: {
queueMicrotask(() => { virtualizer!.measureElement(el); @@ -582,6 +682,10 @@ const MalloyTableRoot = (_props: { )} @@ -596,7 +700,7 @@ const MalloyTableRoot = (_props: { {/* header */} -
+
{field => } @@ -604,10 +708,17 @@ const MalloyTableRoot = (_props: { {/* rows */} - {row => ( -
+ {(row, idx) => ( +
- {field => } + {field => ( + + )}
)} @@ -628,16 +739,37 @@ const MalloyTable: Component<{ scrollEl?: HTMLElement; disableVirtualization?: boolean; shouldFillWidth?: boolean; + currentRow?: number[]; + dimensionContext?: DimensionContextEntry[]; }> = props => { const metadata = useResultContext(); const hasTableCtx = !!useTableContext(); - const tableCtx = createMemo(() => { + const tableCtx = createMemo(() => { if (hasTableCtx) { const parentCtx = useTableContext()!; - + const parentRecord = props.data.parentRecord; + const dimensionContext: DimensionContextEntry[] = []; + if (parentRecord) { + const dimensions = parentRecord.field.allFields.filter( + f => f.isAtomicField() && f.sourceWasDimension() + ) as AtomicField[]; + dimensions.forEach(field => { + dimensionContext.push({ + fieldDef: field.expression, + value: parentRecord.cell(field).value as + | string + | number + | boolean + | Date, + }); + }); + } return { ...parentCtx, root: false, + currentRow: props.currentRow ?? parentCtx.currentRow, + currentExplore: props.data.field.fieldPath, + dimensionContext: [...parentCtx.dimensionContext, ...dimensionContext], }; } @@ -648,6 +780,9 @@ const MalloyTable: Component<{ store, setStore, headerSizeStore: createStore({}), + currentRow: [], + currentExplore: props.data.field.fieldPath, + dimensionContext: [], }; }); diff --git a/packages/malloy-render/src/component/types.ts b/packages/malloy-render/src/component/types.ts index 656a174b0..6dedb560b 100644 --- a/packages/malloy-render/src/component/types.ts +++ b/packages/malloy-render/src/component/types.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import { @@ -119,7 +119,20 @@ export type TableConfig = { disableVirtualization: boolean; rowLimit: number; shouldFillWidth: boolean; + enableDrill: boolean; }; export type DashboardConfig = { disableVirtualization: boolean; }; + +export type DrillData = { + dimensionFilters: DimensionContextEntry[]; + copyQueryToClipboard: () => Promise; + query: string; + whereClause: string; +}; + +export type DimensionContextEntry = { + fieldDef: string; + value: string | number | boolean | Date; +}; diff --git a/packages/malloy-render/src/component/vega/base-vega-config.ts b/packages/malloy-render/src/component/vega/base-vega-config.ts index 42395c79f..3dcc9bca6 100644 --- a/packages/malloy-render/src/component/vega/base-vega-config.ts +++ b/packages/malloy-render/src/component/vega/base-vega-config.ts @@ -43,40 +43,36 @@ export const baseVegaConfig: () => Config = () => ({ 'ordinal': ['#05214D', '#083E89', '#1877F2', '#76B6FF', '#A8D1FF'], 'ramp': ['#05214D', '#083E89', '#1877F2', '#76B6FF', '#A8D1FF'], }, - 'axisY': { + 'axis': { gridColor: gridGray, - grid: true, tickColor: gridGray, domain: false, labelFont: 'Inter, sans-serif', labelFontSize: 10, labelFontWeight: 'normal', - labelColor: grayMedium, labelPadding: 5, + labelColor: grayMedium, titleColor: grayMedium, titleFont: 'Inter, sans-serif', titleFontSize: 10, titleFontWeight: 500, - titlePadding: 10, - labelOverlap: false, }, 'axisX': { - gridColor: gridGray, - tickColor: gridGray, tickSize: 0, - domain: false, - labelFont: 'Inter, sans-serif', - labelFontSize: 10, - labelFontWeight: 'normal', - labelPadding: 5, - labelColor: grayMedium, - titleColor: grayMedium, - titleFont: 'Inter, sans-serif', - titleFontSize: 10, - titleFontWeight: 500, titlePadding: 16, }, + 'axisY': { + grid: true, + labelOverlap: false, + titlePadding: 10, + }, 'view': { strokeWidth: 0, }, + 'signals': [ + { + name: 'referenceLineFont', + value: 'Inter, sans-serif', + }, + ], }); diff --git a/packages/malloy-render/src/component/vega/measure-axis.ts b/packages/malloy-render/src/component/vega/measure-axis.ts index a66ad942c..1771593c5 100644 --- a/packages/malloy-render/src/component/vega/measure-axis.ts +++ b/packages/malloy-render/src/component/vega/measure-axis.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {Axis, GroupMark, RectMark} from 'vega'; @@ -239,7 +239,7 @@ function createAxisReferenceLines(options: AxisReferenceLineOptions) { strokeWidth: {value: 3}, fontSize: {value: 10}, fontWeight: {value: 'normal'}, - font: {value: 'Inter, sans-serif'}, + font: {signal: 'referenceLineFont'}, strokeOpacity: {value: 1}, }, update: { @@ -267,7 +267,7 @@ function createAxisReferenceLines(options: AxisReferenceLineOptions) { fill: {value: grayMedium}, fontSize: {value: 10}, fontWeight: {value: 'normal'}, - font: {value: 'Inter, sans-serif'}, + font: {signal: 'referenceLineFont'}, }, update: { y: { @@ -409,7 +409,7 @@ function createAxisReferenceLines(options: AxisReferenceLineOptions) { // strokeWidth: {value: 3}, // fontSize: {value: 10}, // fontWeight: {value: 'normal'}, -// font: {value: 'Inter, sans-serif'}, +// font: {signal: "referenceLineFont"}, // strokeOpacity: {value: 1}, // }, // update: { @@ -443,7 +443,7 @@ function createAxisReferenceLines(options: AxisReferenceLineOptions) { // fill: {value: grayMedium}, // fontSize: {value: 10}, // fontWeight: {value: 'normal'}, -// font: {value: 'Inter, sans-serif'}, +// font: {signal: "referenceLineFont"}, // }, // update: { // y: { @@ -479,7 +479,7 @@ function createAxisReferenceLines(options: AxisReferenceLineOptions) { // strokeWidth: {value: 3}, // fontSize: {value: 10}, // fontWeight: {value: 'normal'}, -// font: {value: 'Inter, sans-serif'}, +// font: {signal: "referenceLineFont"}, // strokeOpacity: {value: 1}, // }, // update: { @@ -513,7 +513,7 @@ function createAxisReferenceLines(options: AxisReferenceLineOptions) { // fill: {value: grayMedium}, // fontSize: {value: 10}, // fontWeight: {value: 'normal'}, -// font: {value: 'Inter, sans-serif'}, +// font: {signal: "referenceLineFont"}, // }, // update: { // y: { diff --git a/packages/malloy-render/src/component/vega/vega-chart.tsx b/packages/malloy-render/src/component/vega/vega-chart.tsx index 582751ada..e30017d16 100644 --- a/packages/malloy-render/src/component/vega/vega-chart.tsx +++ b/packages/malloy-render/src/component/vega/vega-chart.tsx @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {createEffect, createSignal, untrack} from 'solid-js'; diff --git a/packages/malloy-render/src/component/vega/vega-expr-addons.ts b/packages/malloy-render/src/component/vega/vega-expr-addons.ts index dfc924bcc..1393506f8 100644 --- a/packages/malloy-render/src/component/vega/vega-expr-addons.ts +++ b/packages/malloy-render/src/component/vega/vega-expr-addons.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {expressionFunction} from 'vega'; diff --git a/packages/malloy-render/src/component/vega/vega-utils.ts b/packages/malloy-render/src/component/vega/vega-utils.ts index 1bae553ee..348e61125 100644 --- a/packages/malloy-render/src/component/vega/vega-utils.ts +++ b/packages/malloy-render/src/component/vega/vega-utils.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {Item, SignalListenerHandler, View} from 'vega'; diff --git a/packages/malloy-render/src/html/html_view.ts b/packages/malloy-render/src/html/html_view.ts index 193792383..9e066ff8b 100644 --- a/packages/malloy-render/src/html/html_view.ts +++ b/packages/malloy-render/src/html/html_view.ts @@ -52,6 +52,10 @@ export class HTMLView { if (hasNextRenderer) { const el = this.document.createElement('malloy-render'); el.result = result; + const nextRendererOptions = options.nextRendererOptions ?? {}; + for (const [key, val] of Object.entries(nextRendererOptions)) { + el[key] = val; + } return el; } else { // eslint-disable-next-line no-console diff --git a/packages/malloy-render/src/html/renderer_types.ts b/packages/malloy-render/src/html/renderer_types.ts index db3d832e6..a3cd800cf 100644 --- a/packages/malloy-render/src/html/renderer_types.ts +++ b/packages/malloy-render/src/html/renderer_types.ts @@ -21,6 +21,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +import {MalloyRenderProps} from '../component/render'; import {DataStyles} from './data_styles'; export interface RendererOptions { @@ -29,6 +30,7 @@ export interface RendererOptions { onDrill?: DrillFunction; titleCase?: boolean; queryTimezone?: string; + nextRendererOptions?: Partial; } export type DrillFunction = ( diff --git a/packages/malloy-syntax-highlight/package.json b/packages/malloy-syntax-highlight/package.json index 5b0f9d5b9..e1a4175b0 100644 --- a/packages/malloy-syntax-highlight/package.json +++ b/packages/malloy-syntax-highlight/package.json @@ -1,6 +1,6 @@ { "name": "@malloydata/syntax-highlight", - "version": "0.0.218", + "version": "0.0.222", "description": "A package to simplify the process of developing, testing, and syncnig Malloy syntax highlighting grammars", "files": [ "grammars/**/*.tmGrammar.json", diff --git a/packages/malloy/package.json b/packages/malloy/package.json index 6d968bb66..a44578677 100644 --- a/packages/malloy/package.json +++ b/packages/malloy/package.json @@ -1,6 +1,6 @@ { "name": "@malloydata/malloy", - "version": "0.0.218", + "version": "0.0.222", "license": "MIT", "exports": { ".": "./dist/index.js", diff --git a/packages/malloy/src/connection/base_connection.ts b/packages/malloy/src/connection/base_connection.ts index a791cf645..590fab3e1 100644 --- a/packages/malloy/src/connection/base_connection.ts +++ b/packages/malloy/src/connection/base_connection.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import { diff --git a/packages/malloy/src/dialect/dialect.ts b/packages/malloy/src/dialect/dialect.ts index f70d58f99..496d6b6df 100644 --- a/packages/malloy/src/dialect/dialect.ts +++ b/packages/malloy/src/dialect/dialect.ts @@ -37,6 +37,7 @@ import { LeafAtomicTypeDef, isRawCast, isLeafAtomic, + OrderBy, } from '../model/malloy_types'; import {DialectFunctionOverloadDef} from './functions'; @@ -73,6 +74,13 @@ export interface QueryInfo { systemTimezone?: string; } +export type FieldReferenceType = + | 'table' + | 'nest source' + | 'array[scalar]' + | 'array[record]' + | 'record'; + const allUnits = [ 'microsecond', 'millisecond', @@ -165,6 +173,11 @@ export abstract class Dialect { nativeBoolean = true; + // Can have arrays of arrays + nestedArrays = true; + // An array or record will reveal type of contents on schema read + compoundObjectInSchema = true; + abstract getDialectFunctionOverrides(): { [name: string]: DialectFunctionOverloadDef[]; }; @@ -220,11 +233,10 @@ export abstract class Dialect { abstract sqlGenerateUUID(): string; abstract sqlFieldReference( - alias: string, - fieldName: string, - fieldType: string, - isNested: boolean, - isArray: boolean + parentAlias: string, + parentType: FieldReferenceType, + childName: string, + childType: string ): string; abstract sqlUnnestPipelineHead( @@ -237,7 +249,8 @@ export abstract class Dialect { abstract sqlCreateFunctionCombineLastStage( lastStageName: string, - fieldList: DialectFieldList + fieldList: DialectFieldList, + orderBy: OrderBy[] | undefined ): string; abstract sqlCreateTableAsSelect(tableName: string, sql: string): string; @@ -281,21 +294,8 @@ export abstract class Dialect { abstract sqlLiteralRegexp(literal: string): string; abstract sqlRegexpMatch(df: RegexMatchExpr): string; - // abstract sqlLiteralRecord(lit: RecordLiteralNode): string; - // abstract sqlLiteralArray(lit: ArrayLiteralNode): string; - // SHOULD BE ABSTRACT BUT A PLACEHOLDER FOR NOW - sqlLiteralArray(lit: ArrayLiteralNode): string { - const array = lit.kids.values.map(val => val.sql); - return '[' + array.join(',') + ']'; - } - - sqlLiteralRecord(lit: RecordLiteralNode): string { - const pairs = Object.entries(lit.kids).map( - ([propName, propVal]) => - `${this.sqlMaybeQuoteIdentifier(propName)}:${propVal.sql}` - ); - return '{' + pairs.join(',') + '}'; - } + abstract sqlLiteralArray(lit: ArrayLiteralNode): string; + abstract sqlLiteralRecord(lit: RecordLiteralNode): string; /** * The dialect has a chance to over-ride how expressions are translated. If diff --git a/packages/malloy/src/dialect/duckdb/dialect_functions.ts b/packages/malloy/src/dialect/duckdb/dialect_functions.ts index 9f22b9d51..ae1be02fb 100644 --- a/packages/malloy/src/dialect/duckdb/dialect_functions.ts +++ b/packages/malloy/src/dialect/duckdb/dialect_functions.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import { diff --git a/packages/malloy/src/dialect/duckdb/duckdb.ts b/packages/malloy/src/dialect/duckdb/duckdb.ts index cd9a722ed..364192c4f 100644 --- a/packages/malloy/src/dialect/duckdb/duckdb.ts +++ b/packages/malloy/src/dialect/duckdb/duckdb.ts @@ -32,6 +32,9 @@ import { MeasureTimeExpr, LeafAtomicTypeDef, TD, + RecordLiteralNode, + OrderBy, + mkFieldDef, } from '../../model/malloy_types'; import {indent} from '../../model/utils'; import { @@ -39,10 +42,11 @@ import { expandOverrideMap, expandBlueprintMap, } from '../functions'; -import {DialectFieldList, inDays} from '../dialect'; +import {DialectFieldList, FieldReferenceType, inDays} from '../dialect'; import {PostgresBase} from '../pg_impl'; import {DUCKDB_DIALECT_FUNCTIONS} from './dialect_functions'; import {DUCKDB_MALLOY_STANDARD_OVERLOADS} from './function_overrides'; +import {TinyParseError, TinyParser, TinyToken} from '../tiny_parser'; // need to refactor runSQL to take a SQLBlock instead of just a sql string. const hackSplitComment = '-- hack: split on this'; @@ -212,19 +216,18 @@ export class DuckDBDialect extends PostgresBase { } sqlFieldReference( - alias: string, - fieldName: string, - _fieldType: string, - _isNested: boolean, - isArray: boolean + parentAlias: string, + parentType: FieldReferenceType, + childName: string, + _childType: string ): string { // LTNOTE: hack, in duckdb we can't have structs as tables so we kind of simulate it. - if (!this.unnestWithNumbers && fieldName === '__row_id') { - return `${alias}_outer.__row_id`; - } else if (isArray) { - return alias; + if (!this.unnestWithNumbers && childName === '__row_id') { + return `${parentAlias}_outer.__row_id`; + } else if (parentType === 'array[scalar]') { + return parentAlias; } else { - return `${alias}.${this.sqlMaybeQuoteIdentifier(fieldName)}`; + return `${parentAlias}.${this.sqlMaybeQuoteIdentifier(childName)}`; } } @@ -247,11 +250,28 @@ export class DuckDBDialect extends PostgresBase { sqlCreateFunctionCombineLastStage( lastStageName: string, - dialectFieldList: DialectFieldList + dialectFieldList: DialectFieldList, + orderBy: OrderBy[] | undefined ): string { + let o = ''; + if (orderBy) { + const clauses: string[] = []; + for (const c of orderBy) { + if (typeof c.field === 'string') { + clauses.push(`${c.field} ${c.dir || 'asc'}`); + } else { + clauses.push( + `${dialectFieldList[c.field].sqlOutputName} ${c.dir || 'asc'}` + ); + } + } + if (clauses.length > 0) { + o = ` ORDER BY ${clauses.join(', ')}`; + } + } return `SELECT LIST(STRUCT_PACK(${dialectFieldList .map(d => this.sqlMaybeQuoteIdentifier(d.sqlOutputName)) - .join(',')})) FROM ${lastStageName}\n`; + .join(',')})${o}) FROM ${lastStageName}\n`; } sqlSelectAliasAsStruct( @@ -262,17 +282,6 @@ export class DuckDBDialect extends PostgresBase { .map(d => `${alias}.${d.sqlOutputName}`) .join(', ')})`; } - // TODO - // sqlMaybeQuoteIdentifier(identifier: string): string { - // return keywords.indexOf(identifier.toUpperCase()) > 0 || - // identifier.match(/[a-zA-Z][a-zA-Z0-9]*/) === null || true - // ? '"' + identifier + '"' - // : identifier; - // } - - sqlMaybeQuoteIdentifier(identifier: string): string { - return '"' + identifier + '"'; - } // The simple way to do this is to add a comment on the table // with the expiration time. https://www.postgresql.org/docs/current/sql-comment.html @@ -365,6 +374,19 @@ export class DuckDBDialect extends PostgresBase { return malloyType.type; } + parseDuckDBType(sqlType: string): AtomicTypeDef { + const parser = new DuckDBTypeParser(sqlType); + try { + return parser.typeDef(); + } catch (e) { + if (e instanceof TinyParseError) { + return {type: 'sql native', rawType: sqlType}; + } else { + throw e; + } + } + } + sqlTypeToMalloyType(sqlType: string): LeafAtomicTypeDef { // Remove decimal precision const ddbType = sqlType.replace(/^DECIMAL\(\d+,\d+\)/g, 'DECIMAL'); @@ -428,4 +450,126 @@ export class DuckDBDialect extends PostgresBase { } return `DATE_SUB('${df.units}', ${lVal}, ${rVal})`; } + + sqlLiteralRecord(lit: RecordLiteralNode): string { + const pairs = Object.entries(lit.kids).map( + ([propName, propVal]) => + `${this.sqlMaybeQuoteIdentifier(propName)}:${propVal.sql}` + ); + return '{' + pairs.join(',') + '}'; + } +} + +class DuckDBTypeParser extends TinyParser { + constructor(input: string) { + super(input, { + /* whitespace */ space: /^\s+/, + /* single quoted string */ qsingle: /^'([^']|'')*'/, + /* double quoted string */ qdouble: /^"([^"]|"")*"/, + /* (n) size */ size: /^\(\d+\)/, + /* (n1,n2) precision */ precision: /^\(\d+,\d+\)/, + /* T[] -> array of T */ arrayOf: /^\[]/, + /* other punctuation */ char: /^[,:[\]()-]/, + /* unquoted word */ id: /^\w+/, + }); + } + + unquoteName(token: TinyToken): string { + if (token.type === 'qsingle') { + return token.text.replace("''", ''); + } else if (token.type === 'qdouble') { + return token.text.replace('""', ''); + } + return token.text; + } + + sqlID(token: TinyToken) { + return token.text.toUpperCase(); + } + + typeDef(): AtomicTypeDef { + const unknownStart = this.parseCursor; + const wantID = this.next('id'); + const id = this.sqlID(wantID); + let baseType: AtomicTypeDef; + if (id === 'VARCHAR') { + if (this.peek().type === 'size') { + this.next(); + } + } + if ( + (id === 'DECIMAL' || id === 'NUMERIC') && + this.peek().type === 'precision' + ) { + this.next(); + baseType = {type: 'number', numberType: 'float'}; + } else if (id === 'TIMESTAMP') { + if (this.peek().text === 'WITH') { + this.nextText('WITH', 'TIME', 'ZONE'); + } + baseType = {type: 'timestamp'}; + } else if (duckDBToMalloyTypes[id]) { + baseType = duckDBToMalloyTypes[id]; + } else if (id === 'STRUCT') { + this.next('('); + baseType = {type: 'record', fields: []}; + for (;;) { + const fieldName = this.next(); + if ( + fieldName.type === 'qsingle' || + fieldName.type === 'qdouble' || + fieldName.type === 'id' + ) { + const fieldType = this.typeDef(); + baseType.fields.push( + mkFieldDef(fieldType, this.unquoteName(fieldName), 'duckdb') + ); + } else { + if (fieldName.type !== ')') { + throw this.parseError('Expected identifier or ) to end STRUCT'); + } + break; + } + if (this.peek().type === ',') { + this.next(); + } + } + } else { + if (wantID.type === 'id') { + for (;;) { + const next = this.peek(); + // Might be WEIRDTYP(a,b)[] ... stop at the [] + if (next.type === 'arrayOf' || next.type === 'eof') { + break; + } + this.next(); + } + baseType = { + type: 'sql native', + rawType: this.input.slice( + unknownStart, + this.parseCursor - unknownStart + 1 + ), + }; + } else { + throw this.parseError('Could not understand type'); + } + } + while (this.peek().type === 'arrayOf') { + this.next(); + if (baseType.type === 'record') { + baseType = { + type: 'array', + elementTypeDef: {type: 'record_element'}, + fields: baseType.fields, + }; + } else { + baseType = { + type: 'array', + elementTypeDef: baseType, + }; + } + } + return baseType; + } } diff --git a/packages/malloy/src/dialect/duckdb/function_overrides.ts b/packages/malloy/src/dialect/duckdb/function_overrides.ts index 1e132b225..5f039dead 100644 --- a/packages/malloy/src/dialect/duckdb/function_overrides.ts +++ b/packages/malloy/src/dialect/duckdb/function_overrides.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {MalloyStandardFunctionImplementations as OverrideMap} from '../functions/malloy_standard_functions'; diff --git a/packages/malloy/src/dialect/functions/malloy_standard_functions.ts b/packages/malloy/src/dialect/functions/malloy_standard_functions.ts index d34829769..5f601f90e 100644 --- a/packages/malloy/src/dialect/functions/malloy_standard_functions.ts +++ b/packages/malloy/src/dialect/functions/malloy_standard_functions.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {LeafExpressionType} from '../../model/malloy_types'; diff --git a/packages/malloy/src/dialect/index.ts b/packages/malloy/src/dialect/index.ts index 140716594..f87da7767 100644 --- a/packages/malloy/src/dialect/index.ts +++ b/packages/malloy/src/dialect/index.ts @@ -42,7 +42,7 @@ export { sql, } from './functions/util'; export {Dialect, qtz} from './dialect'; -export type {DialectFieldList, QueryInfo} from './dialect'; +export type {DialectFieldList, QueryInfo, FieldReferenceType} from './dialect'; export {StandardSQLDialect} from './standardsql'; export {PostgresDialect} from './postgres'; export {DuckDBDialect} from './duckdb'; @@ -52,3 +52,5 @@ export {MySQLDialect} from './mysql'; export {getDialect, registerDialect} from './dialect_map'; export {getMalloyStandardFunctions} from './functions'; export type {MalloyStandardFunctionImplementations} from './functions'; +export type {TinyToken} from './tiny_parser'; +export {TinyParser} from './tiny_parser'; diff --git a/packages/malloy/src/dialect/mysql/dialect_functions.ts b/packages/malloy/src/dialect/mysql/dialect_functions.ts index 21274ad6c..ce5a9447f 100644 --- a/packages/malloy/src/dialect/mysql/dialect_functions.ts +++ b/packages/malloy/src/dialect/mysql/dialect_functions.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import { diff --git a/packages/malloy/src/dialect/mysql/function_overrides.ts b/packages/malloy/src/dialect/mysql/function_overrides.ts index af13242aa..662c3a2eb 100644 --- a/packages/malloy/src/dialect/mysql/function_overrides.ts +++ b/packages/malloy/src/dialect/mysql/function_overrides.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {MalloyStandardFunctionImplementations as OverrideMap} from '../functions/malloy_standard_functions'; diff --git a/packages/malloy/src/dialect/mysql/index.ts b/packages/malloy/src/dialect/mysql/index.ts index a8efa1de2..b4256497c 100644 --- a/packages/malloy/src/dialect/mysql/index.ts +++ b/packages/malloy/src/dialect/mysql/index.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ export * from './mysql'; diff --git a/packages/malloy/src/dialect/mysql/mysql.ts b/packages/malloy/src/dialect/mysql/mysql.ts index 133fb7575..de2e5b646 100644 --- a/packages/malloy/src/dialect/mysql/mysql.ts +++ b/packages/malloy/src/dialect/mysql/mysql.ts @@ -43,9 +43,17 @@ import { LeafAtomicTypeDef, TD, AtomicTypeDef, + ArrayLiteralNode, + RecordLiteralNode, } from '../../model/malloy_types'; import {indent} from '../../model/utils'; -import {Dialect, DialectFieldList, qtz, QueryInfo} from '../dialect'; +import { + Dialect, + DialectFieldList, + FieldReferenceType, + qtz, + QueryInfo, +} from '../dialect'; import { DialectFunctionOverloadDef, expandBlueprintMap, @@ -119,6 +127,7 @@ export class MySQLDialect extends Dialect { readsNestedData = false; supportsComplexFilteredSources = false; supportsArraysInData = false; + compoundObjectInSchema = false; malloyTypeToSQLType(malloyType: AtomicTypeDef): string { switch (malloyType.type) { @@ -230,34 +239,46 @@ export class MySQLDialect extends Dialect { return fields.join(',\n'); } - jsonTable(source: string, fieldList: DialectFieldList): string { - return `JSON_TABLE(${source}, '$[*]' + jsonTable( + source: string, + fieldList: DialectFieldList, + isSingleton: boolean + ): string { + let fields = this.unnestColumns(fieldList); + if (isSingleton) { + // LTNOTE: we need the type of array here. + fields = "`value` JSON PATH '$'"; + } + return `JSON_TABLE(CAST(${source} AS JSON), '$[*]' COLUMNS ( __row_id FOR ORDINALITY, - ${this.unnestColumns(fieldList)} + ${fields} ) )`; } - // LTNOTE: We'll make this work with Arrays once MToy's changes land. sqlUnnestAlias( source: string, alias: string, fieldList: DialectFieldList, _needDistinctKey: boolean, - _isArray: boolean, + isArray: boolean, _isInNestedPipeline: boolean ): string { return ` - LEFT JOIN ${this.jsonTable(source, fieldList)} as ${alias} ON 1=1`; + LEFT JOIN ${this.jsonTable( + source, + fieldList, + isArray + )} as ${alias} ON 1=1`; } sqlUnnestPipelineHead( - _isSingleton: boolean, + isSingleton: boolean, sourceSQLExpression: string, fieldList: DialectFieldList ): string { - return this.jsonTable(sourceSQLExpression, fieldList); + return this.jsonTable(sourceSQLExpression, fieldList, isSingleton); } sqlSumDistinctHashedKey(_sqlDistinctKey: string): string { @@ -285,30 +306,28 @@ export class MySQLDialect extends Dialect { } sqlFieldReference( - alias: string, - fieldName: string, - fieldType: string, - isNested: boolean, - _isArray: boolean + parentAlias: string, + parentType: FieldReferenceType, + childName: string, + childType: string ): string { - let ret = `${alias}.\`${fieldName}\``; - if (isNested) { - switch (fieldType) { + if (parentType === 'array[scalar]' || parentType === 'record') { + let ret = `JSON_UNQUOTE(JSON_EXTRACT(${parentAlias},'$.${childName}'))`; + if (parentType === 'array[scalar]') { + ret = `JSON_UNQUOTE(${parentAlias}.\`value\`)`; + } + switch (childType) { case 'string': - ret = `CONCAT(${ret}, '')`; - break; - // TODO: Fix this. + return `CONCAT(${ret}, '')`; case 'number': - ret = `CAST(${ret} as double)`; - break; - case 'struct': - ret = `CAST(${ret} as JSON)`; - break; + return `CAST(${ret} as double)`; + case 'record': + case 'array': + return `CAST(${ret} as JSON)`; } - return ret; - } else { - return `${alias}.\`${fieldName}\``; } + const child = this.sqlMaybeQuoteIdentifier(childName); + return `${parentAlias}.${child}`; } sqlCreateFunction(id: string, funcText: string): string { @@ -331,7 +350,7 @@ export class MySQLDialect extends Dialect { } sqlMaybeQuoteIdentifier(identifier: string): string { - return `\`${identifier}\``; + return '`' + identifier.replace(/`/g, '``') + '`'; } // TODO: Check what this is. @@ -543,4 +562,17 @@ export class MySQLDialect extends Dialect { // Parentheses, Commas: NUMERIC(5, 2) return sqlType.match(/^[A-Za-z\s(),0-9]*$/) !== null; } + + sqlLiteralArray(lit: ArrayLiteralNode): string { + const array = lit.kids.values.map(val => val.sql); + return `JSON_ARRAY(${array.join(',')})`; + } + + sqlLiteralRecord(lit: RecordLiteralNode): string { + const pairs = Object.entries(lit.kids).map( + ([propName, propVal]) => + `${this.sqlLiteralString(propName)},${propVal.sql}` + ); + return `JSON_OBJECT(${pairs.join(', ')})`; + } } diff --git a/packages/malloy/src/dialect/pg_impl.ts b/packages/malloy/src/dialect/pg_impl.ts index 69111981b..09d3a7a67 100644 --- a/packages/malloy/src/dialect/pg_impl.ts +++ b/packages/malloy/src/dialect/pg_impl.ts @@ -2,10 +2,12 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import { + ArrayLiteralNode, + RecordLiteralNode, RegexMatchExpr, TD, TimeExtractExpr, @@ -95,4 +97,17 @@ export abstract class PostgresBase extends Dialect { } return `TIMESTAMP '${lt.literal}'`; } + + sqlLiteralRecord(_lit: RecordLiteralNode): string { + throw new Error('Cannot create a record literal for postgres base dialect'); + } + + sqlLiteralArray(lit: ArrayLiteralNode): string { + const array = lit.kids.values.map(val => val.sql); + return 'ARRAY[' + array.join(',') + ']'; + } + + sqlMaybeQuoteIdentifier(identifier: string): string { + return '"' + identifier.replace(/"/g, '""') + '"'; + } } diff --git a/packages/malloy/src/dialect/postgres/dialect_functions.ts b/packages/malloy/src/dialect/postgres/dialect_functions.ts index 4aeac2ddb..8fab64e97 100644 --- a/packages/malloy/src/dialect/postgres/dialect_functions.ts +++ b/packages/malloy/src/dialect/postgres/dialect_functions.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import { diff --git a/packages/malloy/src/dialect/postgres/function_overrides.ts b/packages/malloy/src/dialect/postgres/function_overrides.ts index f6131ed70..9dc67a32b 100644 --- a/packages/malloy/src/dialect/postgres/function_overrides.ts +++ b/packages/malloy/src/dialect/postgres/function_overrides.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {MalloyStandardFunctionImplementations as OverrideMap} from '../functions/malloy_standard_functions'; diff --git a/packages/malloy/src/dialect/postgres/postgres.ts b/packages/malloy/src/dialect/postgres/postgres.ts index 67877cf72..77eff55cb 100644 --- a/packages/malloy/src/dialect/postgres/postgres.ts +++ b/packages/malloy/src/dialect/postgres/postgres.ts @@ -32,13 +32,15 @@ import { TypecastExpr, MeasureTimeExpr, LeafAtomicTypeDef, + RecordLiteralNode, + ArrayLiteralNode, } from '../../model/malloy_types'; import { DialectFunctionOverloadDef, expandOverrideMap, expandBlueprintMap, } from '../functions'; -import {DialectFieldList, QueryInfo} from '../dialect'; +import {DialectFieldList, FieldReferenceType, QueryInfo} from '../dialect'; import {PostgresBase} from '../pg_impl'; import {POSTGRES_DIALECT_FUNCTIONS} from './dialect_functions'; import {POSTGRES_MALLOY_STANDARD_OVERLOADS} from './function_overrides'; @@ -108,6 +110,7 @@ export class PostgresDialect extends PostgresBase { experimental = false; readsNestedData = false; supportsComplexFilteredSources = false; + compoundObjectInSchema = false; quoteTablePath(tablePath: string): string { return tablePath @@ -219,9 +222,9 @@ export class PostgresDialect extends PostgresBase { ): string { if (isArray) { if (needDistinctKey) { - return `LEFT JOIN UNNEST(ARRAY((SELECT jsonb_build_object('__row_id', row_number() over (), 'value', v) FROM UNNEST(${source}) as v))) as ${alias} ON true`; + return `LEFT JOIN UNNEST(ARRAY((SELECT jsonb_build_object('__row_id', row_number() over (), 'value', v) FROM JSONB_ARRAY_ELEMENTS(TO_JSONB(${source})) as v))) as ${alias} ON true`; } else { - return `LEFT JOIN UNNEST(ARRAY((SELECT jsonb_build_object('value', v) FROM UNNEST(${source}) as v))) as ${alias} ON true`; + return `LEFT JOIN UNNEST(ARRAY((SELECT jsonb_build_object('value', v) FROM JSONB_ARRAY_ELEMENTS(TO_JSONB(${source})) as v))) as ${alias} ON true`; } } else if (needDistinctKey) { // return `UNNEST(ARRAY(( SELECT AS STRUCT GENERATE_UUID() as __distinct_key, * FROM UNNEST(${source})))) as ${alias}`; @@ -241,27 +244,33 @@ export class PostgresDialect extends PostgresBase { } sqlFieldReference( - alias: string, - fieldName: string, - fieldType: string, - isNested: boolean, - _isArray: boolean + parentAlias: string, + parentType: FieldReferenceType, + childName: string, + childType: string ): string { - let ret = `(${alias}->>'${fieldName}')`; - if (isNested) { - switch (fieldType) { + if (childName === '__row_id') { + return `(${parentAlias}->>'__row_id')`; + } + if (parentType !== 'table') { + let ret = `JSONB_EXTRACT_PATH_TEXT(${parentAlias},'${childName}')`; + switch (childType) { case 'string': break; case 'number': ret = `${ret}::double precision`; break; case 'struct': - ret = `${ret}::jsonb`; + case 'array': + case 'record': + case 'array[record]': + ret = `JSONB_EXTRACT_PATH(${parentAlias},'${childName}')`; break; } return ret; } else { - return `${alias}."${fieldName}"`; + const child = this.sqlMaybeQuoteIdentifier(childName); + return `${parentAlias}.${child}`; } } @@ -293,10 +302,6 @@ export class PostgresDialect extends PostgresBase { sqlSelectAliasAsStruct(alias: string): string { return `ROW(${alias})`; } - // TODO - sqlMaybeQuoteIdentifier(identifier: string): string { - return `"${identifier}"`; - } // The simple way to do this is to add a comment on the table // with the expiration time. https://www.postgresql.org/docs/current/sql-comment.html @@ -446,4 +451,17 @@ export class PostgresDialect extends PostgresBase { // Square Brackets: INT64[] return sqlType.match(/^[A-Za-z\s(),[\]0-9]*$/) !== null; } + + sqlLiteralRecord(lit: RecordLiteralNode): string { + const props: string[] = []; + for (const [kName, kVal] of Object.entries(lit.kids)) { + props.push(`'${kName}',${kVal.sql}`); + } + return `JSONB_BUILD_OBJECT(${props.join(', ')})`; + } + + sqlLiteralArray(lit: ArrayLiteralNode): string { + const array = lit.kids.values.map(val => val.sql); + return 'JSONB_BUILD_ARRAY(' + array.join(',') + ')'; + } } diff --git a/packages/malloy/src/dialect/snowflake/dialect_functions.ts b/packages/malloy/src/dialect/snowflake/dialect_functions.ts index f7d6aef54..70507c948 100644 --- a/packages/malloy/src/dialect/snowflake/dialect_functions.ts +++ b/packages/malloy/src/dialect/snowflake/dialect_functions.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {AggregateOrderByNode} from '../../model'; diff --git a/packages/malloy/src/dialect/snowflake/function_overrides.ts b/packages/malloy/src/dialect/snowflake/function_overrides.ts index b42add056..1c742ef9c 100644 --- a/packages/malloy/src/dialect/snowflake/function_overrides.ts +++ b/packages/malloy/src/dialect/snowflake/function_overrides.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {MalloyStandardFunctionImplementations as OverrideMap} from '../functions/malloy_standard_functions'; diff --git a/packages/malloy/src/dialect/snowflake/snowflake.ts b/packages/malloy/src/dialect/snowflake/snowflake.ts index 1adc023fd..c0328c36e 100644 --- a/packages/malloy/src/dialect/snowflake/snowflake.ts +++ b/packages/malloy/src/dialect/snowflake/snowflake.ts @@ -37,13 +37,24 @@ import { RegexMatchExpr, LeafAtomicTypeDef, TD, + ArrayLiteralNode, + RecordLiteralNode, + isAtomic, + isRepeatedRecord, + isScalarArray, } from '../../model/malloy_types'; import { DialectFunctionOverloadDef, expandOverrideMap, expandBlueprintMap, } from '../functions'; -import {Dialect, DialectFieldList, QueryInfo, qtz} from '../dialect'; +import { + Dialect, + DialectFieldList, + FieldReferenceType, + QueryInfo, + qtz, +} from '../dialect'; import {SNOWFLAKE_DIALECT_FUNCTIONS} from './dialect_functions'; import {SNOWFLAKE_MALLOY_STANDARD_OVERLOADS} from './function_overrides'; @@ -191,11 +202,12 @@ export class SnowflakeDialect extends Dialect { isArray: boolean, _isInNestedPipeline: boolean ): string { + const as = this.sqlMaybeQuoteIdentifier(alias); if (isArray) { - return `,LATERAL FLATTEN(INPUT => ${source}) AS ${alias}_1, LATERAL (SELECT ${alias}_1.INDEX, object_construct('value', ${alias}_1.value) as value ) as ${alias}`; + return `,LATERAL FLATTEN(INPUT => ${source}) AS ${alias}_1, LATERAL (SELECT ${alias}_1.INDEX, object_construct('value', ${alias}_1.value) as value ) as ${as}`; } else { // have to have a non empty row or it treats it like an inner join :barf-emoji: - return `LEFT JOIN LATERAL FLATTEN(INPUT => ifnull(${source},[1])) AS ${alias}`; + return `LEFT JOIN LATERAL FLATTEN(INPUT => ifnull(${source},[1])) AS ${as}`; } } @@ -242,25 +254,39 @@ export class SnowflakeDialect extends Dialect { } sqlFieldReference( - alias: string, - fieldName: string, - fieldType: string, - isNested: boolean, - _isArray: boolean + parentAlias: string, + parentType: FieldReferenceType, + childName: string, + childType: string ): string { - if (fieldName === '__row_id') { - return `${alias}.INDEX::varchar`; - } else if (!isNested) { - return `${alias}."${fieldName}"`; - } else { - let snowflakeType = fieldType; - if (fieldType === 'string') { - snowflakeType = 'varchar'; - } else if (fieldType === 'struct') { - snowflakeType = 'variant'; + const sqlName = this.sqlMaybeQuoteIdentifier(childName); + if (childName === '__row_id') { + return `"${parentAlias}".INDEX::varchar`; + } else if ( + parentType === 'array[scalar]' || + parentType === 'array[record]' + ) { + const arrayRef = `"${parentAlias}".value:${sqlName}`; + switch (childType) { + case 'record': + case 'array': + childType = 'VARIANT'; + break; + case 'string': + childType = 'VARCHAR'; + break; + case 'number': + childType = 'DOUBLE'; + break; + case 'struct': + throw new Error('NOT STRUCT PLEASE'); + // boolean and timestamp and date are all ok } - return `${alias}.value:"${fieldName}"::${snowflakeType}`; + return `${arrayRef}::${childType}`; + } else if (parentType === 'record') { + return `${parentAlias}:${sqlName}`; } + return `${parentAlias}.${sqlName}`; } sqlUnnestPipelineHead( @@ -286,8 +312,9 @@ export class SnowflakeDialect extends Dialect { sqlSelectAliasAsStruct(alias: string): string { return `OBJECT_CONSTRUCT_KEEP_NULL(${alias}.*)`; } + sqlMaybeQuoteIdentifier(identifier: string): string { - return `"${identifier}"`; + return '"' + identifier.replace(/"/g, '""') + '"'; } sqlCreateTableAsSelect(tableName: string, sql: string): string { @@ -453,12 +480,31 @@ ${indent(sql)} } malloyTypeToSQLType(malloyType: AtomicTypeDef): string { - if (malloyType.type === 'number') { + if (malloyType.type === 'string') { + return 'VARCHAR'; + } else if (malloyType.type === 'number') { if (malloyType.numberType === 'integer') { - return 'integer'; + return 'INTEGER'; } else { - return 'double'; + return 'DOUBLE'; } + } else if (malloyType.type === 'record' || isRepeatedRecord(malloyType)) { + const sqlFields = malloyType.fields.reduce((ret, f) => { + if (isAtomic(f)) { + const name = f.as ?? f.name; + const oneSchema = `${this.sqlMaybeQuoteIdentifier( + name + )} ${this.malloyTypeToSQLType(f)}`; + ret.push(oneSchema); + } + return ret; + }, [] as string[]); + const recordScehma = `OBJECT(${sqlFields.join(',')})`; + return malloyType.type === 'record' + ? recordScehma + : `ARRAY(${recordScehma})`; + } else if (isScalarArray(malloyType)) { + return `ARRAY(${this.malloyTypeToSQLType(malloyType.elementTypeDef)})`; } return malloyType.type; } @@ -490,4 +536,24 @@ ${indent(sql)} // Square Brackets: INT64[] return sqlType.match(/^[A-Za-z\s(),[\]0-9]*$/) !== null; } + + sqlLiteralRecord(lit: RecordLiteralNode): string { + const rowVals: string[] = []; + for (const f of lit.typeDef.fields) { + const name = f.as ?? f.name; + const propName = `'${name}'`; + const propVal = lit.kids[name].sql ?? 'internal-error-record-literal'; + rowVals.push(`${propName},${propVal}`); + } + return `OBJECT_CONSTRUCT_KEEP_NULL(${rowVals.join(',')})`; + } + + sqlLiteralArray(lit: ArrayLiteralNode): string { + const array = lit.kids.values.map(val => val.sql); + const arraySchema = `[${array.join(',')}]`; + return arraySchema; + // return lit.typeDef.elementTypeDef.type === 'record_element' + // ? `${arraySchema}::${this.malloyTypeToSQLType(lit.typeDef)}` + // : arraySchema; + } } diff --git a/packages/malloy/src/dialect/standardsql/dialect_functions.ts b/packages/malloy/src/dialect/standardsql/dialect_functions.ts index ea698529f..1ac983823 100644 --- a/packages/malloy/src/dialect/standardsql/dialect_functions.ts +++ b/packages/malloy/src/dialect/standardsql/dialect_functions.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import { diff --git a/packages/malloy/src/dialect/standardsql/function_overrides.ts b/packages/malloy/src/dialect/standardsql/function_overrides.ts index 89aca5ddf..88ac6066e 100644 --- a/packages/malloy/src/dialect/standardsql/function_overrides.ts +++ b/packages/malloy/src/dialect/standardsql/function_overrides.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {MalloyStandardFunctionImplementations as OverrideMap} from '../functions/malloy_standard_functions'; diff --git a/packages/malloy/src/dialect/standardsql/standardsql.ts b/packages/malloy/src/dialect/standardsql/standardsql.ts index cea118a90..7ea2dcdff 100644 --- a/packages/malloy/src/dialect/standardsql/standardsql.ts +++ b/packages/malloy/src/dialect/standardsql/standardsql.ts @@ -37,6 +37,8 @@ import { MeasureTimeExpr, LeafAtomicTypeDef, TD, + RecordLiteralNode, + ArrayLiteralNode, } from '../../model/malloy_types'; import { DialectFunctionOverloadDef, @@ -123,6 +125,7 @@ export class StandardSQLDialect extends Dialect { supportsNesting = true; cantPartitionWindowFunctionsOnExpressions = true; hasModOperator = false; + nestedArrays = false; // Can't have an array of arrays for some reason quoteTablePath(tablePath: string): string { return `\`${tablePath}\``; @@ -223,13 +226,13 @@ export class StandardSQLDialect extends Dialect { } sqlFieldReference( - alias: string, - fieldName: string, - _fieldType: string, - _isNested: boolean, - _isArray: boolean + parentAlias: string, + _parentType: unknown, + childName: string, + _childType: string ): string { - return `${alias}.${fieldName}`; + const child = this.sqlMaybeQuoteIdentifier(childName); + return `${parentAlias}.${child}`; } sqlUnnestPipelineHead( @@ -269,107 +272,7 @@ ${indent(sql)} return `(SELECT AS STRUCT ${alias}.*)`; } - keywords = ` - ALL - AND - ANY - ARRAY - AS - ASC - ASSERT_ROWS_MODIFIED - AT - BETWEEN - BY - CASE - CAST - COLLATE - CONTAINS - CREATE - CROSS - CUBE - CURRENT - DEFAULT - DEFINE - DESC - DISTINCT - ELSE - END - ENUM - ESCAPE - EXCEPT - EXCLUDE - EXISTS - EXTRACT - FALSE - FETCH - FOLLOWING - FOR - FROM - FULL - GROUP - GROUPING - GROUPS - HASH - HAVING - IF - IGNORE - IN - INNER - INTERSECT - INTERVAL - INTO - IS - JOIN - LATERAL - LEFT - LIKE - LIMIT - LOOKUP - MERGE - NATURAL - NEW - NO - NOT - NULL - NULLS - OF - ON - OR - ORDER - OUTER - OVER - PARTITION - PRECEDING - PROTO - RANGE - RECURSIVE - RESPECT - RIGHT - ROLLUP - ROWS - SELECT - SET - SOME - STRUCT - TABLESAMPLE - THEN - TO - TREAT - TRUE - UNBOUNDED - UNION - UNNEST - USING - WHEN - WHERE - WINDOW - WITH - WITHIN`.split(/\s/); - sqlMaybeQuoteIdentifier(identifier: string): string { - // return this.keywords.indexOf(identifier.toUpperCase()) > 0 - // ? '`' + identifier + '`' - // : identifier; return '`' + identifier + '`'; } @@ -568,4 +471,18 @@ ${indent(sql)} // Angle Brackets: ARRAY return sqlType.match(/^[A-Za-z\s(),<>0-9]*$/) !== null; } + + sqlLiteralRecord(lit: RecordLiteralNode): string { + const ents: string[] = []; + for (const [name, val] of Object.entries(lit.kids)) { + const expr = val.sql || 'internal-error-literal-record'; + ents.push(`${expr} AS ${this.sqlMaybeQuoteIdentifier(name)}`); + } + return `STRUCT(${ents.join(',')})`; + } + + sqlLiteralArray(lit: ArrayLiteralNode): string { + const array = lit.kids.values.map(val => val.sql); + return '[' + array.join(',') + ']'; + } } diff --git a/packages/malloy/src/dialect/tiny_parser.ts b/packages/malloy/src/dialect/tiny_parser.ts new file mode 100644 index 000000000..8650be0be --- /dev/null +++ b/packages/malloy/src/dialect/tiny_parser.ts @@ -0,0 +1,161 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +export interface TinyToken { + type: string; + text: string; +} + +/** + * Simple framework for writing schema parsers. The parsers using this felt + * better than the more ad-hoc code they replaced, and are smaller than + * using a parser generator. + * + * NOTE: All parse errors are exceptions. + */ +export class TinyParseError extends Error {} +export class TinyParser { + private tokens: Generator; + protected parseCursor = 0; + private lookAhead?: TinyToken; + private tokenMap: Record; + + /** + * The token map is tested in order. Return TinyToken + * is {type: tokenMapKey, text: matchingText }, except + * for the special tokenMapKeys: + * * space: skipped and never returned + * * char: matched string return in both .type and .text + * * q*: any token name starting with 'q' is assumed to be + * a quoted string and the text will have the first and + * last characters stripped + */ + constructor( + readonly input: string, + tokenMap?: Record + ) { + this.tokenMap = tokenMap ?? { + space: /^\s+/, + char: /^[,:[\]()-]/, + id: /^\w+/, + qstr: /^"\w+"/, + }; + this.tokens = this.tokenize(input); + } + + parseError(str: string) { + const errText = + `INTERNAL ERROR parsing schema: ${str}\n` + + `${this.input}\n` + + `${' '.repeat(this.parseCursor)}^`; + return new TinyParseError(errText); + } + + peek(): TinyToken { + if (this.lookAhead) { + return this.lookAhead; + } else { + const {value} = this.tokens.next(); + const peekVal = value ?? {type: 'eof', text: ''}; + this.lookAhead = peekVal; + return peekVal; + } + } + + private getNext(): TinyToken { + const next = this.lookAhead ?? this.peek(); + this.lookAhead = undefined; + return next; + } + + /** + * Return next token, if any token types are passed, read and require those + * tokens, then return the last one. + * @param types list of token types + * @returns The last token read + */ + next(...types: string[]): TinyToken { + if (types.length === 0) return this.getNext(); + let next: TinyToken | undefined = undefined; + let expected = types[0]; + for (const typ of types) { + next = this.getNext(); + expected = typ; + if (next.type !== typ) { + next = undefined; + break; + } + } + if (next) return next; + throw this.parseError(`Expected token type '${expected}'`); + } + + nextText(...texts: string[]): TinyToken { + if (texts.length === 0) return this.getNext(); + let next: TinyToken | undefined = undefined; + let expected = texts[0]; + for (const txt of texts) { + next = this.getNext(); + expected = txt; + if (next.text !== txt) { + next = undefined; + break; + } + } + if (next) return next; + throw this.parseError(`Expected '${expected}'`); + } + + skipTo(type: string) { + for (;;) { + const next = this.next(); + if (next.type === 'eof') { + throw this.parseError(`Expected token '${type}`); + } + if (next.type === type) { + return; + } + } + } + + dump(): TinyToken[] { + const p = this.parseCursor; + const parts = [...this.tokenize(this.input)]; + this.parseCursor = p; + return parts; + } + + private *tokenize(src: string): Generator { + const tokenList = this.tokenMap; + while (this.parseCursor < src.length) { + let notFound = true; + for (const tokenType in tokenList) { + const srcAtCursor = src.slice(this.parseCursor); + const foundToken = srcAtCursor.match(tokenList[tokenType]); + if (foundToken) { + notFound = false; + let tokenText = foundToken[0]; + this.parseCursor += tokenText.length; + if (tokenType !== 'space') { + if (tokenType[0] === 'q') { + tokenText = tokenText.slice(1, -1); // strip quotes + } + yield { + type: tokenType === 'char' ? tokenText : tokenType, + text: tokenText, + }; + break; + } + } + } + if (notFound) { + yield {type: 'unexpected token', text: src}; + return; + } + } + } +} diff --git a/packages/malloy/src/dialect/trino/dialect_functions.ts b/packages/malloy/src/dialect/trino/dialect_functions.ts index 13e5797e9..0893892fa 100644 --- a/packages/malloy/src/dialect/trino/dialect_functions.ts +++ b/packages/malloy/src/dialect/trino/dialect_functions.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import { @@ -235,6 +235,48 @@ const percent_rank: DefinitionBlueprint = { impl: {function: 'PERCENT_RANK', needsWindowOrderBy: true}, }; +const url_extract_fragment: DefinitionBlueprint = { + takes: {'url': 'string'}, + returns: 'string', + impl: {function: 'URL_EXTRACT_FRAGMENT'}, +}; + +const url_extract_host: DefinitionBlueprint = { + takes: {'url': 'string'}, + returns: 'string', + impl: {function: 'URL_EXTRACT_HOST'}, +}; + +const url_extract_parameter: DefinitionBlueprint = { + takes: {'url': 'string', 'parameter': 'string'}, + returns: 'string', + impl: {function: 'URL_EXTRACT_PARAMETER'}, +}; + +const url_extract_path: DefinitionBlueprint = { + takes: {'url': 'string'}, + returns: 'string', + impl: {function: 'URL_EXTRACT_PATH'}, +}; + +const url_extract_port: DefinitionBlueprint = { + takes: {'url': 'string'}, + returns: 'number', + impl: {function: 'URL_EXTRACT_PORT'}, +}; + +const url_extract_protocol: DefinitionBlueprint = { + takes: {'url': 'string'}, + returns: 'string', + impl: {function: 'URL_EXTRACT_PROTOCOL'}, +}; + +const url_extract_query: DefinitionBlueprint = { + takes: {'url': 'string'}, + returns: 'string', + impl: {function: 'URL_EXTRACT_QUERY'}, +}; + export const TRINO_DIALECT_FUNCTIONS: DefinitionBlueprintMap = { // aggregate functions approx_percentile, @@ -262,6 +304,13 @@ export const TRINO_DIALECT_FUNCTIONS: DefinitionBlueprintMap = { regexp_like, regexp_replace, to_unixtime, + url_extract_fragment, + url_extract_host, + url_extract_parameter, + url_extract_path, + url_extract_port, + url_extract_protocol, + url_extract_query, // window functions percent_rank, diff --git a/packages/malloy/src/dialect/trino/function_overrides.ts b/packages/malloy/src/dialect/trino/function_overrides.ts index b75718124..c920a86f4 100644 --- a/packages/malloy/src/dialect/trino/function_overrides.ts +++ b/packages/malloy/src/dialect/trino/function_overrides.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {arg, spread, sql} from '../functions/util'; diff --git a/packages/malloy/src/dialect/trino/trino.ts b/packages/malloy/src/dialect/trino/trino.ts index c5bcd69c9..40423cd54 100644 --- a/packages/malloy/src/dialect/trino/trino.ts +++ b/packages/malloy/src/dialect/trino/trino.ts @@ -37,6 +37,8 @@ import { TimeExtractExpr, LeafAtomicTypeDef, TD, + RecordLiteralNode, + isAtomic, } from '../../model/malloy_types'; import { DialectFunctionOverloadDef, @@ -276,17 +278,16 @@ export class TrinoDialect extends PostgresBase { } sqlFieldReference( - alias: string, - fieldName: string, - _fieldType: string, - _isNested: boolean, - _isArray: boolean + parentAlias: string, + _parentType: unknown, + childName: string, + _childType: string ): string { // LTNOTE: hack, in duckdb we can't have structs as tables so we kind of simulate it. - if (fieldName === '__row_id') { - return `__row_id_from_${alias}`; + if (childName === '__row_id') { + return `__row_id_from_${parentAlias}`; } - return `${alias}.${this.sqlMaybeQuoteIdentifier(fieldName)}`; + return `${parentAlias}.${this.sqlMaybeQuoteIdentifier(childName)}`; } sqlUnnestPipelineHead( @@ -431,10 +432,6 @@ ${indent(sql)} WITH WITHIN`.split(/\s/); - sqlMaybeQuoteIdentifier(identifier: string): string { - return '"' + identifier + '"'; - } - sqlAlterTimeExpr(df: TimeDeltaExpr): string { let timeframe = df.units; let n = df.kids.delta.sql; @@ -541,16 +538,33 @@ ${indent(sql)} } malloyTypeToSQLType(malloyType: AtomicTypeDef): string { - if (malloyType.type === 'number') { - if (malloyType.numberType === 'integer') { - return 'BIGINT'; - } else { - return 'DOUBLE'; + switch (malloyType.type) { + case 'number': + return malloyType.numberType === 'integer' ? 'BIGINT' : 'DOUBLE'; + case 'string': + return 'VARCHAR'; + case 'record': { + const typeSpec: string[] = []; + for (const f of malloyType.fields) { + if (isAtomic(f)) { + typeSpec.push(`${f.name} ${this.malloyTypeToSQLType(f)}`); + } + } + return `ROW(${typeSpec.join(',')})`; } - } else if (malloyType.type === 'string') { - return 'VARCHAR'; + case 'sql native': + return malloyType.rawType || 'UNKNOWN-NATIVE'; + case 'array': { + if (malloyType.elementTypeDef.type !== 'record_element') { + return `ARRAY<${this.malloyTypeToSQLType( + malloyType.elementTypeDef + )}>`; + } + return malloyType.type.toUpperCase(); + } + default: + return malloyType.type.toUpperCase(); } - return malloyType.type; } sqlTypeToMalloyType(sqlType: string): LeafAtomicTypeDef { @@ -618,6 +632,20 @@ ${indent(sql)} const extracted = `EXTRACT(${pgUnits} FROM ${extractFrom})`; return from.units === 'day_of_week' ? `mod(${extracted}+1,7)` : extracted; } + + sqlLiteralRecord(lit: RecordLiteralNode): string { + const rowVals: string[] = []; + const rowTypes: string[] = []; + for (const f of lit.typeDef.fields) { + if (isAtomic(f)) { + const name = f.as ?? f.name; + rowVals.push(lit.kids[name].sql ?? 'internal-error-record-literal'); + const elType = this.malloyTypeToSQLType(f); + rowTypes.push(`${name} ${elType}`); + } + } + return `CAST(ROW(${rowVals.join(',')}) AS ROW(${rowTypes.join(',')}))`; + } } export class PrestoDialect extends TrinoDialect { diff --git a/packages/malloy/src/doc/fielddef.md b/packages/malloy/src/doc/fielddef.md index 1b27f0391..49b4c7a46 100644 --- a/packages/malloy/src/doc/fielddef.md +++ b/packages/malloy/src/doc/fielddef.md @@ -72,7 +72,7 @@ interface JoinBase { } ``` -* `isJoined(fd)` which will return true and grant typed access to the `JoinBase` properties of the `FieldDef`, and because all joined fields are structs, also the `StructDef` properties as well. +* `isJoined(def)` which will return true and grant typed access to the `JoinBase` properties of the object, and because all joined fields are structs, also the `StructDef` properties as well. ## Views @@ -91,11 +91,11 @@ are an array of ... * `join_XXX:` always on query joins -## Descriminators +## Discriminators -* `isTemporalField` -- `date` or `timestamp` type -* `isAtomicFieldType` -- Does the data in this field fit in one column of a table -* `isRepeatedRecord(FieldDef)` -- In some databases this is a type, in other this is an array of record -* `isScalarArray(FieldDef|SrtuctDef)` -- Is a ".each" array -* `isAtomic(FieldDef)` -- Like `isAtomicFieldType` for `FieldDef` instead -* `isLeafAtomic(FieldDef | QueryFieldDef)` -- an Atomic field can be stored in a column, a LeafAtomic is one which isn't a join \ No newline at end of file +* `isTemporalType` -- `date` or `timestamp` type +* `isAtomicFieldType` -- Does type string match the type of one of the atomiv types +* `isRepeatedRecord` -- In some databases this is a type, in other this is an array of record +* `isScalarArray` -- Is a ".each" array +* `isAtomic` -- Like `isAtomicFieldType` for `FieldDef` instead +* `isLeafAtomic` -- an Atomic field can be stored in a column, a LeafAtomic is one which isn't a join \ No newline at end of file diff --git a/packages/malloy/src/index.ts b/packages/malloy/src/index.ts index d69b3758c..d05588647 100644 --- a/packages/malloy/src/index.ts +++ b/packages/malloy/src/index.ts @@ -42,6 +42,7 @@ export { literal, spread, Dialect, + TinyParser, } from './dialect'; export type { DialectFieldList, @@ -51,6 +52,7 @@ export type { DefinitionBlueprint, DefinitionBlueprintMap, OverloadedDefinitionBlueprint, + TinyToken, } from './dialect'; // TODO tighten up exports export type { @@ -73,7 +75,9 @@ export type { // Needed for drills in render FilterCondition, SQLSentence, - // Used in Composer Demo + // Used in Composer + Argument, + Parameter, FieldDef, PipeSegment, QueryFieldDef, @@ -108,10 +112,13 @@ export type { ArrayTypeDef, RecordTypeDef, RepeatedRecordTypeDef, + RecordDef, + RepeatedRecordDef, + // Used in array/record tests + RecordLiteralNode, + ArrayLiteralNode, } from './model'; export { - arrayEachFields, - isRepeatedRecord, isSourceDef, // Used in Composer Demo Segment, @@ -121,6 +128,10 @@ export { isSamplingEnable, isSamplingPercent, isSamplingRows, + isRepeatedRecord, + isScalarArray, + mkArrayDef, + mkFieldDef, expressionIsAggregate, expressionIsAnalytic, expressionIsCalculation, diff --git a/packages/malloy/src/lang/ast/expressions/case.ts b/packages/malloy/src/lang/ast/expressions/case.ts index 687b407f3..755ea9c6f 100644 --- a/packages/malloy/src/lang/ast/expressions/case.ts +++ b/packages/malloy/src/lang/ast/expressions/case.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {ExprValue, computedExprValue} from '../types/expr-value'; diff --git a/packages/malloy/src/lang/ast/expressions/expr-aggregate-function.ts b/packages/malloy/src/lang/ast/expressions/expr-aggregate-function.ts index dbd104d47..a76da7147 100644 --- a/packages/malloy/src/lang/ast/expressions/expr-aggregate-function.ts +++ b/packages/malloy/src/lang/ast/expressions/expr-aggregate-function.ts @@ -29,8 +29,8 @@ import { AggregateExpr, Expr, hasExpression, - isJoined, isAtomic, + isJoined, } from '../../../model/malloy_types'; import {exprWalk} from '../../../model/utils'; diff --git a/packages/malloy/src/lang/ast/expressions/expr-array-literal.ts b/packages/malloy/src/lang/ast/expressions/expr-array-literal.ts new file mode 100644 index 000000000..a7e9836a2 --- /dev/null +++ b/packages/malloy/src/lang/ast/expressions/expr-array-literal.ts @@ -0,0 +1,69 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import {ArrayLiteralNode, ArrayTypeDef, Expr} from '../../../model'; +import {ExprValue, computedExprValue} from '../types/expr-value'; +import {ExpressionDef} from '../types/expression-def'; +import {FieldSpace} from '../types/field-space'; +import * as TDU from '../typedesc-utils'; + +export class ArrayLiteral extends ExpressionDef { + elementType = 'array literal'; + constructor(readonly elements: ExpressionDef[]) { + super(); + this.has({elements}); + } + + getExpression(fs: FieldSpace): ExprValue { + const values: Expr[] = []; + const fromValues: ExprValue[] = []; + let firstValue: ExprValue | undefined = undefined; + if (this.elements.length > 0) { + for (const nextElement of this.elements) { + const v = nextElement.getExpression(fs); + fromValues.push(v); + if (v.type === 'error') { + continue; + } + if (firstValue) { + if (!TDU.typeEq(firstValue, v)) { + nextElement.logError( + 'array-values-incompatible', + 'All array elements must be same type' + ); + continue; + } + } else { + firstValue = v; + } + values.push(v.value); + } + } + const elementTypeDef = TDU.atomicDef(firstValue || {type: 'number'}); + const typeDef: ArrayTypeDef = + elementTypeDef.type === 'record' + ? { + type: 'array', + elementTypeDef: {type: 'record_element'}, + fields: elementTypeDef.fields, + } + : { + type: 'array', + elementTypeDef, + }; + const aLit: ArrayLiteralNode = { + node: 'arrayLiteral', + kids: {values}, + typeDef, + }; + return computedExprValue({ + dataType: typeDef, + value: aLit, + from: fromValues, + }); + } +} diff --git a/packages/malloy/src/lang/ast/expressions/expr-record-literal.ts b/packages/malloy/src/lang/ast/expressions/expr-record-literal.ts index 66e9191ec..4c33474c4 100644 --- a/packages/malloy/src/lang/ast/expressions/expr-record-literal.ts +++ b/packages/malloy/src/lang/ast/expressions/expr-record-literal.ts @@ -2,19 +2,15 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ -// import { -// isAtomicFieldType, -// TD, -// RecordLiteralNode, -// TypedExpr, -// } from '../../../model'; -import {ExprValue} from '../types/expr-value'; +import {TD, RecordLiteralNode, mkFieldDef} from '../../../model'; +import {ExprValue, computedExprValue} from '../types/expr-value'; import {ExpressionDef} from '../types/expression-def'; import {FieldSpace} from '../types/field-space'; import {MalloyElement} from '../types/malloy-element'; +import * as TDU from '../typedesc-utils'; export class RecordElement extends MalloyElement { elementType = 'record element'; @@ -34,32 +30,35 @@ export class RecordLiteral extends ExpressionDef { this.has({pairs}); } - getExpression(_fs: FieldSpace): ExprValue { - throw new Error('get expression on record todo'); - // const recLit: RecordLiteralNode = { - // node: 'recordLiteral', - // kids: {}, - // }; - // const dependents: ExprValue[] = []; - // for (const el of this.pairs) { - // const xVal = el.value.getExpression(fs); - // const expr: TypedExpr = {typeDef: {type: 'error'}, ...xVal.value}; - // if (TD.isError(expr.typeDef) && isAtomicFieldType(xVal.dataType)) { - // expr.typeDef = xVal.dataType; - // } - // if (TD.isError(expr.typeDef) && xVal.dataType !== 'error') { - // this.logError( - // 'illegal-record-property-type', - // `Type '${xVal.dataType}' not a legal record value` - // ); - // } - // recLit.kids[el.key] = expr; - // dependents.push(xVal); - // } - // return computedExprValue({ - // dataType: 'record', - // value: recLit, - // from: dependents, - // }); + getExpression(fs: FieldSpace): ExprValue { + const recLit: RecordLiteralNode = { + node: 'recordLiteral', + kids: {}, + typeDef: { + type: 'record', + fields: [], + }, + }; + const dependents: ExprValue[] = []; + for (const el of this.pairs) { + const xVal = el.value.getExpression(fs); + if (TD.isAtomic(xVal)) { + dependents.push(xVal); + recLit.kids[el.key] = xVal.value; + recLit.typeDef.fields.push( + mkFieldDef(TDU.atomicDef(xVal), el.key, fs.dialectName()) + ); + } else { + this.logError( + 'illegal-record-property-type', + `Record property '${el.key} is type '${xVal.type}', which is not a legal property value type` + ); + } + } + return computedExprValue({ + value: recLit, + dataType: recLit.typeDef, + from: dependents, + }); } } diff --git a/packages/malloy/src/lang/ast/expressions/expr-time-extract.ts b/packages/malloy/src/lang/ast/expressions/expr-time-extract.ts index 0ad933293..947c3183c 100644 --- a/packages/malloy/src/lang/ast/expressions/expr-time-extract.ts +++ b/packages/malloy/src/lang/ast/expressions/expr-time-extract.ts @@ -24,7 +24,7 @@ import { ExtractUnit, isExtractUnit, - isTemporalField, + isTemporalType, isTimestampUnit, mkTemporal, TD, @@ -89,13 +89,13 @@ export class ExprTimeExtract extends ExpressionDef { from: [first, last], }); } - if (!isTemporalField(first.type)) { + if (!isTemporalType(first.type)) { return from.first.loggedErrorExpr( 'invalid-type-for-time-extraction', `Can't extract ${extractTo} from '${first.type}'` ); } - if (!isTemporalField(last.type)) { + if (!isTemporalType(last.type)) { return from.last.loggedErrorExpr( 'invalid-type-for-time-extraction', `Cannot extract ${extractTo} from '${last.type}'` @@ -151,7 +151,7 @@ export class ExprTimeExtract extends ExpressionDef { }); } else { const argV = from.getExpression(fs); - if (isTemporalField(argV.type)) { + if (isTemporalType(argV.type)) { return computedExprValue({ dataType: {type: 'number', numberType: 'integer'}, value: { diff --git a/packages/malloy/src/lang/ast/expressions/expr-time.ts b/packages/malloy/src/lang/ast/expressions/expr-time.ts index a965bcc2a..78d305c57 100644 --- a/packages/malloy/src/lang/ast/expressions/expr-time.ts +++ b/packages/malloy/src/lang/ast/expressions/expr-time.ts @@ -25,7 +25,7 @@ import { Expr, TemporalFieldType, TypecastExpr, - isTemporalField, + isTemporalType, } from '../../../model/malloy_types'; import {FieldSpace} from '../types/field-space'; @@ -58,7 +58,7 @@ export class ExprTime extends ExpressionDef { dstType: {type: timeType}, e: expr.value, }; - if (isTemporalField(expr.type)) { + if (isTemporalType(expr.type)) { toTs.srcType = {type: expr.type}; } value = toTs; diff --git a/packages/malloy/src/lang/ast/expressions/time-literal.ts b/packages/malloy/src/lang/ast/expressions/time-literal.ts index d8711ceaa..a064819dc 100644 --- a/packages/malloy/src/lang/ast/expressions/time-literal.ts +++ b/packages/malloy/src/lang/ast/expressions/time-literal.ts @@ -26,7 +26,7 @@ import {DateTime as LuxonDateTime} from 'luxon'; import { TemporalFieldType, TimestampUnit, - isTemporalField, + isTemporalType, TimeLiteralNode, } from '../../../model/malloy_types'; @@ -206,7 +206,7 @@ class GranularLiteral extends TimeLiteral { } // Compiler is unsure about rangeEnd = newEnd for some reason - if (rangeEnd && isTemporalField(testValue.type)) { + if (rangeEnd && isTemporalType(testValue.type)) { const rangeType = testValue.type; const range = new Range( new ExprTime(rangeType, rangeStart.value), diff --git a/packages/malloy/src/lang/ast/expressions/top-by.ts b/packages/malloy/src/lang/ast/expressions/top-by.ts deleted file mode 100644 index 5c8c1f205..000000000 --- a/packages/malloy/src/lang/ast/expressions/top-by.ts +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2023 Google LLC - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files - * (the "Software"), to deal in the Software without restriction, - * including without limitation the rights to use, copy, modify, merge, - * publish, distribute, sublicense, and/or sell copies of the Software, - * and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -import {By, expressionIsAggregate} from '../../../model/malloy_types'; - -import {ExpressionDef} from '../types/expression-def'; -import {FieldSpace} from '../types/field-space'; -import {MalloyElement} from '../types/malloy-element'; - -export class TopBy extends MalloyElement { - elementType = 'topBy'; - constructor(readonly by: string | ExpressionDef) { - super(); - if (by instanceof ExpressionDef) { - this.has({by: by}); - } - } - - getBy(fs: FieldSpace): By { - if (this.by instanceof ExpressionDef) { - const byExpr = this.by.getExpression(fs); - if (!expressionIsAggregate(byExpr.expressionType)) { - this.logError( - 'top-by-non-aggregate', - 'top by expression must be an aggregate' - ); - } - return {by: 'expression', e: byExpr.value}; - } - return {by: 'name', name: this.by}; - } -} diff --git a/packages/malloy/src/lang/ast/field-space/ast-view-field.ts b/packages/malloy/src/lang/ast/field-space/ast-view-field.ts index 729d5cfda..09f20d6b5 100644 --- a/packages/malloy/src/lang/ast/field-space/ast-view-field.ts +++ b/packages/malloy/src/lang/ast/field-space/ast-view-field.ts @@ -39,7 +39,11 @@ export class ASTViewField extends ViewField { return this.view.getFieldDef(fs); } + private turtleDef: TurtleDef | undefined = undefined; fieldDef(): TurtleDef { - return this.view.getFieldDef(this.inSpace); + if (this.turtleDef === undefined) { + this.turtleDef = this.view.getFieldDef(this.inSpace); + } + return this.turtleDef; } } diff --git a/packages/malloy/src/lang/ast/field-space/index-field-space.ts b/packages/malloy/src/lang/ast/field-space/index-field-space.ts index e5c4d5b12..1d0a5e536 100644 --- a/packages/malloy/src/lang/ast/field-space/index-field-space.ts +++ b/packages/malloy/src/lang/ast/field-space/index-field-space.ts @@ -21,12 +21,17 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +import { + emptyCompositeFieldUsage, + emptyNarrowedCompositeFieldResolution, +} from '../../../model/composite_source_utils'; import { IndexSegment, PipeSegment, IndexFieldDef, expressionIsScalar, TD, + CompositeFieldUsage, } from '../../../model/malloy_types'; import { FieldReference, @@ -67,15 +72,21 @@ export class IndexFieldSpace extends QueryOperationSpace { ); return {type: 'index', indexFields: []}; } + let compositeFieldUsage = emptyCompositeFieldUsage(); + let narrowedCompositeFieldResolution = + emptyNarrowedCompositeFieldResolution(); const indexFields: IndexFieldDef[] = []; + const source = this.inputSpace().structDef(); for (const [name, field] of this.entries()) { if (field instanceof SpaceField) { - const wildPath = this.expandedWild[name]; - if (wildPath) { - indexFields.push({type: 'fieldref', path: wildPath}); - continue; - } - if (field instanceof ReferenceField) { + let nextCompositeFieldUsage: CompositeFieldUsage | undefined = + undefined; + let logTo: MalloyElement | undefined = undefined; + const wild = this.expandedWild[name]; + if (wild) { + indexFields.push({type: 'fieldref', path: wild.path}); + nextCompositeFieldUsage = wild.entry.typeDesc().compositeFieldUsage; + } else if (field instanceof ReferenceField) { // attempt to cause a type check const fieldRef = field.fieldRef; const check = fieldRef.getField(this.exprSpace); @@ -83,10 +94,24 @@ export class IndexFieldSpace extends QueryOperationSpace { fieldRef.logError(check.error.code, check.error.message); } else { indexFields.push(fieldRef.refToField); + nextCompositeFieldUsage = + check.found.typeDesc().compositeFieldUsage; + logTo = fieldRef; } } + const next = this.applyNextCompositeFieldUsage( + source, + compositeFieldUsage, + narrowedCompositeFieldResolution, + nextCompositeFieldUsage, + logTo + ); + compositeFieldUsage = next.compositeFieldUsage; + narrowedCompositeFieldResolution = + next.narrowedCompositeFieldResolution; } } + this._compositeFieldUsage = compositeFieldUsage; return {type: 'index', indexFields}; } @@ -135,7 +160,7 @@ export class IndexFieldSpace extends QueryOperationSpace { name, ]); if (this.entry(indexName)) { - const conflict = this.expandedWild[indexName]?.join('.'); + const conflict = this.expandedWild[indexName].path?.join('.'); wild.logError( 'name-conflict-in-wildcard-expansion', `Cannot expand '${name}' in '${ @@ -153,7 +178,10 @@ export class IndexFieldSpace extends QueryOperationSpace { (dialect === undefined || !dialect.ignoreInProject(name)) ) { expandEntries.push({name: indexName, entry}); - this.expandedWild[indexName] = joinPath.concat(name); + this.expandedWild[indexName] = { + path: joinPath.concat(name), + entry, + }; } } } diff --git a/packages/malloy/src/lang/ast/field-space/join-space-field.ts b/packages/malloy/src/lang/ast/field-space/join-space-field.ts index 41c861e10..960d358fc 100644 --- a/packages/malloy/src/lang/ast/field-space/join-space-field.ts +++ b/packages/malloy/src/lang/ast/field-space/join-space-field.ts @@ -28,9 +28,8 @@ import {StructSpaceField} from './static-space'; export class JoinSpaceField extends StructSpaceField { constructor( readonly parameterSpace: ParameterSpace, - readonly join: Join, - parentDialect: string + readonly join: Join ) { - super(join.structDef(parameterSpace), parentDialect); + super(join.structDef(parameterSpace)); } } diff --git a/packages/malloy/src/lang/ast/field-space/project-field-space.ts b/packages/malloy/src/lang/ast/field-space/project-field-space.ts index 8859b3920..e7d14c4e5 100644 --- a/packages/malloy/src/lang/ast/field-space/project-field-space.ts +++ b/packages/malloy/src/lang/ast/field-space/project-field-space.ts @@ -37,7 +37,7 @@ export class ProjectFieldSpace extends QuerySpace { canContain(typeDesc: TypeDesc | undefined): boolean { if ( typeDesc === undefined || - !TD.isLeafAtomic(typeDesc) || + !TD.isAtomic(typeDesc) || expressionIsAggregate(typeDesc.expressionType) ) { // We don't need to log here, because an error should have already been logged. diff --git a/packages/malloy/src/lang/ast/field-space/query-spaces.ts b/packages/malloy/src/lang/ast/field-space/query-spaces.ts index 9fddeec6a..0c9f5bc56 100644 --- a/packages/malloy/src/lang/ast/field-space/query-spaces.ts +++ b/packages/malloy/src/lang/ast/field-space/query-spaces.ts @@ -52,6 +52,7 @@ import { isEmptyCompositeFieldUsage, mergeCompositeFieldUsage, narrowCompositeFieldResolution, + NarrowedCompositeFieldResolution, } from '../../../model/composite_source_utils'; import {StructSpaceFieldBase} from './struct-space-field-base'; @@ -67,7 +68,7 @@ export abstract class QueryOperationSpace { protected exprSpace: QueryInputSpace; abstract readonly segmentType: 'reduce' | 'project' | 'index'; - expandedWild: Record = {}; + expandedWild: Record = {}; compositeFieldUsers: ( | {type: 'filter'; filter: model.FilterCondition; logTo: MalloyElement} | { @@ -78,7 +79,8 @@ export abstract class QueryOperationSpace } )[] = []; - // Composite field usage is not computed until `queryFieldDefs` is called; if anyone + // Composite field usage is not computed until `queryFieldDefs` is called + // (or `getPipeSegment` for index segments); if anyone // tries to access it before that, they'll get an error _compositeFieldUsage: model.CompositeFieldUsage | undefined = undefined; get compositeFieldUsage(): model.CompositeFieldUsage { @@ -164,7 +166,7 @@ export abstract class QueryOperationSpace continue; } if (this.entry(name)) { - const conflict = this.expandedWild[name]?.join('.'); + const conflict = this.expandedWild[name]?.path.join('.'); wild.logError( 'name-conflict-in-wildcard-expansion', `Cannot expand '${name}' in '${ @@ -181,7 +183,10 @@ export abstract class QueryOperationSpace (dialect === undefined || !dialect.ignoreInProject(name)) ) { expandEntries.push({name, entry}); - this.expandedWild[name] = joinPath.concat(name); + this.expandedWild[name] = { + path: joinPath.concat(name), + entry, + }; } } } @@ -212,7 +217,7 @@ export abstract class QueryOperationSpace ): model.CompositeFieldUsage { const reference = joinPath.map(n => new FieldName(n)); this.astEl.has({reference}); - const lookup = this.queryInputSpace.lookup(reference); + const lookup = this.exprSpace.lookup(reference); // Should always be found... if (lookup.found && lookup.found instanceof StructSpaceFieldBase) { return ( @@ -252,6 +257,45 @@ export abstract class QueryOperationSpace } super.newEntry(name, logTo, entry); } + + protected applyNextCompositeFieldUsage( + source: model.SourceDef, + compositeFieldUsage: model.CompositeFieldUsage, + narrowedCompositeFieldResolution: NarrowedCompositeFieldResolution, + nextCompositeFieldUsage: model.CompositeFieldUsage | undefined, + logTo: MalloyElement | undefined + ) { + if (nextCompositeFieldUsage) { + const newCompositeFieldUsage = + this.getCompositeFieldUsageIncludingJoinOns( + compositeFieldUsageDifference( + nextCompositeFieldUsage, + compositeFieldUsage + ) + ); + compositeFieldUsage = mergeCompositeFieldUsage( + compositeFieldUsage, + newCompositeFieldUsage + ); + if (!isEmptyCompositeFieldUsage(newCompositeFieldUsage)) { + const result = narrowCompositeFieldResolution( + source, + compositeFieldUsage, + narrowedCompositeFieldResolution + ); + if (result.error) { + (logTo ?? this).logError('invalid-composite-field-usage', { + newUsage: newCompositeFieldUsage, + allUsage: compositeFieldUsage, + }); + } else { + narrowedCompositeFieldResolution = + result.narrowedCompositeFieldResolution; + } + } + } + return {compositeFieldUsage, narrowedCompositeFieldResolution}; + } } // Project and Reduce or "QuerySegments" are built from a QuerySpace @@ -318,70 +362,45 @@ export abstract class QuerySpace extends QueryOperationSpace { const {name, field} = user; const wildPath = this.expandedWild[name]; if (wildPath) { - fields.push({type: 'fieldref', path: wildPath}); - continue; - } - // TODO handle wildcards for composite sources - const fieldQueryDef = field.getQueryFieldDef(this.exprSpace); - if (fieldQueryDef) { - const typeDesc = field.typeDesc(); - nextCompositeFieldUsage = typeDesc.compositeFieldUsage; - // Filter out fields whose type is 'error', which means that a totally bad field - // isn't sent to the compiler, where it will wig out. - // TODO Figure out how to make errors generated by `canContain` go in the right place, - // maybe by adding a logable element to SpaceFields. - if ( - typeDesc && - typeDesc.type !== 'error' && - this.canContain(typeDesc) && - !isEmptyNest(fieldQueryDef) - ) { - fields.push(fieldQueryDef); - } - } - // TODO I removed the error here because during calculation of the refinement space, - // (see creation of a QuerySpace) we add references to all the fields from - // the refinement, but they don't have definitions. So in the case where we - // don't have a field def, we "know" that that field is already in the query, - // and we don't need to worry about actually adding it. Previously, this was also true for - // project statements, where we added "*" as a field and also all the individual - // fields, but the individual fields didn't have field defs. - } - if (nextCompositeFieldUsage) { - const newCompositeFieldUsage = - this.getCompositeFieldUsageIncludingJoinOns( - compositeFieldUsageDifference( - nextCompositeFieldUsage, - compositeFieldUsage - ) - ); - compositeFieldUsage = mergeCompositeFieldUsage( - compositeFieldUsage, - newCompositeFieldUsage - ); - if (!isEmptyCompositeFieldUsage(newCompositeFieldUsage)) { - const result = narrowCompositeFieldResolution( - source, - compositeFieldUsage, - narrowedCompositeFieldResolution - ); - if (result.error) { - if (user.logTo) { - user.logTo.logError('invalid-composite-field-usage', { - newUsage: newCompositeFieldUsage, - allUsage: compositeFieldUsage, - }); - } else { - // This should not happen; logTo should only be not set for a field from a refinement, - // which should never fail the composite resolution - throw new Error('Unexpected invalid composite field resolution'); + fields.push({type: 'fieldref', path: wildPath.path}); + nextCompositeFieldUsage = + wildPath.entry.typeDesc().compositeFieldUsage; + } else { + const fieldQueryDef = field.getQueryFieldDef(this.exprSpace); + if (fieldQueryDef) { + const typeDesc = field.typeDesc(); + nextCompositeFieldUsage = typeDesc.compositeFieldUsage; + // Filter out fields whose type is 'error', which means that a totally bad field + // isn't sent to the compiler, where it will wig out. + // TODO Figure out how to make errors generated by `canContain` go in the right place, + // maybe by adding a logable element to SpaceFields. + if ( + typeDesc && + typeDesc.type !== 'error' && + this.canContain(typeDesc) && + !isEmptyNest(fieldQueryDef) + ) { + fields.push(fieldQueryDef); } - } else { - narrowedCompositeFieldResolution = - result.narrowedCompositeFieldResolution; } + // TODO I removed the error here because during calculation of the refinement space, + // (see creation of a QuerySpace) we add references to all the fields from + // the refinement, but they don't have definitions. So in the case where we + // don't have a field def, we "know" that that field is already in the query, + // and we don't need to worry about actually adding it. Previously, this was also true for + // project statements, where we added "*" as a field and also all the individual + // fields, but the individual fields didn't have field defs. } } + const next = this.applyNextCompositeFieldUsage( + source, + compositeFieldUsage, + narrowedCompositeFieldResolution, + nextCompositeFieldUsage, + user.logTo + ); + compositeFieldUsage = next.compositeFieldUsage; + narrowedCompositeFieldResolution = next.narrowedCompositeFieldResolution; } this._compositeFieldUsage = compositeFieldUsage; return fields; diff --git a/packages/malloy/src/lang/ast/field-space/reference-field.ts b/packages/malloy/src/lang/ast/field-space/reference-field.ts index af9bbff5c..86f7e7555 100644 --- a/packages/malloy/src/lang/ast/field-space/reference-field.ts +++ b/packages/malloy/src/lang/ast/field-space/reference-field.ts @@ -26,6 +26,7 @@ import { QueryFieldDef, TD, TypeDesc, + mkFieldDef, } from '../../../model/malloy_types'; import * as TDU from '../typedesc-utils'; import {FieldReference} from '../query-items/field-references'; @@ -69,12 +70,11 @@ export class ReferenceField extends SpaceField { const foundType = check.found.typeDesc(); if (TD.isAtomic(foundType)) { this.queryFieldDef = { - ...TDU.atomicDef(foundType), - name: path[0], + ...mkFieldDef(TDU.atomicDef(foundType), path[0], fs.dialectName()), e: {node: 'parameter', path}, }; } else { - // mtoy todo + // not sure what to do here, if we get here throw new Error('impossible turtle/join parameter'); } } else { diff --git a/packages/malloy/src/lang/ast/field-space/static-space.ts b/packages/malloy/src/lang/ast/field-space/static-space.ts index 590d090b3..eadd5c3c0 100644 --- a/packages/malloy/src/lang/ast/field-space/static-space.ts +++ b/packages/malloy/src/lang/ast/field-space/static-space.ts @@ -28,7 +28,7 @@ import { StructDef, SourceDef, isJoined, - isTurtleDef, + isTurtle, isSourceDef, JoinFieldDef, } from '../../../model/malloy_types'; @@ -72,8 +72,8 @@ export class StaticSpace implements FieldSpace { defToSpaceField(from: FieldDef): SpaceField { if (isJoined(from)) { - return new StructSpaceField(from, this.fromStruct.dialect); - } else if (isTurtleDef(from)) { + return new StructSpaceField(from); + } else if (isTurtle(from)) { return new IRViewField(this, from); } return new ColumnSpaceField(from); @@ -142,7 +142,7 @@ export class StaticSpace implements FieldSpace { lookup(path: FieldName[]): LookupResult { const head = path[0]; const rest = path.slice(1); - const found = this.entry(head.refString); + let found = this.entry(head.refString); if (!found) { return { error: { @@ -155,6 +155,17 @@ export class StaticSpace implements FieldSpace { if (found instanceof SpaceField) { const definition = found.fieldDef(); if (definition) { + if (!(found instanceof StructSpaceFieldBase) && isJoined(definition)) { + // We have looked up a field which is a join, but not a StructSpaceField + // because it is someting like "dimension: joinedArray is arrayComputation" + // which wasn't known to be a join when the fieldspace was constructed. + // TODO don't make one of these every time you do a lookup + found = new StructSpaceField(definition); + } + // cswenson review todo I don't know how to count the reference properly now + // i tried only writing it as a join reference if there was more in the path + // but that failed because lookup([JOINNAME]) is called when translating JOINNAME.AGGREGATE(...) + // with a 1-length-path but that IS a join reference and there is a test head.addReference({ type: found instanceof StructSpaceFieldBase @@ -183,7 +194,7 @@ export class StaticSpace implements FieldSpace { }; } } - } + } // cswenson review todo { else this is SpaceEntry not a field which can only be a param and what is going on? } const joinPath = found instanceof StructSpaceFieldBase ? [{...found.joinPathElement, name: head.refString}] @@ -217,10 +228,8 @@ export class StaticSpace implements FieldSpace { } export class StructSpaceField extends StructSpaceFieldBase { - private parentDialect: string; - constructor(def: JoinFieldDef, dialect: string) { + constructor(def: JoinFieldDef) { super(def); - this.parentDialect = dialect; } get fieldSpace(): FieldSpace { diff --git a/packages/malloy/src/lang/ast/field-space/view-field.ts b/packages/malloy/src/lang/ast/field-space/view-field.ts index 6390c8815..70c0527ed 100644 --- a/packages/malloy/src/lang/ast/field-space/view-field.ts +++ b/packages/malloy/src/lang/ast/field-space/view-field.ts @@ -21,7 +21,8 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -import {FieldDef, QueryFieldDef} from '../../../model/malloy_types'; +import {emptyCompositeFieldUsage} from '../../../model/composite_source_utils'; +import {QueryFieldDef, TurtleDef} from '../../../model/malloy_types'; import * as TDU from '../typedesc-utils'; import {FieldSpace} from '../types/field-space'; @@ -33,9 +34,14 @@ export abstract class ViewField extends SpaceField { } abstract getQueryFieldDef(fs: FieldSpace): QueryFieldDef | undefined; - abstract fieldDef(): FieldDef; + abstract fieldDef(): TurtleDef; typeDesc() { - return TDU.viewT; + const fieldDef = this.fieldDef(); + return { + ...TDU.viewT, + compositeFieldUsage: + fieldDef.compositeFieldUsage ?? emptyCompositeFieldUsage(), + }; } } diff --git a/packages/malloy/src/lang/ast/index.ts b/packages/malloy/src/lang/ast/index.ts index c4fc24ea7..544cd350f 100644 --- a/packages/malloy/src/lang/ast/index.ts +++ b/packages/malloy/src/lang/ast/index.ts @@ -79,10 +79,10 @@ export * from './expressions/partial-compare'; export * from './expressions/partition_by'; export * from './expressions/pick-when'; export * from './expressions/case'; +export * from './expressions/expr-array-literal'; export * from './expressions/expr-record-literal'; export * from './expressions/range'; export * from './expressions/time-frame'; -export * from './expressions/top-by'; export * from './expressions/unary'; export * from './statements/import-statement'; export * from './query-properties/extend'; @@ -116,7 +116,6 @@ export * from './query-properties/ordering'; export * from './query-properties/project-statement'; export * from './query-properties/qop-desc'; export * from './query-properties/sampling'; -export * from './query-properties/top'; export * from './source-elements/named-source'; export * from './source-elements/query-source'; export * from './source-elements/sql-source'; diff --git a/packages/malloy/src/lang/ast/query-builders/index-builder.ts b/packages/malloy/src/lang/ast/query-builders/index-builder.ts index d71810127..c6176067d 100644 --- a/packages/malloy/src/lang/ast/query-builders/index-builder.ts +++ b/packages/malloy/src/lang/ast/query-builders/index-builder.ts @@ -22,6 +22,7 @@ */ import { + CompositeFieldUsage, FilterCondition, PipeSegment, Sampling, @@ -41,6 +42,10 @@ import {QueryBuilder} from '../types/query-builder'; import {QueryInputSpace} from '../field-space/query-input-space'; import {QueryOperationSpace} from '../field-space/query-spaces'; import {MalloyElement} from '../types/malloy-element'; +import { + emptyCompositeFieldUsage, + mergeCompositeFieldUsage, +} from '../../../model/composite_source_utils'; export class IndexBuilder implements QueryBuilder { filters: FilterCondition[] = []; @@ -94,6 +99,10 @@ export class IndexBuilder implements QueryBuilder { } } + get compositeFieldUsage(): CompositeFieldUsage { + return this.resultFS.compositeFieldUsage; + } + finalize(from: PipeSegment | undefined): PipeSegment { if (from && !isIndexSegment(from) && !isPartialSegment(from)) { this.resultFS.logError( @@ -134,6 +143,15 @@ export class IndexBuilder implements QueryBuilder { indexSegment.alwaysJoins = [...this.alwaysJoins]; } + const fromCompositeFieldUsage = + from && from.type === 'index' + ? from.compositeFieldUsage ?? emptyCompositeFieldUsage() + : emptyCompositeFieldUsage(); + indexSegment.compositeFieldUsage = mergeCompositeFieldUsage( + fromCompositeFieldUsage, + this.compositeFieldUsage + ); + return indexSegment; } } diff --git a/packages/malloy/src/lang/ast/query-builders/reduce-builder.ts b/packages/malloy/src/lang/ast/query-builders/reduce-builder.ts index 2f79a8dc1..4b6cb5cf9 100644 --- a/packages/malloy/src/lang/ast/query-builders/reduce-builder.ts +++ b/packages/malloy/src/lang/ast/query-builders/reduce-builder.ts @@ -26,18 +26,23 @@ import { FilterCondition, PartialSegment, PipeSegment, + QueryFieldDef, QuerySegment, ReduceSegment, + canOrderBy, + expressionIsAggregate, + expressionIsAnalytic, + hasExpression, isPartialSegment, isQuerySegment, isReduceSegment, + isTemporalType, } from '../../../model/malloy_types'; import {ErrorFactory} from '../error-factory'; -import {SourceFieldSpace} from '../types/field-space'; +import {FieldName, SourceFieldSpace} from '../types/field-space'; import {Limit} from '../query-properties/limit'; import {Ordering} from '../query-properties/ordering'; -import {Top} from '../query-properties/top'; import {QueryProperty} from '../types/query-property'; import {QueryBuilder} from '../types/query-builder'; import { @@ -52,8 +57,15 @@ import { mergeCompositeFieldUsage, } from '../../../model/composite_source_utils'; +function queryFieldName(qf: QueryFieldDef): string { + if (qf.type === 'fieldref') { + return qf.path[qf.path.length - 1]; + } + return qf.name; +} + export abstract class QuerySegmentBuilder implements QueryBuilder { - order?: Top | Ordering; + order?: Ordering; limit?: number; alwaysJoins: string[] = []; abstract inputFS: QueryInputSpace; @@ -68,25 +80,6 @@ export abstract class QuerySegmentBuilder implements QueryBuilder { } if (qp instanceof DefinitionList) { this.resultFS.pushFields(...qp.list); - } else if (qp instanceof Top) { - if (this.limit) { - qp.logError( - 'limit-already-specified', - 'Query operation already limited' - ); - } else { - this.limit = qp.limit; - } - if (qp.by) { - if (this.order) { - qp.logError( - 'ordering-already-specified', - 'Query operation is already sorted' - ); - } else { - this.order = qp; - } - } } else if (qp instanceof Limit) { if (this.limit) { qp.logError( @@ -116,30 +109,21 @@ export abstract class QuerySegmentBuilder implements QueryBuilder { refineFrom(from: PipeSegment | undefined, to: QuerySegment): void { if (from && from.type !== 'index' && from.type !== 'raw') { - if (!this.order) { - if (from.orderBy) { - to.orderBy = from.orderBy; - } else if (from.by) { - to.by = from.by; - } + if (!this.limit && from.orderBy && !from.defaultOrderBy) { + to.orderBy = from.orderBy; } if (!this.limit && from.limit) { to.limit = from.limit; } } - if (this.limit) { - to.limit = this.limit; + if (this.order) { + to.orderBy = this.order.getOrderBy(this.inputFS); + delete to.defaultOrderBy; } - if (this.order instanceof Top) { - const topBy = this.order.getBy(this.inputFS); - if (topBy) { - to.by = topBy; - } - } - if (this.order instanceof Ordering) { - to.orderBy = this.order.getOrderBy(this.inputFS); + if (this.limit) { + to.limit = this.limit; } const oldFilters = from?.filterList || []; @@ -196,6 +180,60 @@ export class ReduceBuilder extends QuerySegmentBuilder implements QueryBuilder { const reduceSegment = this.resultFS.getQuerySegment(from); this.refineFrom(from, reduceSegment); + if (reduceSegment.orderBy) { + // In the modern world, we will ONLY allow names and not numbers in order by lists + for (const by of reduceSegment.orderBy) { + if (typeof by.field === 'number') { + const by_field = reduceSegment.queryFields[by.field - 1]; + by.field = queryFieldName(by_field); + } + } + } + if (reduceSegment.orderBy === undefined || reduceSegment.defaultOrderBy) { + // In the modern world, we will order all reduce segments with the default ordering + let usableDefaultOrderField: string | undefined; + for (const field of reduceSegment.queryFields) { + let fieldAggregate = false; + let fieldAnalytic = false; + let fieldType: string; + const fieldName = queryFieldName(field); + if (field.type === 'fieldref') { + const lookupPath = field.path.map(el => new FieldName(el)); + const refField = this.inputFS.lookup(lookupPath).found; + if (refField) { + const typeDesc = refField.typeDesc(); + fieldType = typeDesc.type; + fieldAggregate = expressionIsAggregate(typeDesc.expressionType); + fieldAnalytic = expressionIsAnalytic(typeDesc.expressionType); + } else { + continue; + } + } else { + fieldType = field.type; + fieldAggregate = + hasExpression(field) && expressionIsAggregate(field.expressionType); + fieldAnalytic = + hasExpression(field) && expressionIsAnalytic(field.expressionType); + } + if (isTemporalType(fieldType) || fieldAggregate) { + reduceSegment.defaultOrderBy = true; + reduceSegment.orderBy = [{field: fieldName, dir: 'desc'}]; + usableDefaultOrderField = undefined; + break; + } + if ( + canOrderBy(fieldType) && + !fieldAnalytic && + !usableDefaultOrderField + ) { + usableDefaultOrderField = fieldName; + } + } + if (usableDefaultOrderField) { + reduceSegment.defaultOrderBy = true; + reduceSegment.orderBy = [{field: usableDefaultOrderField, dir: 'asc'}]; + } + } return reduceSegment; } } diff --git a/packages/malloy/src/lang/ast/query-elements/query-base.ts b/packages/malloy/src/lang/ast/query-elements/query-base.ts index 99e18cfe2..a3b572864 100644 --- a/packages/malloy/src/lang/ast/query-elements/query-base.ts +++ b/packages/malloy/src/lang/ast/query-elements/query-base.ts @@ -26,7 +26,12 @@ import { isEmptyCompositeFieldUsage, resolveCompositeSources, } from '../../../model/composite_source_utils'; -import {isQuerySegment, Query, SourceDef} from '../../../model/malloy_types'; +import { + isIndexSegment, + isQuerySegment, + Query, + SourceDef, +} from '../../../model/malloy_types'; import {detectAndRemovePartialStages} from '../query-utils'; import {MalloyElement} from '../types/malloy-element'; import {QueryComp} from '../types/query-comp'; @@ -36,9 +41,12 @@ export abstract class QueryBase extends MalloyElement { query(): Query { const {inputStruct, query} = this.queryComp(true); - // TODO add an error if a raw/index query is done against a composite source + // TODO add an error if a raw query is done against a composite source let compositeResolvedSourceDef: SourceDef | undefined = undefined; - if (query.pipeline[0] && isQuerySegment(query.pipeline[0])) { + if ( + query.pipeline[0] && + (isQuerySegment(query.pipeline[0]) || isIndexSegment(query.pipeline[0])) + ) { const compositeFieldUsage = query.pipeline[0].compositeFieldUsage ?? emptyCompositeFieldUsage(); if ( diff --git a/packages/malloy/src/lang/ast/query-items/field-declaration.ts b/packages/malloy/src/lang/ast/query-items/field-declaration.ts index 7e2174e42..330853be6 100644 --- a/packages/malloy/src/lang/ast/query-items/field-declaration.ts +++ b/packages/malloy/src/lang/ast/query-items/field-declaration.ts @@ -29,9 +29,9 @@ import { TypeDesc, FieldDef, AtomicFieldDef, - TemporalTypeDef, isAtomic, FieldDefType, + mkFieldDef, } from '../../../model/malloy_types'; import * as TDU from '../typedesc-utils'; @@ -123,66 +123,36 @@ export abstract class AtomicFieldDeclaration type: 'error', }; } - let retType = exprValue.type; - if (retType === 'null') { + if (exprValue.type === 'null') { this.expr.logWarning( 'null-typed-field-definition', 'null value defaults to type number, use "null::TYPE" to specify correct type' ); - retType = 'number'; + const nullAsNumber: ExprValue = { + type: 'number', + value: exprValue.value, + expressionType: exprValue.expressionType, + evalSpace: exprValue.evalSpace, + compositeFieldUsage: exprValue.compositeFieldUsage, + }; + exprValue = nullAsNumber; } - if (isAtomicFieldType(retType) && retType !== 'error') { + if (isAtomicFieldType(exprValue.type) && exprValue.type !== 'error') { this.typecheckExprValue(exprValue); - let ret: AtomicFieldDef; - switch (retType) { - case 'date': - case 'timestamp': { - const timeRet: TemporalTypeDef & AtomicFieldDef = { - name: exprName, - type: retType, - location: this.location, - e: exprValue.value, - compositeFieldUsage: exprValue.compositeFieldUsage, - }; - if (isGranularResult(exprValue)) { - timeRet.timeframe = exprValue.timeframe; - } - ret = timeRet; - break; - } - case 'json': - case 'boolean': - case 'string': - case 'number': - case 'sql native': { - ret = { - type: retType, - name: exprName, - location: this.location, - e: exprValue.value, - compositeFieldUsage: exprValue.compositeFieldUsage, - }; - break; - } - case 'record': { - const fields: FieldDef[] = []; - ret = { - type: 'record', - name: exprName, - location: this.location, - join: 'one', - fields, - e: exprValue.value, - compositeFieldUsage: exprValue.compositeFieldUsage, - dialect: exprFS.dialectName(), - }; - break; - } - case 'array': - throw this.internalError( - 'Cannot return an array result from a query (yet)' - ); + const ret = mkFieldDef( + TDU.atomicDef(exprValue), + exprName, + exprFS.dialectName() + ); + if ( + (ret.type === 'date' || ret.type === 'timestamp') && + isGranularResult(exprValue) + ) { + ret.timeframe = exprValue.timeframe; } + ret.location = this.location; + ret.e = exprValue.value; + ret.compositeFieldUsage = exprValue.compositeFieldUsage; if (exprValue.expressionType) { ret.expressionType = exprValue.expressionType; } diff --git a/packages/malloy/src/lang/ast/query-properties/filters.ts b/packages/malloy/src/lang/ast/query-properties/filters.ts index 8c42c2ee6..92937bda9 100644 --- a/packages/malloy/src/lang/ast/query-properties/filters.ts +++ b/packages/malloy/src/lang/ast/query-properties/filters.ts @@ -95,9 +95,6 @@ export class Filter ): FilterCondition | undefined { const fExpr = filter.filterCondition(fs); - // mtoy todo is having we never set then queryRefinementStage might be wrong - // ... calculations and aggregations must go last - // Aggregates are ALSO checked at SQL generation time, but checking // here allows better reflection of errors back to user. if (this.havingClause !== undefined) { diff --git a/packages/malloy/src/lang/ast/query-properties/nest.ts b/packages/malloy/src/lang/ast/query-properties/nest.ts index 4e0ffe8db..89bfab684 100644 --- a/packages/malloy/src/lang/ast/query-properties/nest.ts +++ b/packages/malloy/src/lang/ast/query-properties/nest.ts @@ -52,6 +52,10 @@ export class NestFieldDeclaration fs, fs.outputSpace() ); + const compositeFieldUsage = + pipeline[0] && model.isQuerySegment(pipeline[0]) + ? pipeline[0].compositeFieldUsage + : undefined; const checkedPipeline = detectAndRemovePartialStages(pipeline, this); this.turtleDef = { type: 'turtle', @@ -59,6 +63,7 @@ export class NestFieldDeclaration pipeline: checkedPipeline, annotation: {...this.note, inherits: annotation}, location: this.location, + compositeFieldUsage, }; return this.turtleDef; } diff --git a/packages/malloy/src/lang/ast/query-properties/top.ts b/packages/malloy/src/lang/ast/query-properties/top.ts deleted file mode 100644 index e171e64b3..000000000 --- a/packages/malloy/src/lang/ast/query-properties/top.ts +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2023 Google LLC - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files - * (the "Software"), to deal in the Software without restriction, - * including without limitation the rights to use, copy, modify, merge, - * publish, distribute, sublicense, and/or sell copies of the Software, - * and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -import { - By as ModelBy, - expressionIsAggregate, - expressionIsAnalytic, -} from '../../../model/malloy_types'; - -import {ExpressionDef} from '../types/expression-def'; -import {FieldName, FieldSpace} from '../types/field-space'; -import {MalloyElement} from '../types/malloy-element'; -import { - LegalRefinementStage, - QueryPropertyInterface, -} from '../types/query-property-interface'; - -type TopInit = FieldName | ExpressionDef; - -export class Top extends MalloyElement implements QueryPropertyInterface { - elementType = 'top'; - queryRefinementStage = LegalRefinementStage.Tail; - forceQueryClass = undefined; - - constructor( - readonly limit: number, - readonly by?: TopInit - ) { - super(); - this.has({by: by}); - } - - getBy(fs: FieldSpace): ModelBy | undefined { - if (this.by) { - if (this.by instanceof FieldName) { - if (fs.isQueryFieldSpace()) { - // TODO jump-to-definition now that we can lookup fields in the output space, - // we need to actually add the reference when we do so. - const output = fs.outputSpace(); - const entry = this.by.getField(output); - if (entry.error) { - this.by.logError(entry.error.code, entry.error.message); - } - if (!entry.found || !entry.isOutputField) { - this.by.logError( - 'top-by-not-found-in-output', - `Unknown field ${this.by.refString} in output space` - ); - } - if (expressionIsAnalytic(entry.found?.typeDesc().expressionType)) { - this.by.logError( - 'top-by-analytic', - `Illegal order by of analytic field ${this.by.refString}` - ); - } - } - return {by: 'name', name: this.by.refString}; - } else { - const byExpr = this.by.getExpression(fs); - if (expressionIsAggregate(byExpr.expressionType)) { - this.by.logError( - 'top-by-aggregate', - 'top by expression must not be an aggregate' - ); - } - if (byExpr.evalSpace === 'output') { - this.by.logError( - 'top-by-not-in-output', - 'top by expression must be an output expression' - ); - } - return {by: 'expression', e: byExpr.value}; - } - } - return undefined; - } -} diff --git a/packages/malloy/src/lang/ast/source-properties/join.ts b/packages/malloy/src/lang/ast/source-properties/join.ts index 71fd76469..18cea5492 100644 --- a/packages/malloy/src/lang/ast/source-properties/join.ts +++ b/packages/malloy/src/lang/ast/source-properties/join.ts @@ -64,7 +64,7 @@ export abstract class Join fs.newEntry( this.name.refString, this, - new JoinSpaceField(fs.parameterSpace(), this, fs.dialect) + new JoinSpaceField(fs.parameterSpace(), this) ); } diff --git a/packages/malloy/src/lang/ast/typedesc-utils.ts b/packages/malloy/src/lang/ast/typedesc-utils.ts index 1c130ee44..75f789068 100644 --- a/packages/malloy/src/lang/ast/typedesc-utils.ts +++ b/packages/malloy/src/lang/ast/typedesc-utils.ts @@ -27,6 +27,7 @@ import { expressionIsScalar, ExpressionType, ExpressionValueType, + isRepeatedRecord, TD, TypeDesc, } from '../../model'; @@ -144,23 +145,19 @@ export function atomicDef(td: AtomicTypeDef | TypeDesc): AtomicTypeDef { if (TD.isAtomic(td)) { switch (td.type) { case 'array': { - return { - name: '', - type: 'array', - join: 'many', - elementTypeDef: td.elementTypeDef, - dialect: td.dialect, - fields: td.fields, - }; + return isRepeatedRecord(td) + ? { + type: 'array', + elementTypeDef: td.elementTypeDef, + fields: td.fields, + } + : { + type: 'array', + elementTypeDef: td.elementTypeDef, + }; } case 'record': { - return { - name: '', - type: 'record', - join: 'one', - dialect: td.dialect, - fields: td.fields, - }; + return {type: 'record', fields: td.fields}; } case 'number': { return td.numberType diff --git a/packages/malloy/src/lang/ast/types/binary_operators.ts b/packages/malloy/src/lang/ast/types/binary_operators.ts index a710f2f00..8693ab1c8 100644 --- a/packages/malloy/src/lang/ast/types/binary_operators.ts +++ b/packages/malloy/src/lang/ast/types/binary_operators.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {BinaryOperator} from '../../../model'; diff --git a/packages/malloy/src/lang/ast/types/dialect-name-space.ts b/packages/malloy/src/lang/ast/types/dialect-name-space.ts index b5c80334f..91a47a6cf 100644 --- a/packages/malloy/src/lang/ast/types/dialect-name-space.ts +++ b/packages/malloy/src/lang/ast/types/dialect-name-space.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {Dialect} from '../../../dialect'; diff --git a/packages/malloy/src/lang/ast/types/expression-def.ts b/packages/malloy/src/lang/ast/types/expression-def.ts index e76f4853a..89346a553 100644 --- a/packages/malloy/src/lang/ast/types/expression-def.ts +++ b/packages/malloy/src/lang/ast/types/expression-def.ts @@ -25,7 +25,7 @@ import { Expr, TimestampUnit, isDateUnit, - isTemporalField, + isTemporalType, expressionIsAggregate, TD, LeafExpressionType, @@ -181,7 +181,7 @@ export class ExprDuration extends ExpressionDef { ): ExprValue { const lhs = left.getExpression(fs); this.typeCheck(this, lhs); - if (isTemporalField(lhs.type) && (op === '+' || op === '-')) { + if (isTemporalType(lhs.type) && (op === '+' || op === '-')) { const num = this.n.getExpression(fs); if (!TDU.typeEq(num, TDU.numberT)) { this.logError( @@ -253,8 +253,8 @@ function timeCompare( op: CompareMalloyOperator, rhs: ExprValue ): Expr | undefined { - const leftIsTime = isTemporalField(lhs.type); - const rightIsTime = isTemporalField(rhs.type); + const leftIsTime = isTemporalType(lhs.type); + const rightIsTime = isTemporalType(rhs.type); const node = getExprNode(op); if (leftIsTime && rightIsTime) { if (lhs.type !== rhs.type) { @@ -459,7 +459,7 @@ function delta( return noGo; } - const timeLHS = isTemporalField(lhs.type); + const timeLHS = isTemporalType(lhs.type); const err = errorCascade(timeLHS ? 'error' : 'number', lhs, rhs); if (err) return err; diff --git a/packages/malloy/src/lang/ast/view-elements/reference-view.ts b/packages/malloy/src/lang/ast/view-elements/reference-view.ts index 157953f56..aa34a6ac4 100644 --- a/packages/malloy/src/lang/ast/view-elements/reference-view.ts +++ b/packages/malloy/src/lang/ast/view-elements/reference-view.ts @@ -25,7 +25,7 @@ import { PipeSegment, SourceDef, isAtomic, - isTurtleDef, + isTurtle, sourceBase, } from '../../../model/malloy_types'; import {ErrorFactory} from '../error-factory'; @@ -104,7 +104,7 @@ export class ReferenceView extends View { name, outputStruct, }; - } else if (isTurtleDef(fieldDef)) { + } else if (isTurtle(fieldDef)) { if (this.reference.list.length > 1) { if (forRefinement) { this.logError( diff --git a/packages/malloy/src/lang/ast/view-elements/refine-utils.ts b/packages/malloy/src/lang/ast/view-elements/refine-utils.ts index 47929a299..c5a02b3b6 100644 --- a/packages/malloy/src/lang/ast/view-elements/refine-utils.ts +++ b/packages/malloy/src/lang/ast/view-elements/refine-utils.ts @@ -64,13 +64,9 @@ export function refine( } if (from.type !== 'index' && to.type !== 'index' && from.type !== 'raw') { - if (from.orderBy !== undefined || from.by !== undefined) { - if (to.orderBy === undefined && to.by === undefined) { - if (from.orderBy) { - to.orderBy = from.orderBy; - } else if (from.by) { - to.by = from.by; - } + if (from.orderBy !== undefined && !from.defaultOrderBy) { + if (to.orderBy === undefined || to.defaultOrderBy) { + to.orderBy = from.orderBy; } else { logTo.logError( 'ordering-overridden-in-refinement', diff --git a/packages/malloy/src/lang/grammar/MalloyParser.g4 b/packages/malloy/src/lang/grammar/MalloyParser.g4 index b518b5034..8b5623f9e 100644 --- a/packages/malloy/src/lang/grammar/MalloyParser.g4 +++ b/packages/malloy/src/lang/grammar/MalloyParser.g4 @@ -487,7 +487,7 @@ bySpec ; topStatement - : TOP INTEGER_LITERAL bySpec? + : TOP INTEGER_LITERAL ; indexElement @@ -585,7 +585,7 @@ fieldExpr : fieldPath # exprFieldPath | literal # exprLiteral | OBRACK fieldExpr (COMMA fieldExpr)* COMMA? CBRACK # exprArrayLiteral - | OCURLY recordElement (COMMA recordElement)* closeCurly # exprLiteralRecord + | OCURLY recordElement (COMMA recordElement)* CCURLY # exprLiteralRecord | fieldExpr fieldProperties # exprFieldProps | fieldExpr timeframe # exprDuration | fieldExpr DOT timeframe # exprTimeTrunc diff --git a/packages/malloy/src/lang/malloy-to-ast.ts b/packages/malloy/src/lang/malloy-to-ast.ts index c01d0ac18..6b49eb28a 100644 --- a/packages/malloy/src/lang/malloy-to-ast.ts +++ b/packages/malloy/src/lang/malloy-to-ast.ts @@ -1133,30 +1133,9 @@ export class MalloyToAST return this.astAt(new ast.Ordering(orderList), pcx); } - visitTopStatement(pcx: parse.TopStatementContext): ast.Top { - const byCx = pcx.bySpec(); + visitTopStatement(pcx: parse.TopStatementContext): ast.Limit { const topN = this.getNumber(pcx.INTEGER_LITERAL()); - let top: ast.Top | undefined; - if (byCx) { - this.m4advisory( - byCx, - 'top-by', - 'by clause of top statement unupported. Use order_by instead' - ); - const nameCx = byCx.fieldName(); - if (nameCx) { - const name = this.getFieldName(nameCx); - top = new ast.Top(topN, name); - } - const exprCx = byCx.fieldExpr(); - if (exprCx) { - top = new ast.Top(topN, this.getFieldExpr(exprCx)); - } - } - if (!top) { - top = new ast.Top(topN, undefined); - } - return this.astAt(top, pcx); + return this.astAt(new ast.Limit(topN), pcx); } visitTopLevelQueryDefs( @@ -2075,11 +2054,6 @@ export class MalloyToAST } visitExprLiteralRecord(pcx: parse.ExprLiteralRecordContext) { - this.contextError( - pcx, - 'not-yet-implemented', - 'Record data is not yet implemented' - ); const els = this.only( pcx.recordElement().map(elCx => this.astAt(this.visit(elCx), elCx)), visited => visited instanceof ast.RecordElement && visited, @@ -2088,13 +2062,10 @@ export class MalloyToAST return new ast.RecordLiteral(els); } - visitExprArrayLiteral(pcx: parse.ExprArrayLiteralContext): ast.Unimplemented { - this.contextError( - pcx, - 'not-yet-implemented', - 'Array data is not yet implemented' - ); - return new ast.Unimplemented(); + visitExprArrayLiteral(pcx: parse.ExprArrayLiteralContext): ast.ArrayLiteral { + const contents = pcx.fieldExpr().map(fcx => this.getFieldExpr(fcx)); + const literal = new ast.ArrayLiteral(contents); + return this.astAt(literal, pcx); } visitExprWarnLike(pcx: parse.ExprWarnLikeContext): ast.ExprCompare { diff --git a/packages/malloy/src/lang/parse-log.ts b/packages/malloy/src/lang/parse-log.ts index a342cd60a..cc5ecc767 100644 --- a/packages/malloy/src/lang/parse-log.ts +++ b/packages/malloy/src/lang/parse-log.ts @@ -396,6 +396,7 @@ type MessageParameterTypes = { 'duplicate-include': string; 'exclude-after-include': string; 'cannot-rename-non-field': string; + 'array-values-incompatible': string; }; export const MESSAGE_FORMATTERS: PartialErrorCodeMessageMap = { diff --git a/packages/malloy/src/lang/test/composite-field-usage.spec.ts b/packages/malloy/src/lang/test/composite-field-usage.spec.ts index 36c6fd64d..280b2ea5c 100644 --- a/packages/malloy/src/lang/test/composite-field-usage.spec.ts +++ b/packages/malloy/src/lang/test/composite-field-usage.spec.ts @@ -166,11 +166,16 @@ describe('composite sources', () => { ##! experimental.composite_sources run: compose(a, a extend { dimension: one is 1 }) -> { group_by: one } `).toTranslate()); - test.skip('index on composite fails', () => + test('index on composite translates', () => expect(` ##! experimental.composite_sources - run: compose(a extend { dimension: two is 2 }, a extend { dimension: one is 1 }) -> { index: * } - `).toLog(errorMessage('Cannot index on a composite source'))); + source: x is compose( + a extend { except: ai }, + a + ) + run: x -> { index: ai } + run: x -> { index: * } + `).toTranslate()); test('raw run of composite source fails', () => expect(` ##! experimental.composite_sources @@ -257,5 +262,29 @@ describe('composite sources', () => { run: foo -> { group_by: x } `).toLog(errorMessage("'x' is internal")); }); + test('array.each is okay', () => { + expect(` + ##! experimental { composite_sources } + source: foo is compose( + a extend { dimension: x is 1 }, + a extend { dimension: y is 2 } + ) extend { + dimension: arr is [1, 2, 3] + } + run: foo -> { group_by: y, arr.each } + `).toTranslate(); + }); + test('timevalue extract okay', () => { + expect(` + ##! experimental { composite_sources } + source: foo is compose( + a extend { dimension: x is 1 }, + a extend { dimension: y is 2 } + ) extend { + dimension: time is now + } + run: foo -> { group_by: y, time.day } + `).toTranslate(); + }); }); }); diff --git a/packages/malloy/src/lang/test/field-symbols.spec.ts b/packages/malloy/src/lang/test/field-symbols.spec.ts index 22003e459..83e184947 100644 --- a/packages/malloy/src/lang/test/field-symbols.spec.ts +++ b/packages/malloy/src/lang/test/field-symbols.spec.ts @@ -112,7 +112,7 @@ describe('structdef comprehension', () => { }); test('import repeated record', () => { - const field: model.ArrayDef = { + const field: model.ScalarArrayDef = { name: 't', type: 'array', dialect: 'standardsql', @@ -131,7 +131,7 @@ describe('structdef comprehension', () => { }); test('import inline field', () => { - const field: model.RecordFieldDef = { + const field: model.RecordDef = { name: 't', type: 'record', dialect: 'standardsql', diff --git a/packages/malloy/src/lang/test/query.spec.ts b/packages/malloy/src/lang/test/query.spec.ts index a38575fe6..bce500906 100644 --- a/packages/malloy/src/lang/test/query.spec.ts +++ b/packages/malloy/src/lang/test/query.spec.ts @@ -898,6 +898,12 @@ describe('query:', () => { const fields = getFirstSegmentFieldNames(selstar.translator.getQuery(0)); expect(fields).toEqual(filterdFields); }); + test('array in query is passed into fields', () => { + const selArray = model`run: a -> { select: ais }`; + expect(selArray).toTranslate(); + const fields = getFirstSegmentFieldNames(selArray.translator.getQuery(0)); + expect(fields).toEqual(['ais']); + }); test('star error checking', () => { expect(markSource`run: a->{select: ${'zzz'}.*}`).toLog( errorMessage("No such field as 'zzz'") @@ -1018,50 +1024,75 @@ describe('query:', () => { test('top N', () => { expect('run: a->{ top: 5; group_by: astr }').toTranslate(); }); - test('top N by field', () => { - expect( - `##! m4warnings=warn - run: a->{top: 5 ${'by astr'}; group_by: astr}` - ).toLog( - warningMessage( - 'by clause of top statement unupported. Use order_by instead' - ) - ); - }); - test('top N by expression', () => { - expect( - `##! m4warnings=warn - run: ab->{top: 5 by ai + 1; group_by: ai}` - ).toLog( - warningMessage( - 'by clause of top statement unupported. Use order_by instead' - ) - ); - }); test('limit N', () => { expect('run: a->{ limit: 5; group_by: astr }').toTranslate(); }); - test('order by', () => { - expect('run: a->{ order_by: astr; group_by: astr }').toTranslate(); - }); - test('order by preserved over refinement', () => { - expect(` - query: a1 is a -> { group_by: astr } - run: a1 + { order_by: astr } - `).toTranslate(); - }); - test('order by must be in the output space', () => - expect('run: a -> { order_by: af; group_by: astr }').toLog( - errorMessage('Unknown field af in output space') - )); - test('order by asc', () => { - expect('run: a->{ order_by: astr asc; group_by: astr }').toTranslate(); - }); - test('order by desc', () => { - expect('run: a->{ order_by: astr desc; group_by: astr }').toTranslate(); - }); - test('order by N', () => { - expect('run: a->{ order_by: 1 asc; group_by: astr }').toTranslate(); + describe('order by variations', () => { + test('order by', () => { + expect('run: a->{ order_by: astr; group_by: astr }').toTranslate(); + }); + test('order by preserved over refinement', () => { + expect(` + query: a1 is a -> { group_by: astr } + run: a1 + { order_by: astr } + `).toTranslate(); + }); + test('order by must be in the output space', () => + expect('run: a -> { order_by: af; group_by: astr }').toLog( + errorMessage('Unknown field af in output space') + )); + test('order by asc', () => { + expect('run: a->{ order_by: astr asc; group_by: astr }').toTranslate(); + }); + test('order by desc', () => { + expect('run: a->{ order_by: astr desc; group_by: astr }').toTranslate(); + }); + test('order by N', () => { + expect('run: a->{ order_by: 1 asc; group_by: astr }').toTranslate(); + }); + test('first aggregate used for default ordering', () => { + const m = model`run: a->{ + group_by: astr + aggregate: t is ai.sum() + }`; + expect(m).toTranslate(); + const runStmt = m.translator.getQuery(0)!; + expect(runStmt).toBeDefined(); + const reduce = runStmt.pipeline[0]; + expect(reduce.type).toEqual('reduce'); + if (reduce.type === 'reduce') { + expect(reduce.defaultOrderBy).toBeTruthy(); + expect(reduce.orderBy).toEqual([{field: 't', dir: 'desc'}]); + } + }); + test('first temporal used for default ordering', () => { + const m = model`run: a->{ + group_by: astr, ats + }`; + expect(m).toTranslate(); + const runStmt = m.translator.getQuery(0)!; + expect(runStmt).toBeDefined(); + const reduce = runStmt.pipeline[0]; + expect(reduce.type).toEqual('reduce'); + if (reduce.type === 'reduce') { + expect(reduce.defaultOrderBy).toBeTruthy(); + expect(reduce.orderBy).toEqual([{field: 'ats', dir: 'desc'}]); + } + }); + test('first used for ordering when appropriate', () => { + const m = model`run: a->{ + group_by: astr, big is upper(astr) + }`; + expect(m).toTranslate(); + const runStmt = m.translator.getQuery(0)!; + expect(runStmt).toBeDefined(); + const reduce = runStmt.pipeline[0]; + expect(reduce.type).toEqual('reduce'); + if (reduce.type === 'reduce') { + expect(reduce.defaultOrderBy).toBeTruthy(); + expect(reduce.orderBy).toEqual([{field: 'astr', dir: 'asc'}]); + } + }); }); test('order by multiple', () => { expect(` diff --git a/packages/malloy/src/lang/test/test-translator.ts b/packages/malloy/src/lang/test/test-translator.ts index 64da47eb7..1acbf6e45 100644 --- a/packages/malloy/src/lang/test/test-translator.ts +++ b/packages/malloy/src/lang/test/test-translator.ts @@ -41,6 +41,8 @@ import { TableSourceDef, SQLSourceDef, SQLSentence, + NumberTypeDef, + mkArrayDef, } from '../../model/malloy_types'; import {ExpressionDef, MalloyElement} from '../ast'; import {NameSpace} from '../ast/types/name-space'; @@ -57,7 +59,7 @@ import {EventStream} from '../../runtime_types'; export function pretty(thing: any): string { return inspect(thing, {breakLength: 72, depth: Infinity}); } - +const intType: NumberTypeDef = {type: 'number', numberType: 'integer'}; const mockSchema: Record = { 'aTable': { type: 'table', @@ -68,7 +70,7 @@ const mockSchema: Record = { fields: [ {type: 'string', name: 'astr'}, {type: 'number', name: 'af', numberType: 'float'}, - {type: 'number', name: 'ai', numberType: 'integer'}, + {...intType, name: 'ai'}, {type: 'date', name: 'ad'}, {type: 'boolean', name: 'abool'}, {type: 'timestamp', name: 'ats'}, @@ -91,11 +93,12 @@ const mockSchema: Record = { { type: 'record', name: 'aninline', - fields: [{type: 'number', name: 'column', numberType: 'integer'}], + fields: [{...intType, name: 'column'}], join: 'one', matrixOperation: 'left', dialect: 'standardsql', }, + mkArrayDef(intType, 'ais', 'standardsql'), ], }, 'malloytest.carriers': { diff --git a/packages/malloy/src/model/composite_source_utils.ts b/packages/malloy/src/model/composite_source_utils.ts index 791d71575..d3f149520 100644 --- a/packages/malloy/src/model/composite_source_utils.ts +++ b/packages/malloy/src/model/composite_source_utils.ts @@ -42,7 +42,7 @@ function _resolveCompositeSources( let narrowedSources: SingleNarrowedCompositeFieldResolution | undefined = undefined; const nonCompositeFields = getNonCompositeFields(source); - if (compositeFieldUsage.fields.length > 0) { + if (compositeFieldUsage.fields.length > 0 || source.type === 'composite') { if (source.type === 'composite') { let found = false; // The narrowed source list is either the one given when this function was called, @@ -126,13 +126,6 @@ function _resolveCompositeSources( } else { return {error: {code: 'not_a_composite_source', data: {path}}}; } - } else if (source.type === 'composite') { - const first = source.sources[0]; - base = { - ...first, - fields: [...nonCompositeFields, ...base.fields], - filterList: [...(source.filterList ?? []), ...(first.filterList ?? [])], - }; } const fieldsByName: {[name: string]: FieldDef} = {}; const narrowedJoinedSources = narrowedCompositeFieldResolution?.joined ?? {}; @@ -149,10 +142,13 @@ function _resolveCompositeSources( error: {code: 'composite_source_not_defined', data: {path: newPath}}, }; } - if (!isJoined(join) || !isSourceDef(join)) { + if (!isJoined(join)) { return { error: {code: 'composite_source_not_a_join', data: {path: newPath}}, }; + } else if (!isSourceDef(join)) { + // Non-source join, like an array, skip it (no need to resolve) + continue; } const resolved = _resolveCompositeSources( newPath, @@ -193,7 +189,7 @@ type SingleNarrowedCompositeFieldResolution = { nested?: SingleNarrowedCompositeFieldResolution | undefined; }[]; -interface NarrowedCompositeFieldResolution { +export interface NarrowedCompositeFieldResolution { source: SingleNarrowedCompositeFieldResolution | undefined; joined: {[name: string]: NarrowedCompositeFieldResolution}; } @@ -338,20 +334,18 @@ export function compositeFieldUsageDifference( return { fields: arrayDifference(a.fields, b.fields), joinedUsage: Object.fromEntries( - Object.entries(a.joinedUsage) - .map( - ([joinName, joinedUsage]) => - [ - joinName, - joinName in b.joinedUsage - ? compositeFieldUsageDifference( - joinedUsage, - b.joinedUsage[joinName] - ) - : joinedUsage, - ] as [string, CompositeFieldUsage] - ) - .filter(([_, joinedUsage]) => countCompositeFieldUsage(joinedUsage) > 0) + Object.entries(a.joinedUsage).map( + ([joinName, joinedUsage]) => + [ + joinName, + joinName in b.joinedUsage + ? compositeFieldUsageDifference( + joinedUsage, + b.joinedUsage[joinName] + ) + : joinedUsage, + ] as [string, CompositeFieldUsage] + ) ), }; } diff --git a/packages/malloy/src/model/malloy_query.ts b/packages/malloy/src/model/malloy_query.ts index f5903ea2b..3ef9fe2e8 100644 --- a/packages/malloy/src/model/malloy_query.ts +++ b/packages/malloy/src/model/malloy_query.ts @@ -21,7 +21,12 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ import {v4 as uuidv4} from 'uuid'; -import {Dialect, DialectFieldList, getDialect} from '../dialect'; +import { + Dialect, + DialectFieldList, + FieldReferenceType, + getDialect, +} from '../dialect'; import {StandardSQLDialect} from '../dialect/standardsql/standardsql'; import { AggregateFunctionType, @@ -37,11 +42,11 @@ import { FunctionOverloadDef, FunctionParameterDef, getIdentifier, - getAtomicFields, hasExpression, IndexFieldDef, IndexSegment, isLiteral, + isAtomic, isIndexSegment, isQuerySegment, isRawSegment, @@ -81,7 +86,6 @@ import { SourceDef, isSourceDef, fieldIsIntrinsic, - AtomicFieldDef, StringFieldDef, NumberFieldDef, BooleanFieldDef, @@ -97,14 +101,17 @@ import { isJoinedSource, QueryResultDef, isScalarArray, - RecordFieldDef, + RecordDef, FinalizeSourceDef, QueryToMaterialize, PrepareResultOptions, - RepeatedRecordFieldDef, + RepeatedRecordDef, CaseExpr, TemporalTypeDef, mkTemporal, + JoinFieldDef, + LeafAtomicDef, + Expression, } from './malloy_types'; import {Connection} from '../connection/types'; @@ -241,7 +248,8 @@ class StageWriter { } sql += dialect.sqlCreateFunctionCombineLastStage( lastStageName, - getDialectFieldList(structDef) + getDialectFieldList(structDef), + (structDef.resultMetadata as ResultStructMetadataDef)?.orderBy ); const id = `${dialect.udfPrefix}${this.root().udfs.length}`; @@ -332,30 +340,22 @@ class StageWriter { if (!this.useCTE) { return dialect.sqlCreateFunctionCombineLastStage( `(${this.withs[0]})`, - getDialectFieldList(structDef) + getDialectFieldList(structDef), + (structDef.resultMetadata as ResultStructMetadataDef)?.orderBy ); } else { return ( this.combineStages(true).sql + dialect.sqlCreateFunctionCombineLastStage( this.getName(this.withs.length - 1), - getDialectFieldList(structDef) + getDialectFieldList(structDef), + (structDef.resultMetadata as ResultStructMetadataDef)?.orderBy ) ); } } } -type QuerySomething = QueryField | QueryStruct | QueryTurtle; - -// type QueryNodeType = -// | "abstract" -// | "dimension" -// | "measure" -// | "query" -// | "turtle" -// | "struct"; - class GenerateState { whereSQL?: string; applyValue?: string; @@ -392,7 +392,7 @@ abstract class QueryNode { this.referenceId = referenceId ?? uuidv4(); } abstract getIdentifier(): string; - getChildByName(_name: string): QuerySomething | undefined { + getChildByName(_name: string): QueryField | undefined { return undefined; } } @@ -424,6 +424,10 @@ class QueryField extends QueryNode { return parent; } + isAtomic() { + return isAtomic(this.fieldDef); + } + caseGroup(groupSets: number[], s: string): string { if (groupSets.length === 0) { return s; @@ -447,7 +451,7 @@ class QueryField extends QueryNode { state: GenerateState ): string { // find the structDef and return the path to the field... - const field = context.getFieldByName(expr.path) as QueryField; + const field = context.getFieldByName(expr.path); if (hasExpression(field.fieldDef)) { const ret = this.exprToSQL( resultSet, @@ -1373,48 +1377,70 @@ class QueryField extends QueryNode { ); } - getExpr(): Expr { + generateExpression(resultSet: FieldInstanceResult): string { + // If the field itself is an expression, generate it .. if (hasExpression(this.fieldDef)) { - return this.fieldDef.e; + return this.exprToSQL(resultSet, this.parent, this.fieldDef.e); + } + // The field itself is not an expression, so we would like + // to generate a dotted path to the field, EXCEPT ... + // some of the steps in the dotting might not exist + // in the namespace of their parent, but rather be record + // expressions which should be evaluated in the namespace + // of their parent. + + // So we walk the tree and ask each one to compute itself + for ( + let ancestor: QueryStruct | undefined = this.parent; + ancestor !== undefined; + ancestor = ancestor.parent + ) { + if ( + ancestor.structDef.type === 'record' && + hasExpression(ancestor.structDef) && + ancestor.recordAlias === undefined + ) { + if (!ancestor.parent) { + throw new Error( + 'Inconcievable record ancestor with expression but no parent' + ); + } + const aliasValue = this.exprToSQL( + resultSet, + ancestor.parent, + ancestor.structDef.e + ); + ancestor.informOfAliasValue(aliasValue); + } } - const pType = this.parent.structDef.type; - return { - node: 'genericSQLExpr', - kids: {args: []}, - src: [ - this.parent.dialect.sqlFieldReference( - this.parent.getSQLIdentifier(), - this.fieldDef.name, - this.fieldDef.type, - pType === 'record' || pType === 'array' || pType === 'nest_source', - isScalarArray(this.parent.structDef) - ), - ], - }; - } - - generateExpression(resultSet: FieldInstanceResult): string { - return this.exprToSQL(resultSet, this.parent, this.getExpr()); + return this.parent.sqlChildReference( + this.fieldDef.name, + this.parent.structDef.type === 'record' + ? { + result: resultSet, + field: this, + } + : undefined + ); } } -function isCalculatedField( - f: QueryField -): f is QueryAtomicField { - return f instanceof QueryAtomicField && f.isCalculated(); +function isCalculatedField(f: QueryField): f is QueryFieldAtomic { + return f instanceof AbstractQueryAtomic && f.isCalculated(); } -function isAggregateField( - f: QueryField -): f is QueryAtomicField { - return f instanceof QueryAtomicField && f.isAggregate(); +function isAggregateField(f: QueryField): f is QueryFieldAtomic { + return f instanceof AbstractQueryAtomic && f.isAggregate(); } -function isScalarField(f: QueryField): f is QueryAtomicField { - return f instanceof QueryAtomicField && !f.isCalculated() && !f.isAggregate(); +function isScalarField(f: QueryField): f is QueryFieldAtomic { + return ( + f instanceof AbstractQueryAtomic && !f.isCalculated() && !f.isAggregate() + ); } -class QueryAtomicField extends QueryField { +type QueryFieldAtomic = AbstractQueryAtomic; +class AbstractQueryAtomic extends QueryField { fieldDef: T; constructor(fieldDef: T, parent: QueryStruct, refId?: string) { @@ -1447,25 +1473,33 @@ class QueryAtomicField extends QueryField { hasExpression(): boolean { return hasExpression(this.fieldDef); } + + isAtomic() { + return true; + } } // class QueryMeasure extends QueryField {} -class QueryFieldString extends QueryAtomicField {} -class QueryFieldNumber extends QueryAtomicField {} -class QueryFieldBoolean extends QueryAtomicField {} -class QueryFieldJSON extends QueryAtomicField {} -class QueryFieldUnsupported extends QueryAtomicField {} +class QueryFieldString extends AbstractQueryAtomic {} +class QueryFieldNumber extends AbstractQueryAtomic {} +class QueryFieldBoolean extends AbstractQueryAtomic {} +class QueryFieldJSON extends AbstractQueryAtomic {} +class QueryFieldUnsupported extends AbstractQueryAtomic {} -class QueryFieldDate extends QueryAtomicField { +class QueryFieldDate extends AbstractQueryAtomic { generateExpression(resultSet: FieldInstanceResult): string { const fd = this.fieldDef; + const superExpr = super.generateExpression(resultSet); if (!fd.timeframe) { - return super.generateExpression(resultSet); + return superExpr; } else { const truncated: TimeTruncExpr = { node: 'trunc', - e: mkTemporal(this.getExpr(), 'date'), + e: mkTemporal( + {node: 'genericSQLExpr', src: [superExpr], kids: {args: []}}, + 'date' + ), units: fd.timeframe, }; return this.exprToSQL(resultSet, this.parent, truncated); @@ -1483,7 +1517,7 @@ class QueryFieldDate extends QueryAtomicField { } } -class QueryFieldTimestamp extends QueryAtomicField { +class QueryFieldTimestamp extends AbstractQueryAtomic { // clone ourselves on demand as a timeframe. getChildByName(name: TimestampUnit): QueryFieldTimestamp { const fieldDef = { @@ -1495,7 +1529,7 @@ class QueryFieldTimestamp extends QueryAtomicField { } } -class QueryFieldDistinctKey extends QueryAtomicField { +class QueryFieldDistinctKey extends AbstractQueryAtomic { generateExpression(resultSet: FieldInstanceResult): string { if (this.parent.primaryKey()) { const pk = this.parent.getPrimaryKeyField(this.fieldDef); @@ -1508,20 +1542,18 @@ class QueryFieldDistinctKey extends QueryAtomicField { parentKey || '', // shouldn't have to do this... this.parent.dialect.sqlFieldReference( this.parent.getIdentifier(), + 'table', '__row_id', - 'string', - true, - false + 'string' ) ); } else { // return this.parent.getIdentifier() + "." + "__distinct_key"; return this.parent.dialect.sqlFieldReference( this.parent.getIdentifier(), + 'table', '__distinct_key', - 'string', - false, - false + 'string' ); } } @@ -1804,7 +1836,7 @@ class FieldInstanceResult implements FieldInstance { if (isScalarField(f.f)) { return 'nested'; } - if (f.f instanceof QueryStruct) { + if (f.f instanceof QueryFieldStruct) { ret = 'inline'; } } @@ -1918,12 +1950,14 @@ class FieldInstanceResult implements FieldInstance { findJoins(query: QueryQuery) { for (const dim of this.fields()) { - this.addStructToJoin( - dim.f.getJoinableParent(), - query, - dim.f.uniqueKeyPossibleUse(), - [] - ); + if (!(dim.f instanceof QueryFieldStruct)) { + this.addStructToJoin( + dim.f.getJoinableParent(), + query, + dim.f.uniqueKeyPossibleUse(), + [] + ); + } } for (const s of this.structs()) { s.findJoins(query); @@ -2179,16 +2213,16 @@ class JoinInstance { } } -/** nested query */ -class QueryTurtle extends QueryField {} - /** * Used by the translator to get the output StructDef of a pipe segment * * half translated to the new world of types .. */ export class Segment { - static nextStructDef(structDef: SourceDef, segment: PipeSegment): SourceDef { + static nextStructDef( + structDef: SourceDef, + segment: PipeSegment + ): QueryResultDef { const qs = new QueryStruct( structDef, undefined, @@ -2358,23 +2392,36 @@ class QueryQuery extends QueryField { uniqueKeyPossibleUse: UniqueKeyPossibleUse | undefined, joinStack: string[] ) { - const node = context.getFieldByName(path); - let struct; - if (node instanceof QueryField) { - struct = node.parent; - } else if (node instanceof QueryStruct) { - struct = node; - } else { - throw new Error('Internal Error: Unknown object type'); + if (path.length === 0) { + return; } + const node = context.getFieldByName(path); + const joinableParent = + node instanceof QueryFieldStruct + ? node.queryStruct.getJoinableParent() + : node.parent.getJoinableParent(); resultStruct .root() - .addStructToJoin( - struct.getJoinableParent(), - this, - uniqueKeyPossibleUse, - joinStack - ); + .addStructToJoin(joinableParent, this, uniqueKeyPossibleUse, joinStack); + } + + findRecordAliases(context: QueryStruct, path: string[]) { + for (const seg of path) { + const field = context.getFieldByName([seg]); + if (field instanceof QueryFieldStruct) { + const qs = field.queryStruct; + if ( + qs.structDef.type === 'record' && + hasExpression(qs.structDef) && + qs.parent + ) { + qs.informOfAliasValue( + this.exprToSQL(this.rootResult, qs.parent, qs.structDef.e) + ); + } + context = qs; + } + } } addDependantExpr( @@ -2436,6 +2483,7 @@ class QueryQuery extends QueryField { } } if (expr.node === 'field') { + this.findRecordAliases(context, expr.path); const field = context.getDimensionOrMeasureByName(expr.path); if (hasExpression(field.fieldDef)) { this.addDependantExpr( @@ -2457,6 +2505,7 @@ class QueryQuery extends QueryField { } else if (expr.node === 'aggregate') { if (isAsymmetricExpr(expr)) { if (expr.structPath) { + this.findRecordAliases(context, expr.structPath); this.addDependantPath( resultStruct, context, @@ -2498,7 +2547,7 @@ class QueryQuery extends QueryField { for (const f of this.getSegmentFields(resultStruct)) { const {as, field} = this.expandField(f); - if (field instanceof QueryTurtle || field instanceof QueryQuery) { + if (field instanceof QueryQuery) { if (this.firstSegment.type === 'project') { throw new Error( `Nested views cannot be used in select - '${field.fieldDef.name}'` @@ -2510,7 +2559,7 @@ class QueryQuery extends QueryField { ); this.expandFields(fir); resultStruct.add(as, fir); - } else if (field instanceof QueryAtomicField) { + } else if (field instanceof AbstractQueryAtomic) { resultStruct.addField(as, field, { resultIndex, type: 'result', @@ -2524,16 +2573,11 @@ class QueryQuery extends QueryField { ); } } - // } else if (field instanceof QueryStruct) { - // // this could probably be optimized. We are adding the primary key of the joined structure - // // instead of the foreignKey. We have to do this in at least the INNER join case - // // so i'm just going to let the SQL database do the optimization (which is pretty rudimentary) - // const pkFieldDef = field.getAsQueryField(); - // resultStruct.addField(as, pkFieldDef, { - // resultIndex, - // type: "result", - // }); - // resultStruct.addStructToJoin(field, false); + } else if (field instanceof QueryFieldStruct) { + resultStruct.addField(as, field, { + resultIndex, + type: 'result', + }); } // else if ( // this.firstSegment.type === "project" && @@ -2617,8 +2661,8 @@ class QueryQuery extends QueryField { const alwaysJoins = stage.alwaysJoins ?? []; for (const joinName of alwaysJoins) { const qs = this.parent.getChildByName(joinName); - if (qs instanceof QueryStruct) { - rootResult.addStructToJoin(qs, this, undefined, []); + if (qs instanceof QueryFieldStruct) { + rootResult.addStructToJoin(qs.queryStruct, this, undefined, []); } } } @@ -2697,7 +2741,7 @@ class QueryQuery extends QueryField { getResultStructDef( resultStruct: FieldInstanceResult = this.rootResult, isRoot = true - ): SourceDef { + ): QueryResultDef { const fields: FieldDef[] = []; let primaryKey; this.prepare(undefined); @@ -2713,7 +2757,7 @@ class QueryQuery extends QueryField { ); if (fi.getRepeatedResultType() === 'nested') { - const multiLineNest: RepeatedRecordFieldDef = { + const multiLineNest: RepeatedRecordDef = { ...structDef, type: 'array', elementTypeDef: {type: 'record_element'}, @@ -2723,7 +2767,7 @@ class QueryQuery extends QueryField { }; fields.push(multiLineNest); } else { - const oneLineNest: RecordFieldDef = { + const oneLineNest: RecordDef = { ...structDef, type: 'record', join: 'one', @@ -2734,11 +2778,6 @@ class QueryQuery extends QueryField { } } else if (fi instanceof FieldInstanceField) { if (fi.fieldUsage.type === 'result') { - // mtoy todo -- remember why you commented this out - // if (fi.f instanceof QueryFieldStruct) { - // fields.push(fi.f.getAsJoinedStructDef(name)); - // } - // if there is only one dimension, it is the primaryKey // if there are more, primaryKey is undefined. if (isScalarField(fi.f)) { @@ -2750,17 +2789,27 @@ class QueryQuery extends QueryField { dimCount++; } - const location = fi.f.fieldDef.location; - const annotation = fi.f.fieldDef.annotation; + // Remove computations because they are all resolved + let fOut = fi.f.fieldDef; + if (hasExpression(fOut)) { + fOut = {...fOut}; + // "as" because delete needs the property to be optional + delete (fOut as Expression).e; + delete (fOut as Expression).code; + delete (fOut as Expression).expressionType; + } + + const location = fOut.location; + const annotation = fOut.annotation; // build out the result fields... - switch (fi.f.fieldDef.type) { + switch (fOut.type) { case 'boolean': case 'json': case 'string': fields.push({ name, - type: fi.f.fieldDef.type, + type: fOut.type, resultMetadata, location, annotation, @@ -2768,8 +2817,8 @@ class QueryQuery extends QueryField { break; case 'date': case 'timestamp': { - const timeframe = fi.f.fieldDef.timeframe; - const fd: TemporalTypeDef = {type: fi.f.fieldDef.type}; + const timeframe = fOut.timeframe; + const fd: TemporalTypeDef = {type: fOut.type}; if (timeframe) { fd.timeframe = timeframe; } @@ -2785,7 +2834,7 @@ class QueryQuery extends QueryField { case 'number': fields.push({ name, - numberType: fi.f.fieldDef.numberType, + numberType: fOut.numberType, type: 'number', resultMetadata, location, @@ -2793,11 +2842,14 @@ class QueryQuery extends QueryField { }); break; case 'sql native': - fields.push({...fi.f.fieldDef, resultMetadata, location}); + case 'record': + case 'array': { + fields.push({...fOut, resultMetadata}); break; + } default: throw new Error( - `unknown Field Type in query ${JSON.stringify(fi.f.fieldDef)}` + `unknown Field Type in query ${JSON.stringify(fOut)}` ); } } @@ -2820,7 +2872,11 @@ class QueryQuery extends QueryField { return outputStruct; } - generateSQLJoinBlock(stageWriter: StageWriter, ji: JoinInstance): string { + generateSQLJoinBlock( + stageWriter: StageWriter, + ji: JoinInstance, + depth: number + ): string { let s = ''; const qs = ji.queryStruct; const qsDef = qs.structDef; @@ -2889,7 +2945,7 @@ class QueryQuery extends QueryField { let select = `SELECT ${ji.alias}.*`; let joins = ''; for (const childJoin of ji.children) { - joins += this.generateSQLJoinBlock(stageWriter, childJoin); + joins += this.generateSQLJoinBlock(stageWriter, childJoin, depth + 1); select += `, ${this.parent.dialect.sqlSelectAliasAsStruct( childJoin.alias, getDialectFieldList(childJoin.queryStruct.structDef) @@ -2907,17 +2963,31 @@ class QueryQuery extends QueryField { if (qs.parent === undefined || ji.parent === undefined) { throw new Error('Internal Error, nested structure with no parent.'); } - const fieldExpression = this.parent.dialect.sqlFieldReference( - qs.parent.getSQLIdentifier(), - qsDef.name, - 'struct', - qs.parent.structDef.type === 'array', - isScalarArray(this.parent.structDef) - ); + // We need an SQL expression which results in the array for us to pass to un-nest + let arrayExpression: string; + + if (hasExpression(qsDef)) { + // If this array is NOT contained in the parent, but a computed entity + // then the thing we are joining is not "parent.childName", but + // the expression which is built in that namespace + arrayExpression = this.exprToSQL(this.rootResult, qs.parent, qsDef.e); + } else { + // If this is a reference through an expression at the top level, + // need to generate the expression because the expression is written + // in the top level, this call is being used to generate the join. + // Below the top level, the expression will have been written into + // a join at the top level, and the name will exist. + // ... not sure this is the right way to do this + // ... the test for this is called "source repeated record containing an array" + arrayExpression = qs.parent.sqlChildReference( + qsDef.name, + depth === 0 ? {result: this.rootResult, field: this} : undefined + ); + } // we need to generate primary key. If parent has a primary key combine // console.log(ji.alias, fieldExpression, this.inNestedPipeline()); s += `${this.parent.dialect.sqlUnnestAlias( - fieldExpression, + arrayExpression, ji.alias, ji.getDialectFieldList(), ji.makeUniqueKey, @@ -2932,7 +3002,7 @@ class QueryQuery extends QueryField { throw new Error(`Join type not implemented ${qs.structDef.type}`); } for (const childJoin of ji.children) { - s += this.generateSQLJoinBlock(stageWriter, childJoin); + s += this.generateSQLJoinBlock(stageWriter, childJoin, depth + 1); } return s; } @@ -2988,7 +3058,7 @@ class QueryQuery extends QueryField { } for (const childJoin of ji.children) { - s += this.generateSQLJoinBlock(stageWriter, childJoin); + s += this.generateSQLJoinBlock(stageWriter, childJoin, 0); } return s; } @@ -3802,7 +3872,7 @@ class QueryQuery extends QueryField { generateSQLFromPipeline(stageWriter: StageWriter): { lastStageName: string; - outputStruct: SourceDef; + outputStruct: QueryResultDef; } { this.parent.maybeEmitParameterizedSourceUsage(); this.prepare(stageWriter); @@ -3817,12 +3887,13 @@ class QueryQuery extends QueryField { }; pipeline.shift(); for (const transform of pipeline) { + const parent = this.parent.parent + ? {struct: this.parent.parent} + : {model: this.parent.getModel()}; const s = new QueryStruct( structDef, undefined, - { - model: this.parent.getModel(), - }, + parent, this.parent.prepareResultOptions ); const q = QueryQuery.makeQuery( @@ -3882,7 +3953,7 @@ class QueryQueryIndexStage extends QueryQuery { resultIndex, type: 'result', }); - if (field instanceof QueryAtomicField) { + if (field instanceof AbstractQueryAtomic) { this.addDependancies(resultStruct, field); } resultIndex++; @@ -4030,11 +4101,11 @@ class QueryQueryRaw extends QueryQuery { // Do nothing! } - getResultStructDef(): SourceDef { + getResultStructDef(): QueryResultDef { if (!isSourceDef(this.parent.structDef)) { - throw new Error(`Result cannot by type ${this.parent.structDef.type}`); + throw new Error(`Result cannot be type ${this.parent.structDef.type}`); } - return this.parent.structDef; + return {...this.parent.structDef, type: 'query_result'}; } getResultMetadata( @@ -4076,10 +4147,9 @@ class QueryQueryIndex extends QueryQuery { if (stage === undefined) { const f = this.parent.nameMap.get(fref.path[0]); if ( - f instanceof QueryStruct && - isJoined(f.structDef) && - f.structDef.join === 'many' && - f.structDef.fields.length > 1 + f instanceof QueryFieldStruct && + f.fieldDef.join === 'many' && + f.fieldDef.fields.length > 1 ) { const toStage = [fref]; stageMap[stageRoot] = toStage; @@ -4158,31 +4228,70 @@ class QueryQueryIndex extends QueryQuery { } } -/** Structure object as it is used to build a query */ /* - Sometimes this is built from a field def, as in a join + * The input to a query will always be a QueryStruct. A QueryStruct is also a namespace + * for tracking joins, and so a QueryFieldStruct is a QueryField which has a QueryStruct. + * + * This is a result of it being impossible to inherit both from QueryStruct and QueryField + * for array and record types. + */ +class QueryFieldStruct extends QueryField { + queryStruct: QueryStruct; + fieldDef: JoinFieldDef; + constructor( + jfd: JoinFieldDef, + sourceArguments: Record | undefined, + parent: QueryStruct, + prepareResultOptions: PrepareResultOptions, + referenceId?: string + ) { + super(jfd, parent, referenceId); + this.fieldDef = jfd; + this.queryStruct = new QueryStruct( + jfd, + sourceArguments, + {struct: parent}, + prepareResultOptions + ); + } + + /* + * Proxy the field-like methods that QueryStruct implements, eventually + * those probably should be in here ... I thought this would be important + * but maybe it isn't, it doesn't fix the problem I am working on ... + */ + + // mtoy todo review with lloyd if any of these are needed, had to NOT + // do getIdentifier to pass a test, didn't stop to think why. + // getIdentifier() { + // return this.queryStruct.getIdentifier(); + // } - But sometimes this is built as the intermediate stage between pipelines - and in that case it doesn't have a fieldDef which bugs me because querynode - always has a fielddef so i think that is wrong too + getJoinableParent() { + return this.queryStruct.getJoinableParent(); + } + + getFullOutputName() { + return this.queryStruct.getFullOutputName(); + } +} -*/ -class QueryStruct extends QueryNode { +/** Structure object as it is used to build a query */ +class QueryStruct { parent: QueryStruct | undefined; model: QueryModel; - nameMap = new Map(); + nameMap = new Map(); pathAliasMap: Map; dialect: Dialect; connectionName: string; + recordAlias?: string; constructor( public structDef: StructDef, readonly sourceArguments: Record | undefined, parent: ParentQueryStruct | ParentQueryModel, - readonly prepareResultOptions: PrepareResultOptions, - referenceId?: string + readonly prepareResultOptions: PrepareResultOptions ) { - super(referenceId); this.setParent(parent); if ('model' in parent) { @@ -4203,6 +4312,10 @@ class QueryStruct extends QueryNode { this.addFieldsFromFieldList(structDef.fields); } + informOfAliasValue(av: string): void { + this.recordAlias = av; + } + maybeEmitParameterizedSourceUsage() { if (isSourceDef(this.structDef)) { const paramsAndArgs = { @@ -4270,23 +4383,15 @@ class QueryStruct extends QueryNode { for (const field of fields) { const as = getIdentifier(field); - if (isJoined(field)) { - this.addFieldToNameMap( - as, - new QueryStruct( - field, - undefined, - {struct: this}, - this.prepareResultOptions - ) - ); - } else if (field.type === 'turtle') { + if (field.type === 'turtle') { this.addFieldToNameMap( as, QueryQuery.makeQuery(field, this, undefined, false) ); - } else { + } else if (isAtomic(field) || isJoinedSource(field)) { this.addFieldToNameMap(as, this.makeQueryField(field)); + } else { + throw new Error('mtoy did nit add field'); } } // if we don't have distinct key yet for this struct, add it. @@ -4345,16 +4450,53 @@ class QueryStruct extends QueryNode { } } + sqlChildReference( + name: string, + expand: {result: FieldInstanceResult; field: QueryField} | undefined + ) { + let parentRef = this.getSQLIdentifier(); + if (expand && isAtomic(this.structDef) && hasExpression(this.structDef)) { + parentRef = expand.field.exprToSQL(expand.result, this, this.structDef.e); + } + let refType: FieldReferenceType = 'table'; + if (this.structDef.type === 'record') { + refType = 'record'; + } else if (this.structDef.type === 'array') { + refType = + this.structDef.elementTypeDef.type === 'record_element' + ? 'array[record]' + : 'array[scalar]'; + } else if (this.structDef.type === 'nest_source') { + refType = 'nest source'; + } + const child = this.getChildByName(name); + const childType = child?.fieldDef.type || 'unknown'; + return this.dialect.sqlFieldReference(parentRef, refType, name, childType); + } + // return the name of the field in SQL getIdentifier(): string { // if it is the root table, use provided alias if we have one. if (isBaseTable(this.structDef)) { return 'base'; } + + // If this is a synthetic column, return the expression rather than the name + // because the name will not exist. Only for records because the other types + // will have joins and thus be in the namespace. We can't compute it here + // because we don't have access to the Query to call exprToSQL. + if (this.structDef.type === 'record' && hasExpression(this.structDef)) { + if (this.recordAlias) { + return this.recordAlias; + } + throw new Error('INTERNAL ERROR, record field alias not pre-computed'); + } + // if this is an inline object, include the parents alias. if (this.structDef.type === 'record' && this.parent) { - return ( - this.parent.getSQLIdentifier() + '.' + getIdentifier(this.structDef) + return this.parent.sqlChildReference( + getIdentifier(this.structDef), + undefined ); } // we are somewhere in the join tree. Make sure the alias is unique. @@ -4397,7 +4539,7 @@ class QueryStruct extends QueryNode { return this; } - addFieldToNameMap(as: string, n: QuerySomething) { + addFieldToNameMap(as: string, n: QueryField) { if (this.nameMap.has(as)) { throw new Error(`Redefinition of ${as}`); } @@ -4405,7 +4547,7 @@ class QueryStruct extends QueryNode { } /** the the primary key or throw an error. */ - getPrimaryKeyField(fieldDef: FieldDef): QueryAtomicField { + getPrimaryKeyField(fieldDef: FieldDef): QueryFieldAtomic { let pk; if ((pk = this.primaryKey())) { return pk; @@ -4444,8 +4586,8 @@ class QueryStruct extends QueryNode { } } for (const [, v] of this.nameMap) { - if (v instanceof QueryStruct) { - v.resolveQueryFields(); + if (v instanceof QueryFieldStruct) { + v.queryStruct.resolveQueryFields(); } } } @@ -4481,6 +4623,18 @@ class QueryStruct extends QueryNode { /** makes a new queryable field object from a fieldDef */ makeQueryField(field: FieldDef, referenceId?: string): QueryField { switch (field.type) { + case 'array': + case 'record': + case 'query_source': + case 'table': + case 'sql_select': + case 'composite': + return new QueryFieldStruct( + field, + undefined, + this, + this.prepareResultOptions + ); case 'string': return new QueryFieldString(field, this, referenceId); case 'date': @@ -4495,13 +4649,12 @@ class QueryStruct extends QueryNode { return new QueryFieldJSON(field, this, referenceId); case 'sql native': return new QueryFieldUnsupported(field, this, referenceId); - // case "reduce": - // case "project": - // case "index": case 'turtle': - return new QueryTurtle(field, this, referenceId); + return QueryQuery.makeQuery(field, this, undefined, false); default: - throw new Error(`unknown field definition ${JSON.stringify(field)}`); + throw new Error( + `unknown field definition ${(JSON.stringify(field), undefined, 2)}` + ); } } @@ -4552,7 +4705,7 @@ class QueryStruct extends QueryNode { return this.parent ? this.parent.root() : this; } - primaryKey(): QueryAtomicField | undefined { + primaryKey(): QueryFieldAtomic | undefined { if (isSourceDef(this.structDef) && this.structDef.primaryKey) { return this.getDimensionByName([this.structDef.primaryKey]); } else { @@ -4560,48 +4713,56 @@ class QueryStruct extends QueryNode { } } - getChildByName(name: string): QuerySomething | undefined { + getChildByName(name: string): QueryField | undefined { return this.nameMap.get(name); } /** convert a path into a field reference */ - getFieldByName(path: string[]): QuerySomething { - return path.reduce((lookIn: QuerySomething, childName: string) => { - const r = lookIn.getChildByName(childName); - if (r === undefined) { - throw new Error( - path.length === 1 - ? `'${childName}' not found` - : `'${childName}' not found in '${path.join('.')}'` - ); + getFieldByName(path: string[]): QueryField { + let found: QueryField | undefined = undefined; + let lookIn = this as QueryStruct | undefined; + let notFound = path[0]; + for (const n of path) { + found = lookIn?.getChildByName(n); + if (!found) { + notFound = n; + break; } - return r; - }, this); + lookIn = + found instanceof QueryFieldStruct ? found.queryStruct : undefined; + } + if (found === undefined) { + const pathErr = path.length > 1 ? ` in ${path.join('.')}` : ''; + throw new Error(`${notFound} not found${pathErr}`); + } + return found; } // structs referenced in queries are converted to fields. - getQueryFieldByName(name: string[]): QuerySomething { + getQueryFieldByName(name: string[]): QueryField { const field = this.getFieldByName(name); - if (field instanceof QueryStruct) { + if (field instanceof QueryFieldStruct) { throw new Error(`Cannot reference ${name.join('.')} as a scalar'`); } return field; } getQueryFieldReference( - name: string[], + path: string[], annotation: Annotation | undefined - ): QuerySomething { - const field = this.getFieldByName(name); + ): QueryField { + const field = this.getFieldByName(path); if (annotation) { if (field.parent === undefined) { - throw new Error('Unexpected reference to orphaned query field'); + throw new Error( + 'Inconcievable, field reference to orphaned query field' + ); } // Made a field object from the source, but the annotations were computed by the compiler // when it generated the reference, and has both the source and reference annotations included. - if (field instanceof QueryStruct) { - const newDef = {...field.structDef, annotation}; - return new QueryStruct( + if (field instanceof QueryFieldStruct) { + const newDef = {...field.fieldDef, annotation}; + return new QueryFieldStruct( newDef, undefined, field.parent, @@ -4616,37 +4777,37 @@ class QueryStruct extends QueryNode { return field; } - getDimensionOrMeasureByName( - name: string[] - ): QueryAtomicField { - const query = this.getFieldByName(name); - if (query instanceof QueryAtomicField) { - return query; + getDimensionOrMeasureByName(name: string[]) { + const field = this.getFieldByName(name); + if (!field.isAtomic()) { + throw new Error(`${name} is not an atomic field? Inconceivable!`); } - throw new Error(`${name} is not an atomic field? Inconceivable!`); + return field; } /** returns a query object for the given name */ - getDimensionByName(name: string[]): QueryAtomicField { - const query = this.getFieldByName(name); + getDimensionByName(name: string[]): QueryFieldAtomic { + const field = this.getFieldByName(name); - if (query instanceof QueryAtomicField && isScalarField(query)) { - return query; + if (field.isAtomic() && isScalarField(field)) { + return field; } throw new Error(`${name} is not an atomic scalar field? Inconceivable!`); } /** returns a query object for the given name */ getStructByName(name: string[]): QueryStruct { + if (name.length === 0) { + return this; + } const struct = this.getFieldByName(name); - if (struct instanceof QueryStruct) { - return struct; - } else { - throw new Error(`Error: Path to structure not found '${name.join('.')}'`); + if (struct instanceof QueryFieldStruct) { + return struct.queryStruct; } + throw new Error(`Error: Path to structure not found '${name.join('.')}'`); } - getDistinctKey(): QueryAtomicField { + getDistinctKey(): QueryFieldAtomic { if (this.structDef.type !== 'record') { return this.getDimensionByName(['__distinct_key']); } else if (this.parent) { @@ -4795,9 +4956,16 @@ export class QueryModel { // for (const f of ret.outputStruct.fields) { // fieldNames.push(getIdentifier(f)); // } - const fieldNames = getAtomicFields(ret.outputStruct).map(fieldDef => - q.parent.dialect.sqlMaybeQuoteIdentifier(fieldDef.name) - ); + const fieldNames: string[] = []; + for (const f of ret.outputStruct.fields) { + if (isAtomic(f)) { + const quoted = q.parent.dialect.sqlMaybeQuoteIdentifier(f.name); + fieldNames.push(quoted); + } + } + // const fieldNames = getAtomicFields(ret.outputStruct).map(fieldDef => + // q.parent.dialect.sqlMaybeQuoteIdentifier(fieldDef.name) + // ); ret.lastStageName = stageWriter.addStage( q.parent.dialect.sqlFinalStage(ret.lastStageName, fieldNames) ); @@ -4874,7 +5042,7 @@ export class QueryModel { const struct = this.getStructByName(explore); let indexStar: RefToField[] = []; for (const [fn, fv] of struct.nameMap) { - if (!(fv instanceof QueryStruct)) { + if (!(fv instanceof QueryFieldStruct)) { if (isScalarField(fv) && fv.includeInWildcard()) { indexStar.push({type: 'fieldref', path: [fn]}); } diff --git a/packages/malloy/src/model/malloy_types.ts b/packages/malloy/src/model/malloy_types.ts index f59268e73..751ed9151 100644 --- a/packages/malloy/src/model/malloy_types.ts +++ b/packages/malloy/src/model/malloy_types.ts @@ -96,11 +96,6 @@ export type Expr = | CompositeFieldExpr | ErrorNode; -interface HasTypeDef { - typeDef: AtomicTypeDef; -} -export type TypedExpr = Expr & HasTypeDef; - export type BinaryOperator = | '+' | '-' @@ -337,7 +332,8 @@ export interface BooleanLiteralNode extends ExprLeaf { export interface RecordLiteralNode extends ExprWithKids { node: 'recordLiteral'; - kids: Record; + kids: Record; + typeDef: RecordTypeDef; } export interface ArrayLiteralNode extends ExprWithKids { @@ -492,10 +488,13 @@ export interface ResultMetadataDef { referenceId?: string; } +export interface Ordered { + orderBy?: OrderBy[]; + defaultOrderBy?: boolean; +} // struct specific metadta -export interface ResultStructMetadataDef extends ResultMetadataDef { +export interface ResultStructMetadataDef extends ResultMetadataDef, Ordered { limit?: number; - orderBy?: OrderBy[]; } export interface ResultMetadata { @@ -594,14 +593,17 @@ export function maxOfExpressionTypes(types: ExpressionType[]): ExpressionType { } /** Grants access to the expression properties of a FieldDef */ +export interface HasExpression { + e: Expr; +} export function hasExpression( f: T -): f is T & Expression & {e: Expr} { +): f is T & Expression & HasExpression { return 'e' in f; } export type TemporalFieldType = 'date' | 'timestamp'; -export function isTemporalField(s: string): s is TemporalFieldType { +export function isTemporalType(s: string): s is TemporalFieldType { return s === 'date' || s === 'timestamp'; } export type CastType = @@ -630,6 +632,12 @@ export function isAtomicFieldType(s: string): s is AtomicFieldType { 'error', ].includes(s); } +export function canOrderBy(s: string) { + return ['string', 'number', 'date', 'boolean', 'date', 'timestamp'].includes( + s + ); +} + export function isCastType(s: string): s is CastType { return ['string', 'number', 'date', 'timestamp', 'boolean', 'json'].includes( s @@ -682,25 +690,76 @@ export interface NativeUnsupportedTypeDef { export type NativeUnsupportedFieldDef = NativeUnsupportedTypeDef & AtomicFieldDef; -export interface ArrayTypeDef extends JoinBase, StructDefBase { +export interface ScalarArrayTypeDef { + type: 'array'; + elementTypeDef: Exclude; +} +export interface ScalarArrayDef + extends ScalarArrayTypeDef, + StructDefBase, + JoinBase, + FieldBase { type: 'array'; - elementTypeDef: Exclude | RecordElementTypeDef; join: 'many'; } -export type ArrayDef = ArrayTypeDef & AtomicFieldDef; -export function arrayEachFields(arrayOf: AtomicTypeDef): AtomicFieldDef[] { - return [ - { - name: 'each', - ...arrayOf, - e: {node: 'field', path: ['value']}, - }, - {name: 'value', ...arrayOf}, - ]; +export function mkFieldDef( + atd: AtomicTypeDef, + name: string, + dialect: string +): AtomicFieldDef { + if (isScalarArray(atd)) { + return mkArrayDef(atd.elementTypeDef, name, dialect); + } + if (isRepeatedRecord(atd)) { + const {type, fields, elementTypeDef} = atd; + return {type, fields, elementTypeDef, join: 'many', name, dialect}; + } + if (atd.type === 'record') { + const {type, fields} = atd; + return {type, fields, join: 'one', dialect, name}; + } + return {...atd, name}; +} + +export function mkArrayDef( + ofType: AtomicTypeDef, + name: string, + dialect: string +): ArrayDef { + if (ofType.type === 'record') { + return { + type: 'array', + join: 'many', + name, + dialect, + elementTypeDef: {type: 'record_element'}, + fields: ofType.fields, + }; + } + const valueEnt = mkFieldDef(ofType, 'value', dialect); + return { + type: 'array', + join: 'many', + name, + dialect, + elementTypeDef: ofType, + fields: [ + valueEnt, + {...valueEnt, name: 'each', e: {node: 'field', path: ['value']}}, + ], + }; } -export interface RecordTypeDef extends StructDefBase, JoinBase { +export interface RecordTypeDef { + type: 'record'; + fields: FieldDef[]; +} +export interface RecordDef + extends RecordTypeDef, + StructDefBase, + JoinBase, + FieldBase { type: 'record'; join: 'one'; } @@ -722,19 +781,34 @@ export interface RecordElementTypeDef { type: 'record_element'; } -export interface RepeatedRecordTypeDef extends ArrayDef { +export interface RepeatedRecordTypeDef { type: 'array'; elementTypeDef: RecordElementTypeDef; + fields: FieldDef[]; +} +export interface RepeatedRecordDef + extends RepeatedRecordTypeDef, + StructDefBase, + JoinBase, + FieldBase { + type: 'array'; join: 'many'; } +export type ArrayTypeDef = ScalarArrayTypeDef | RepeatedRecordTypeDef; +export type ArrayDef = ScalarArrayDef | RepeatedRecordDef; -export type RecordFieldDef = RecordTypeDef & AtomicFieldDef; -export type RepeatedRecordFieldDef = RepeatedRecordTypeDef & AtomicFieldDef; - -export function isRepeatedRecord(fd: FieldDef): fd is RepeatedRecordFieldDef { +export function isRepeatedRecord( + fd: FieldDef | QueryFieldDef | StructDef | AtomicTypeDef +): fd is RepeatedRecordTypeDef { return fd.type === 'array' && fd.elementTypeDef.type === 'record_element'; } +export function isScalarArray( + td: AtomicTypeDef | FieldDef | QueryFieldDef | StructDef +): td is ScalarArrayTypeDef { + return td.type === 'array' && td.elementTypeDef.type !== 'record_element'; +} + export interface ErrorTypeDef { type: 'error'; } @@ -754,6 +828,7 @@ export function isMatrixOperation(x: string): x is MatrixOperation { } export type JoinElementType = + | 'composite' | 'table' | 'sql_select' | 'query_source' @@ -770,34 +845,28 @@ export interface JoinBase { } export type Joinable = + | CompositeSourceDef | TableSourceDef | SQLSourceDef | QuerySourceDef - | RecordFieldDef + | RepeatedRecordDef + | RecordDef | ArrayDef; -export type JoinFieldDef = JoinBase & Joinable; -export type JoinFieldTypes = - | 'table' - | 'sql_select' - | 'query_source' - | 'array' - | 'record'; +export type JoinFieldDef = Joinable & JoinBase; export function isJoinable(sd: StructDef): sd is Joinable { return [ + 'composite', 'table', 'sql_select', 'query_source', 'array', 'record', - 'composite', ].includes(sd.type); } -export function isJoined( - fd: T -): fd is T & Joinable & JoinBase { - return 'join' in fd; +export function isJoined(sd: TypedDef): sd is JoinFieldDef { + return 'join' in sd; } export function isJoinedSource(sd: StructDef): sd is SourceDef & JoinBase { @@ -858,30 +927,6 @@ export type FunctionOrderBy = | FunctionOrderByExpression | FunctionOrderByDefaultExpression; -export interface ByName { - by: 'name'; - name: string; -} -export interface ByExpression { - by: 'expression'; - e: Expr; -} -export type By = ByName | ByExpression; - -export function isByName(by: By | undefined): by is ByName { - if (by === undefined) { - return false; - } - return by.by === 'name'; -} - -export function isByExpression(by: By | undefined): by is ByExpression { - if (by === undefined) { - return false; - } - return by.by === 'name'; -} - /** reference to a data source */ // TODO this should be renamed to `SourceRef` export type StructRef = string | SourceDef; @@ -1012,6 +1057,7 @@ export interface IndexSegment extends Filtered { weightMeasure?: string; // only allow the name of the field to use for weights sample?: Sampling; alwaysJoins?: string[]; + compositeFieldUsage?: CompositeFieldUsage; } export function isIndexSegment(pe: PipeSegment): pe is IndexSegment { return (pe as IndexSegment).type === 'index'; @@ -1022,13 +1068,11 @@ export interface CompositeFieldUsage { joinedUsage: Record; } -export interface QuerySegment extends Filtered { +export interface QuerySegment extends Filtered, Ordered { type: 'reduce' | 'project' | 'partial'; queryFields: QueryFieldDef[]; extendSource?: FieldDef[]; limit?: number; - by?: By; - orderBy?: OrderBy[]; // uses output field name or index. queryTimezone?: string; alwaysJoins?: string[]; compositeFieldUsage?: CompositeFieldUsage; @@ -1041,6 +1085,7 @@ export interface TurtleDef extends NamedObject, Pipeline { type: 'turtle'; annotation?: Annotation; accessModifier?: NonDefaultAccessModifierLabel | undefined; + compositeFieldUsage?: CompositeFieldUsage; } interface StructDefBase extends HasLocation, NamedObject { @@ -1152,7 +1197,7 @@ export type SourceDef = /** Is this the "FROM" table of a query tree */ export function isBaseTable(def: StructDef): def is SourceDef { - if (isJoined(def)) { + if (isJoinedSource(def)) { return false; } if (isSourceDef(def)) { @@ -1161,11 +1206,7 @@ export function isBaseTable(def: StructDef): def is SourceDef { return false; } -export function isScalarArray(def: FieldDef | StructDef) { - return def.type === 'array' && def.elementTypeDef.type !== 'record_element'; -} - -export type StructDef = SourceDef | RecordFieldDef | ArrayDef; +export type StructDef = SourceDef | RecordDef | ArrayDef; // "NonAtomic" are types that a name lookup or a computation might // have which are not AtomicFieldDefs. I asked an AI for a word for @@ -1265,17 +1306,25 @@ export type LeafAtomicTypeDef = | JSONTypeDef | NativeUnsupportedTypeDef | ErrorTypeDef; -export type AtomicTypeDef = LeafAtomicTypeDef | ArrayTypeDef | RecordTypeDef; - export type LeafAtomicDef = LeafAtomicTypeDef & FieldBase; -export type AtomicFieldDef = AtomicTypeDef & FieldBase; + +export type AtomicTypeDef = + | LeafAtomicTypeDef + | ScalarArrayTypeDef + | RecordTypeDef + | RepeatedRecordTypeDef; +export type AtomicFieldDef = + | LeafAtomicDef + | ScalarArrayDef + | RecordDef + | RepeatedRecordDef; export function isLeafAtomic( fd: FieldDef | QueryFieldDef | AtomicTypeDef ): fd is LeafAtomicDef { return ( fd.type === 'string' || - isTemporalField(fd.type) || + isTemporalType(fd.type) || fd.type === 'number' || fd.type === 'boolean' || fd.type === 'json' || @@ -1285,7 +1334,7 @@ export function isLeafAtomic( } // Sources have fields like this ... -export type FieldDef = AtomicFieldDef | JoinFieldDef | TurtleDef; +export type FieldDef = LeafAtomicDef | JoinFieldDef | TurtleDef; export type FieldDefType = AtomicFieldType | 'turtle' | JoinElementType; // Queries have fields like this .. @@ -1297,6 +1346,14 @@ export interface RefToField { } export type QueryFieldDef = AtomicFieldDef | TurtleDef | RefToField; +// All these share the same "type" space +export type TypedDef = + | AtomicTypeDef + | JoinFieldDef + | TurtleDef + | RefToField + | StructDef; + /** Get the output name for a NamedObject */ export function getIdentifier(n: AliasedName): string { if (n.as !== undefined) { @@ -1397,11 +1454,11 @@ export interface QueryResult extends CompiledQuery { profilingUrl?: string; } -export function isTurtleDef(def: FieldDef): def is TurtleDef { +export function isTurtle(def: TypedDef): def is TurtleDef { return def.type === 'turtle'; } -export function isAtomic(def: FieldDef): def is AtomicFieldDef { +export function isAtomic(def: TypedDef): def is AtomicTypeDef { return isAtomicFieldType(def.type); } @@ -1413,10 +1470,6 @@ export interface SearchResultRow { export type SearchResult = SearchResultRow[]; -export function getAtomicFields(structDef: StructDef): AtomicFieldDef[] { - return structDef.fields.filter(isAtomic); -} - export function isValueString( value: QueryValue, field: FieldDef @@ -1490,7 +1543,8 @@ export const TD = { isDate: (td: UTD): td is DateTypeDef => td?.type === 'date', isTimestamp: (td: UTD): td is TimestampTypeDef => td?.type === 'timestamp', isTemporal(td: UTD): td is TimestampTypeDef { - return td?.type === 'timestamp' || td?.type === 'date'; + const typ = td?.type ?? ''; + return isTemporalType(typ); }, isError: (td: UTD): td is ErrorTypeDef => td?.type === 'error', eq(x: UTD, y: UTD): boolean { diff --git a/packages/malloy/src/model/materialization/utils.ts b/packages/malloy/src/model/materialization/utils.ts index 9edcb4413..5ae03615d 100644 --- a/packages/malloy/src/model/materialization/utils.ts +++ b/packages/malloy/src/model/materialization/utils.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {Tag} from '../../tags'; diff --git a/packages/malloy/src/version.ts b/packages/malloy/src/version.ts index 306b30569..e1fa74071 100644 --- a/packages/malloy/src/version.ts +++ b/packages/malloy/src/version.ts @@ -1,2 +1,2 @@ // generated with 'generate-version-file' script; do not edit manually -export const MALLOY_VERSION = '0.0.218'; +export const MALLOY_VERSION = '0.0.222'; diff --git a/test/mysql/mysql_start.sh b/test/mysql/mysql_start.sh index ffb47cbde..428bfa1cb 100755 --- a/test/mysql/mysql_start.sh +++ b/test/mysql/mysql_start.sh @@ -5,11 +5,13 @@ rm -rf .tmp mkdir .tmp # run docker -SCRIPTDIR=$(dirname $0) -docker run -p 3306:3306 -d -v $SCRIPTDIR/../data/mysql:/init_data --name mysql-malloy -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -d mysql:8.4.2 +SCRIPTDIR=$(cd $(dirname $0); pwd) +DATADIR=$(dirname $SCRIPTDIR)/data/mysql +docker run -p 3306:3306 -d -v $DATADIR:/init_data --name mysql-malloy -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -d mysql:8.4.2 # wait for server to start counter=0 +echo -n Starting Docker ... while ! docker logs mysql-malloy 2>&1 | grep -q "mysqld: ready for connections" do sleep 10 @@ -23,9 +25,12 @@ do exit 1 break fi + echo -n ... done # load the test data. +echo +echo Loading Test Data docker exec mysql-malloy cp /init_data/malloytest.mysql.gz /tmp docker exec mysql-malloy gunzip /tmp/malloytest.mysql.gz docker exec mysql-malloy mysql -P3306 -h127.0.0.1 -uroot -e 'drop database if exists malloytest; create database malloytest; use malloytest; source /tmp/malloytest.mysql;' diff --git a/test/mysql/mysql_stop.sh b/test/mysql/mysql_stop.sh index 7dce0ace5..8ae38e284 100755 --- a/test/mysql/mysql_stop.sh +++ b/test/mysql/mysql_stop.sh @@ -4,4 +4,4 @@ rm -rf .tmp # stop container -docker rm -f mysql-malloy \ No newline at end of file +docker rm -f mysql-malloy diff --git a/test/package.json b/test/package.json index 35bff7c70..ef1adc9d5 100644 --- a/test/package.json +++ b/test/package.json @@ -21,13 +21,13 @@ }, "dependencies": { "@jest/globals": "^29.4.3", - "@malloydata/db-bigquery": "^0.0.218", - "@malloydata/db-duckdb": "^0.0.218", - "@malloydata/db-postgres": "^0.0.218", - "@malloydata/db-snowflake": "^0.0.218", - "@malloydata/db-trino": "^0.0.218", - "@malloydata/malloy": "^0.0.218", - "@malloydata/render": "^0.0.218", + "@malloydata/db-bigquery": "^0.0.222", + "@malloydata/db-duckdb": "^0.0.222", + "@malloydata/db-postgres": "^0.0.222", + "@malloydata/db-snowflake": "^0.0.222", + "@malloydata/db-trino": "^0.0.222", + "@malloydata/malloy": "^0.0.222", + "@malloydata/render": "^0.0.222", "events": "^3.3.0", "jsdom": "^22.1.0", "luxon": "^2.4.0", @@ -37,5 +37,5 @@ "@types/jsdom": "^21.1.1", "@types/luxon": "^2.4.0" }, - "version": "0.0.218" + "version": "0.0.222" } diff --git a/test/src/databases/all/composite_sources.spec.ts b/test/src/databases/all/composite_sources.spec.ts index 6cfd391d2..1cdace45b 100644 --- a/test/src/databases/all/composite_sources.spec.ts +++ b/test/src/databases/all/composite_sources.spec.ts @@ -35,7 +35,7 @@ describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => { run: y -> { group_by: x.foo } `).malloyResultMatches(runtime, {foo: 1}); }); - it('composite used in join on', async () => { + it('composite field from joined source used in join on', async () => { await expect(` ##! experimental.composite_sources source: state_facts is ${databaseName}.table('malloytest.state_facts') @@ -49,6 +49,41 @@ describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => { run: y -> { group_by: ca.state; where: state = 'IL' } `).malloyResultMatches(runtime, {state: 'CA'}); }); + it('composite field from joining source used in join on', async () => { + await expect(` + ##! experimental.composite_sources + source: state_facts is ${databaseName}.table('malloytest.state_facts') + source: x is compose( + state_facts extend { + dimension: + state_one is 'CA' + state_two is 'IL' + }, + state_facts extend { + dimension: state_one is 'IL' + } + ) extend { + join_one: state_facts on state_one = state_facts.state + } + run: x -> { group_by: state_facts.state } + `).malloyResultMatches(runtime, {state: 'CA'}); + }); + it('query against composite resolves nested composite source even when no composite fields', async () => { + await expect(` + ##! experimental.composite_sources + source: state_facts is ${databaseName}.table('malloytest.state_facts') + source: x is compose( + compose( + state_facts, + state_facts + ), + state_facts + ) extend { + dimension: a is 1 + } + run: x -> { group_by: a } + `).malloyResultMatches(runtime, {a: 1}); + }); // TODO test always join composite field usage it('composite field used in view', async () => { await expect(` @@ -197,4 +232,38 @@ describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => { } `).malloyResultMatches(runtime, {x: 1}); }); + it('reference composite field in nest', async () => { + await expect(` + ##! experimental { composite_sources parameters } + source: state_facts is ${databaseName}.table('malloytest.state_facts') + run: compose(state_facts, state_facts extend { dimension: x is 1 }) -> { + nest: foo is { + group_by: x + } + } + `).malloyResultMatches(runtime, {'foo.x': 1}); + }); + it('composite with select *', async () => { + await expect(` + ##! experimental.composite_sources + source: state_facts is ${databaseName}.table('malloytest.state_facts') + source: x is compose(state_facts, state_facts extend { dimension: foo is 1 }) extend { + accept: foo + } + run: x -> { select: * } + `).malloyResultMatches(runtime, {foo: 1}); + }); + it('composite with each', async () => { + await expect(` + ##! experimental.composite_sources + source: state_facts is ${databaseName}.table('malloytest.state_facts') + source: x is compose( + state_facts extend { measure: foo is sum(0); dimension: bar is 1 }, + state_facts extend { measure: foo is count() } + ) extend { + dimension: arr is [1, 2, 3] + } + run: x -> { aggregate: foo; group_by: bar, arr.each } + `).malloyResultMatches(runtime, {foo: 0}); + }); }); diff --git a/test/src/databases/all/compound-atomic.spec.ts b/test/src/databases/all/compound-atomic.spec.ts new file mode 100644 index 000000000..7e6aa6520 --- /dev/null +++ b/test/src/databases/all/compound-atomic.spec.ts @@ -0,0 +1,538 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import {RuntimeList, allDatabases} from '../../runtimes'; +import {databasesFromEnvironmentOr} from '../../util'; +import '../../util/db-jest-matchers'; +import { + RecordLiteralNode, + ArrayLiteralNode, + ArrayTypeDef, + FieldDef, + Expr, + SQLSourceDef, +} from '@malloydata/malloy'; + +const runtimes = new RuntimeList(databasesFromEnvironmentOr(allDatabases)); + +/* + * Tests for the composite atomic data types "record", "array of values", + * and "array of records". Each starts with a test that the dialect functions + * for literals work, and then bases the rest of the tests on literals, + * so fix that one first if the tests are failing. + */ + +describe.each(runtimes.runtimeList)( + 'compound atomic datatypes %s', + (conName, runtime) => { + const supportsNestedArrays = runtime.dialect.nestedArrays; + const quote = runtime.dialect.sqlMaybeQuoteIdentifier; + function literalNum(num: Number): Expr { + const literal = num.toString(); + return {node: 'numberLiteral', literal, sql: literal}; + } + const empty = `${conName}.sql("SELECT 0 as z")`; + function arraySelectVal(...val: Number[]): string { + const literal: ArrayLiteralNode = { + node: 'arrayLiteral', + typeDef: { + type: 'array', + elementTypeDef: {type: 'number'}, + }, + kids: {values: val.map(v => literalNum(v))}, + }; + return runtime.dialect.sqlLiteralArray(literal); + } + function recordLiteral(fromObj: Record): RecordLiteralNode { + const kids: Record = {}; + const fields: FieldDef[] = Object.keys(fromObj).map(name => { + kids[name] = literalNum(fromObj[name]); + return { + type: 'number', + name, + }; + }); + const literal: RecordLiteralNode = { + node: 'recordLiteral', + typeDef: { + type: 'record', + fields, + }, + kids, + }; + literal.sql = runtime.dialect.sqlLiteralRecord(literal); + return literal; + } + + function recordSelectVal(fromObj: Record): string { + return runtime.dialect.sqlLiteralRecord(recordLiteral(fromObj)); + } + const canReadCompoundSchema = runtime.dialect.compoundObjectInSchema; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const ab = recordSelectVal({a: 0, b: 1}); + + const malloySizes = 'sizes is {s is 0, m is 1, l is 2, xl is 3}'; + const sizesObj = {s: 0, m: 1, l: 2, xl: 3}; + const sizesSQL = recordSelectVal(sizesObj); + // Keeping the pipeline simpler makes debugging easier, so don't add + // and extra stage unless you have to + const sizes = canReadCompoundSchema + ? `${conName}.sql(""" SELECT ${sizesSQL} AS ${quote('sizes')} """)` + : `${conName}.sql('SELECT 0 AS O') -> { select: ${malloySizes}}`; + const evensObj = [2, 4, 6, 8]; + const evensSQL = arraySelectVal(...evensObj); + const evens = `${conName}.sql(""" + SELECT ${evensSQL} AS ${quote('evens')} + """)`; + + describe('simple arrays', () => { + test('array literal dialect function', async () => { + await expect(` + run: ${evens}`).malloyResultMatches(runtime, { + evens: evensObj, + }); + }); + test('select array', async () => { + await expect(` + # test.verbose + run: ${evens}->{select: nn is evens} + `).malloyResultMatches(runtime, {nn: evensObj}); + }); + test.when(canReadCompoundSchema)( + 'schema read allows array-un-nest on each', + async () => { + await expect(` + run: ${evens}->{ select: n is evens.each } + `).malloyResultMatches( + runtime, + evensObj.map(n => ({n})) + ); + } + ); + test('array can be passed to !function', async () => { + // Used as a standin for "unknown function user might call" + const nameOfArrayLenFunction = { + 'duckdb': 'LEN', + 'standardsql': 'ARRAY_LENGTH', + 'postgres': 'JSONB_ARRAY_LENGTH', + 'presto': 'CARDINALITY', + 'trino': 'CARDINALITY', + 'mysql': 'JSON_LENGTH', + 'snowflake': 'ARRAY_SIZE', + }; + const dialect = runtime.dialect.name; + const missing = `Dialect '${dialect}' missing array length function in nameOfArrayLenFunction`; + const fn = nameOfArrayLenFunction[dialect] ?? missing; + expect(fn).not.toEqual(missing); + await expect( + `run: ${evens}->{ select: nby2 is ${fn}!number(evens); } ` + ).malloyResultMatches(runtime, {nby2: evensObj.length}); + }); + test('array.each in source', async () => { + await expect(` + run: ${empty} + extend { dimension: d4 is [1,2,3,4] } + -> { select: die_roll is d4.each } + `).malloyResultMatches(runtime, [ + {die_roll: 1}, + {die_roll: 2}, + {die_roll: 3}, + {die_roll: 4}, + ]); + }); + test('array.each in extend block', async () => { + await expect(` + run: ${empty} -> { + extend: { dimension: d4 is [1,2,3,4] } + select: die_roll is d4.each + } + `).malloyResultMatches(runtime, [ + {die_roll: 1}, + {die_roll: 2}, + {die_roll: 3}, + {die_roll: 4}, + ]); + }); + test.skip('cross join arrays', async () => { + await expect(` + run: ${empty} extend { + dimension: d1 is [1,2,3,4] + join_cross: d2 is [1,2,3,4] + } -> { + group_by: roll is d1.each + d2.each + aggregate: rolls is count() + } + `).malloyResultMatches(runtime, [ + {roll: 2, rolls: 1}, + {roll: 3, rolls: 2}, + {roll: 4, rolls: 3}, + {roll: 5, rolls: 4}, + {roll: 6, rolls: 3}, + {roll: 7, rolls: 2}, + {roll: 8, rolls: 1}, + ]); + }); + // can't use special chars in column names in bq + test.when(conName !== 'bigquery')( + 'array stored field with special chars in name', + async () => { + const special_chars = ["'", '"', '.', '`']; + for (const c of special_chars) { + const qname = '`_\\' + c + '_`'; + const malloySrc = ` + # test.verbose + run: ${empty} + ->{ select: ${qname} is [1]} + -> { select: num is ${qname}.each }`; + await expect(malloySrc).malloyResultMatches(runtime, {}); + const result = await runtime.loadQuery(malloySrc).run(); + const ok = + result.data.path(0, 'num').value === 1 + ? 'ok' + : `Array containing ${c} character is not ok`; + expect(ok).toEqual('ok'); + } + } + ); + test.when(supportsNestedArrays && canReadCompoundSchema)( + 'Can read schema for array of arrays', + async () => { + // a lot of work to make [[1],[2]] on all dialects + const aLit: ArrayLiteralNode = { + node: 'arrayLiteral', + typeDef: {type: 'array', elementTypeDef: {type: 'number'}}, + kids: {values: []}, + }; + const aOne = {...aLit}; + aOne.kids.values[0] = {node: 'numberLiteral', literal: '1', sql: '1'}; + aOne.sql = runtime.dialect.sqlLiteralArray(aOne); + const aTwo = {...aLit, sql: '2'}; + aTwo.kids.values[0] = {node: 'numberLiteral', literal: '2', sql: '2'}; + aTwo.sql = runtime.dialect.sqlLiteralArray(aTwo); + const aoa: ArrayLiteralNode = { + node: 'arrayLiteral', + typeDef: {type: 'array', elementTypeDef: aLit.typeDef}, + kids: {values: [aOne, aTwo]}, + }; + const sql_aoa = runtime.dialect.sqlLiteralArray(aoa); + const asStruct: SQLSourceDef = { + name: 'select_with_aoa', + type: 'sql_select', + connection: conName, + dialect: runtime.dialect.name, + selectStr: `SELECT ${sql_aoa} AS aoa`, + fields: [], + }; + const ret = await runtime.connection.fetchSchemaForSQLStruct( + asStruct, + {} + ); + expect(ret.structDef).toBeDefined(); + const aoa_ent = ret.structDef!.fields[0]; + expect(aoa_ent).toMatchObject(aoa.typeDef); + } + ); + test.when(supportsNestedArrays)('bare array of array', async () => { + await expect(` + run: ${empty} -> { select: aoa is [[1,2]] } + `).malloyResultMatches(runtime, {aoa: [[1, 2]]}); + }); + test.when(supportsNestedArrays)('each.each array of array', async () => { + await expect(` + run: ${empty} extend { dimension: aoa is [[1,2]] } -> { select: aoa.each.each } + `).malloyResultMatches(runtime, [{each: 1}, {each: 2}]); + }); + }); + describe('record', () => { + function rec_eq(as?: string): Record { + const name = as ?? 'sizes'; + return { + [`${name}/s`]: 0, + [`${name}/m`]: 1, + [`${name}/l`]: 2, + [`${name}/xl`]: 3, + }; + } + test('record literal object', async () => { + await expect(` + run: ${conName}.sql("select 0 as o") + -> { select: ${malloySizes}} + `).malloyResultMatches(runtime, rec_eq()); + }); + // can't use special chars in column names in bq + test.when(conName !== 'bigquery')( + 'special character in record property name', + async () => { + const special_chars = ["'", '"', '.', '`']; + for (const c of special_chars) { + const qname = '_\\' + c + '_'; + const name = '_' + c + '_'; + const malloySrc = `run: ${empty} -> { select: \`${qname}\` is 'ok' }`; + // no malloyResultMatches because it treats a special in an expect key + const query = runtime.loadQuery(malloySrc); + const result = await query.run(); + const p = + result.data.path(0, name).value === 'ok' + ? 'ok' + : `Name containing the ${c} character was not ok`; + expect(p).toEqual('ok'); + } + } + ); + // can't use special chars in column names in bq + test.when(conName !== 'bigquery')( + 'record stored in field with special chars in name', + async () => { + const special_chars = ["'", '"', '.', '`']; + for (const c of special_chars) { + const qname = '`_\\' + c + '_`'; + const malloySrc = ` + run: ${empty} + ->{ select: ${qname} is {rnum is 1}} + -> { select: num is ${qname}.rnum }`; + const result = await runtime.loadQuery(malloySrc).run(); + const ok = + result.data.path(0, 'num').value === 1 + ? 'ok' + : `Array containing ${c} character is not ok`; + expect(ok).toEqual('ok'); + } + } + ); + test.when(canReadCompoundSchema)( + 'can read schema of record object', + async () => { + await expect(`run: ${conName}.sql(""" + SELECT ${sizesSQL} AS ${quote('sizes')} + """)`).malloyResultMatches(runtime, rec_eq()); + } + ); + test('simple record.property access', async () => { + await expect(` + run: ${sizes} -> { select: small is sizes.s }`).malloyResultMatches( + runtime, + {small: 0} + ); + }); + test('nested data looks like a record', async () => { + await expect(` + run: ${conName}.sql('SELECT 1 as ${quote('o')}') -> { + group_by: row is 'one_row' + nest: sizes is { + aggregate: + s is sum(o) - 1, + m is sum(o), + x is sum(o) + 1, + xl is sum(o) + 2 + } + } -> { select: small is sizes.s }`).malloyResultMatches(runtime, { + small: 0, + }); + }); + test('record can be selected', async () => { + await expect( + ` + run: ${sizes} -> { select: sizes }` + ).malloyResultMatches(runtime, rec_eq()); + }); + test('record literal can be selected', async () => { + await expect(` + run: ${sizes} -> { select: record is sizes } + `).malloyResultMatches(runtime, rec_eq('record')); + }); + test('select record literal from a source', async () => { + await expect(` + run: ${empty} -> { + extend: { dimension: ${malloySizes} } + select: sizes + } + `).malloyResultMatches(runtime, rec_eq()); + }); + test('computed record.property from a source', async () => { + await expect(` + run: ${empty} + extend { dimension: record is {s is 0, m is 1, l is 2, xl is 3} } + -> { select: small is record.s } + `).malloyResultMatches(runtime, {small: 0}); + }); + test('record.property from an extend block', async () => { + await expect(` + run: ${empty} -> { + extend: { dimension: record is {s is 0, m is 1, l is 2, xl is 3} } + select: small is record.s + } + `).malloyResultMatches(runtime, {small: 0}); + }); + test('simple each on array property inside record', async () => { + await expect(` + run: ${empty} -> { select: nums is { odds is [1,3], evens is [2,4]} } + -> { select: odd is nums.odds.value } + `).malloyResultMatches(runtime, [{odd: 1}, {odd: 3}]); + }); + test('each on array property inside record from source', async () => { + await expect(` + run: ${empty} extend { dimension: nums is { odds is [1,3], evens is [2,4]} } + -> { select: odd is nums.odds.each } + `).malloyResultMatches(runtime, [{odd: 1}, {odd: 3}]); + }); + const abc = "rec is {a is 'a', bc is {b is 'b', c is 'c'}}"; + test('record with a record property', async () => { + await expect(` + run: ${empty} -> { select: ${abc} } + -> { select: rec.a, rec.bc.b, rec.bc.c } + `).malloyResultMatches(runtime, {a: 'a', b: 'b', c: 'c'}); + }); + test('record in source with a record property', async () => { + await expect(` + run: ${empty} extend { dimension: ${abc} } + -> { select: rec.a, rec.bc.b, rec.bc.c } + `).malloyResultMatches(runtime, {a: 'a', b: 'b', c: 'c'}); + }); + test('record dref in source with a record property', async () => { + await expect(` + run: ${empty} extend { dimension: ${abc} } + -> { select: b is pick rec.bc.b when true else 'b' } + `).malloyResultMatches(runtime, {b: 'b'}); + }); + test.todo('array or record where first entries are null'); + }); + describe('repeated record', () => { + const abType: ArrayTypeDef = { + type: 'array', + elementTypeDef: {type: 'record_element'}, + fields: [ + {name: 'a', type: 'number'}, + {name: 'b', type: 'number'}, + ], + }; + const values = [ + recordLiteral({a: 10, b: 11}), + recordLiteral({a: 20, b: 21}), + ]; + + const ab = runtime.dialect.sqlLiteralArray({ + node: 'arrayLiteral', + typeDef: abType, + kids: {values}, + }); + const ab_eq = [ + {a: 10, b: 11}, + {a: 20, b: 21}, + ]; + const abMalloy = '[{a is 10, b is 11}, {a is 20, b is 21}]'; + function selectAB(n: string) { + return `SELECT ${ab} AS ${quote(n)}`; + } + + test('repeated record from nest', async () => { + await expect(` + run: ${conName}.sql(""" + SELECT + 10 as ${quote('a')}, + 11 as ${quote('b')} + UNION ALL SELECT 20 , 21 + """) -> { nest: ab is { select: a, b } } + -> { select: ab.a, ab.b ; order_by: a} + `).malloyResultMatches(runtime, ab_eq); + }); + test('select repeated record from literal dialect functions', async () => { + await expect(` + run: ${conName}.sql(""" ${selectAB('ab')} """) + `).malloyResultMatches(runtime, {ab: ab_eq}); + }); + test('repeat record from malloy literal', async () => { + await expect(` + run: ${empty} + -> { select: ab is ${abMalloy} } + `).malloyResultMatches(runtime, {ab: ab_eq}); + }); + test('repeated record can be selected and renamed', async () => { + const src = ` + run: ${conName}.sql(""" + ${selectAB('sqlAB')} + """) -> { select: ab is sqlAB } + `; + await expect(src).malloyResultMatches(runtime, {ab: ab_eq}); + }); + test('select repeated record passed down pipeline', async () => { + await expect(` + run: ${empty} + -> { select: pipeAb is ${abMalloy} } + -> { select: ab is pipeAb } + `).malloyResultMatches(runtime, {ab: ab_eq}); + }); + test('deref repeat record passed down pipeline', async () => { + await expect(` + run: ${empty} + -> { select: pipeAb is ${abMalloy} } + -> { select: pipeAb.a, pipeAb.b } + `).malloyResultMatches(runtime, ab_eq); + }); + test('select array of records from source', async () => { + await expect(` + run: ${empty} + extend { dimension: abSrc is ${abMalloy} } + -> { select: ab is abSrc } + `).malloyResultMatches(runtime, {ab: ab_eq}); + }); + test('deref array of records from source', async () => { + await expect(` + run: ${empty} + extend { dimension: ab is ${abMalloy} } + -> { select: ab.a, ab.b } + `).malloyResultMatches(runtime, ab_eq); + }); + test('repeated record in source wth record property', async () => { + await expect(` + run: ${empty} extend { dimension: rec is [ {bc is {b is 'b'}} ] } + -> { select: rec.bc.b } + `).malloyResultMatches(runtime, {b: 'b'}); + }); + test('piped repeated record containing an array', async () => { + await expect(` + run: ${empty} -> { + select: rrec is [ + { val is 1, names is ['uno', 'one'] }, + { val is 2, names is ['due', 'two'] } + ] + } -> { + select: val is rrec.val, name is rrec.names.each + order_by: val desc, name asc + } + `).malloyResultMatches(runtime, [ + {val: 2, name: 'due'}, + {val: 2, name: 'two'}, + {val: 1, name: 'one'}, + {val: 1, name: 'uno'}, + ]); + }); + test('source repeated record containing an array', async () => { + await expect(` + run: ${empty} extend { + dimension: rrec is [ + { val is 1, names is ['uno', 'one'] }, + { val is 2, names is ['due', 'two'] } + ] + } -> { + select: val is rrec.val, name is rrec.names.each + order_by: val desc, name asc + } + `).malloyResultMatches(runtime, [ + {val: 2, name: 'due'}, + {val: 2, name: 'two'}, + {val: 1, name: 'one'}, + {val: 1, name: 'uno'}, + ]); + }); + }); + } +); + +afterAll(async () => { + await runtimes.closeAll(); +}); diff --git a/test/src/databases/all/nomodel.spec.ts b/test/src/databases/all/nomodel.spec.ts index d74ead8de..07183384e 100644 --- a/test/src/databases/all/nomodel.spec.ts +++ b/test/src/databases/all/nomodel.spec.ts @@ -1167,6 +1167,85 @@ SELECT row_to_json(finalStage) as row FROM __stage0 AS finalStage`); } ); + test.when( + runtime.supportsNesting && runtime.dialect.supportsPipelinesInViews + )(`Nested pipelines sort properly - ${databaseName}`, async () => { + const doTrace = false; // Have to turn this on to debug this test + const result = await runtime + .loadQuery( + ` + source: state_facts is ${databaseName}.table('malloytest.state_facts') + extend { + view: base_view is { + group_by: state + aggregate: airports is sum(airport_count) + order_by: airports asc + } + -> + { + group_by: state + aggregate: airports.sum() + order_by: airports + } + view: base_view2 is { + group_by: state + aggregate: aircrafts is sum(aircraft_count) + order_by: aircrafts asc + } + -> + { + group_by: state + aggregate: aircrafts.sum() + order_by: aircrafts + } + view: base_view3 is { + group_by: state + aggregate: aircrafts is sum(aircraft_count) + } + -> { + group_by: state + aggregate: aircrafts.sum() + } + view: sort_issue is { + where: popular_name ~ r'I' + group_by: popular_name + nest: base_view + nest: base_view2 + nest: base_view3 + } + } + run: state_facts -> sort_issue + ` + ) + .run(); + if (doTrace) console.log(result.sql); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const d: any = result.data.toObject(); + const baseView: {state: string; airports: number}[] = d[0]['base_view']; + if (doTrace) console.log(baseView); + let baseMax = baseView[0]; + for (const b of baseView) { + expect(b.airports).toBeGreaterThanOrEqual(baseMax.airports); + baseMax = b; + } + + const baseView2: {state: string; aircrafts: number}[] = d[0]['base_view2']; + if (doTrace) console.log(baseView2); + let baseMax2 = baseView2[0]; + for (const b of baseView2) { + expect(b.aircrafts).toBeGreaterThanOrEqual(baseMax2.aircrafts); + baseMax2 = b; + } + // implicit order by + const baseView3: {state: string; aircrafts: number}[] = d[0]['base_view3']; + if (doTrace) console.log(baseView3); + let baseMax3 = baseView3[0]; + for (const b of baseView3) { + expect(b.aircrafts).toBeLessThanOrEqual(baseMax3.aircrafts); + baseMax3 = b; + } + }); + test.when(runtime.supportsNesting)( 'number as null- ${databaseName}', async () => { diff --git a/test/src/databases/duckdb/duckdb.spec.ts b/test/src/databases/duckdb/duckdb.spec.ts index 847d00c87..ec50d28a5 100644 --- a/test/src/databases/duckdb/duckdb.spec.ts +++ b/test/src/databases/duckdb/duckdb.spec.ts @@ -131,7 +131,7 @@ describe.each(allDucks.runtimeList)('duckdb:%s', (dbName, runtime) => { ).malloyResultMatches(runtime, {abc: 'a', abc3: 'a3'}); }); - describe('time', () => { + describe('time oddities', () => { const zone = 'America/Mexico_City'; // -06:00 no DST const zone_2020 = DateTime.fromObject( { @@ -147,13 +147,12 @@ describe.each(allDucks.runtimeList)('duckdb:%s', (dbName, runtime) => { } ); test('can cast TIMESTAMPTZ to timestamp', async () => { - await expect( - `run: duckdb.sql(""" + await expect(` + run: duckdb.sql(""" SELECT TIMESTAMPTZ '2020-02-20 00:00:00 ${zone}' as t_tstz """) -> { select: mex_220 is t_tstz::timestamp - }` - ).malloyResultMatches(runtime, {mex_220: zone_2020.toJSDate()}); + }`).malloyResultMatches(runtime, {mex_220: zone_2020.toJSDate()}); }); }); }); diff --git a/test/src/databases/duckdb/materialization.spec.ts b/test/src/databases/duckdb/materialization.spec.ts index d4434d03f..96b793648 100644 --- a/test/src/databases/duckdb/materialization.spec.ts +++ b/test/src/databases/duckdb/materialization.spec.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {RuntimeList} from '../../runtimes'; diff --git a/test/src/databases/duckdb/reference-id.spec.ts b/test/src/databases/duckdb/reference-id.spec.ts index 49dd020ac..fae349a13 100644 --- a/test/src/databases/duckdb/reference-id.spec.ts +++ b/test/src/databases/duckdb/reference-id.spec.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {RuntimeList} from '../../runtimes'; diff --git a/test/src/databases/presto-trino/presto-trino.spec.ts b/test/src/databases/presto-trino/presto-trino.spec.ts index 4f5fc821c..f2f908b77 100644 --- a/test/src/databases/presto-trino/presto-trino.spec.ts +++ b/test/src/databases/presto-trino/presto-trino.spec.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ /* eslint-disable no-console */ @@ -22,6 +22,18 @@ describe.each(runtimes.runtimeList)( throw new Error("Couldn't build runtime"); } + test.when(databaseName === 'presto')('presto explain parser', async () => { + const abrec = 'CAST(ROW(0,1) AS ROW(a DOUBLE,b DOUBLE))'; + await expect(` + run: ${databaseName}.sql(""" + SELECT + ${abrec} as "abrec", + ARRAY['c', 'd'] as str_array, + array[1,2,3] as int_array, + ARRAY[${abrec}] as array_of_abrec + """) + `).malloyResultMatches(runtime, {}); + }); it(`runs an sql query - ${databaseName}`, async () => { await expect( `run: ${databaseName}.sql("SELECT 1 as n") -> { select: n }` @@ -272,6 +284,35 @@ describe.each(runtimes.runtimeList)( {x: 1, pctrnk: 0}, ]); }); + + it(`runs the url_extract functions - ${databaseName}`, async () => { + await expect(` + run: ${databaseName}.sql( + """ + SELECT 'http://websitetesthost.com:80/path_comp/my_test?first_param=val_one&second_param=2#example_frag' as test_url + """ + ) -> { + select: + fragment is url_extract_fragment(test_url) + host is url_extract_host(test_url) + param_one is url_extract_parameter(test_url, 'first_param') + param_two is url_extract_parameter(test_url, 'second_param') + path is url_extract_path(test_url) + port is url_extract_port(test_url) + protocol is url_extract_protocol(test_url) + query is url_extract_query(test_url) + } + `).malloyResultMatches(runtime, { + fragment: 'example_frag', + host: 'websitetesthost.com', + param_one: 'val_one', + param_two: '2', + path: '/path_comp/my_test', + port: 80, + protocol: 'http', + query: 'first_param=val_one&second_param=2', + }); + }); } ); diff --git a/test/src/events.spec.ts b/test/src/events.spec.ts index b68986538..b422e1ccf 100644 --- a/test/src/events.spec.ts +++ b/test/src/events.spec.ts @@ -2,7 +2,7 @@ * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. + * LICENSE file in the root directory of this source tree. */ import {runtimeFor} from './runtimes'; diff --git a/test/src/util/db-jest-matchers.ts b/test/src/util/db-jest-matchers.ts index 6c35def9b..bc2da23ca 100644 --- a/test/src/util/db-jest-matchers.ts +++ b/test/src/util/db-jest-matchers.ts @@ -203,7 +203,7 @@ expect.extend({ const actuallyGot = got instanceof Date ? got.getTime() : got; if (typeof mustBe === 'number' && typeof actuallyGot !== 'number') { fails.push(`${expected} Got: Non Numeric '${pGot}'`); - } else if (actuallyGot !== mustBe) { + } else if (!objectsMatch(actuallyGot, mustBe)) { fails.push(`${expected} Got: ${pGot}`); } } catch (e) { @@ -332,8 +332,8 @@ function humanReadable(thing: unknown): string { // b is "expected" // a is "actual" -// If expected is an object, all of the keys should also -// match, buy the expected is allowed to have other keys that are not matched +// If expected is an object, all of the keys should also match, +// but the expected is allowed to have other keys that are not matched function objectsMatch(a: unknown, b: unknown): boolean { if ( typeof b === 'string' ||