From 04fb520f97f90c26b79e07dce85b8d3cb7dd296e Mon Sep 17 00:00:00 2001 From: Christopher Swenson Date: Tue, 19 Sep 2023 13:44:39 -0500 Subject: [PATCH] add experimental sql_functions to Malloy * add sql_* functions that allow for general sql text to be used as expressions which return a specific type. * these sql_* functions will also interpolate special ${...} lookml syntax such as ${TABLE}, ${dimension}, etc * add experimental Malloy flag 'sql_functions' that must be enabled for your Malloy source to use this feature. --- .../src/component/table-layout.ts | 2 +- .../src/stories/tables.stories.ts | 2 +- .../src/stories/themes.stories.ts | 6 +- .../src/dialect/functions/all_functions.ts | 13 ++ packages/malloy/src/dialect/functions/sql.ts | 86 +++++++++ .../expressions/expr-aggregate-function.ts | 1 + .../src/lang/ast/expressions/expr-func.ts | 99 ++++++++++- packages/malloy/src/model/malloy_query.ts | 50 ++++++ packages/malloy/src/model/malloy_types.ts | 31 ++++ packages/malloy/src/model/utils.ts | 10 ++ test/src/databases/all/expr.spec.ts | 164 ++++++++++++++++++ 11 files changed, 458 insertions(+), 6 deletions(-) create mode 100644 packages/malloy/src/dialect/functions/sql.ts diff --git a/packages/malloy-render/src/component/table-layout.ts b/packages/malloy-render/src/component/table-layout.ts index 1ffc38beea..a4584bcd2d 100644 --- a/packages/malloy-render/src/component/table-layout.ts +++ b/packages/malloy-render/src/component/table-layout.ts @@ -77,7 +77,7 @@ function getColumnWidth(f: Field, metadata: RenderResultMetadata) { let width = 0; if (f.isAtomicField()) { // TODO: get font styles from theme - const font = `12px Inter, sans-serif`; + const font = '12px Inter, sans-serif'; const titleWidth = getTextWidth(f.name, font); if (f.isAtomicField() && f.isString()) { width = diff --git a/packages/malloy-render/src/stories/tables.stories.ts b/packages/malloy-render/src/stories/tables.stories.ts index 833166cc0c..6cfb8ff00f 100644 --- a/packages/malloy-render/src/stories/tables.stories.ts +++ b/packages/malloy-render/src/stories/tables.stories.ts @@ -25,7 +25,7 @@ export default meta; export const ProductsTable = { args: { source: 'products', - view: `records`, + view: 'records', }, }; diff --git a/packages/malloy-render/src/stories/themes.stories.ts b/packages/malloy-render/src/stories/themes.stories.ts index b9f6f26c85..97e56273b8 100644 --- a/packages/malloy-render/src/stories/themes.stories.ts +++ b/packages/malloy-render/src/stories/themes.stories.ts @@ -25,21 +25,21 @@ export default meta; export const ModelThemeOverride = { args: { source: 'products', - view: `records`, + view: 'records', }, }; export const ViewThemeOverride = { args: { source: 'products', - view: `records_override`, + view: 'records_override', }, }; export const ViewThemeOverrideCSS = { args: { source: 'products', - view: `records_override`, + view: 'records_override', classes: 'night', }, }; diff --git a/packages/malloy/src/dialect/functions/all_functions.ts b/packages/malloy/src/dialect/functions/all_functions.ts index e81e637cdc..f94f76d3ae 100644 --- a/packages/malloy/src/dialect/functions/all_functions.ts +++ b/packages/malloy/src/dialect/functions/all_functions.ts @@ -79,6 +79,13 @@ import { import {fnAvgRolling} from './avg_moving'; import {FunctionMap} from './function_map'; import {fnCoalesce} from './coalesce'; +import { + fnSqlBoolean, + fnSqlDate, + fnSqlNumber, + fnSqlString, + fnSqlTimestamp, +} from './sql'; /** * This is a function map containing default implementations of all Malloy @@ -155,4 +162,10 @@ FUNCTIONS.add('max_window', fnMaxWindow); FUNCTIONS.add('sum_window', fnSumWindow); FUNCTIONS.add('avg_moving', fnAvgRolling); +FUNCTIONS.add('sql_number', fnSqlNumber); +FUNCTIONS.add('sql_string', fnSqlString); +FUNCTIONS.add('sql_date', fnSqlDate); +FUNCTIONS.add('sql_timestamp', fnSqlTimestamp); +FUNCTIONS.add('sql_boolean', fnSqlBoolean); + FUNCTIONS.seal(); diff --git a/packages/malloy/src/dialect/functions/sql.ts b/packages/malloy/src/dialect/functions/sql.ts new file mode 100644 index 0000000000..3bbb4c52da --- /dev/null +++ b/packages/malloy/src/dialect/functions/sql.ts @@ -0,0 +1,86 @@ +/* + * Copyright 2023 Google LLC + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +import { + overload, + minScalar, + DialectFunctionOverloadDef, + makeParam, + maxScalar, + literal, +} from './util'; + +export function fnSqlNumber(): DialectFunctionOverloadDef[] { + const value = makeParam('value', literal(maxScalar('string'))); + return [ + overload( + minScalar('number'), + [value.param], + [{type: 'sql-string', e: [value.arg]}] + ), + ]; +} + +export function fnSqlString(): DialectFunctionOverloadDef[] { + const value = makeParam('value', literal(maxScalar('string'))); + return [ + overload( + minScalar('string'), + [value.param], + [{type: 'sql-string', e: [value.arg]}] + ), + ]; +} + +export function fnSqlDate(): DialectFunctionOverloadDef[] { + const value = makeParam('value', literal(maxScalar('string'))); + return [ + overload( + minScalar('date'), + [value.param], + [{type: 'sql-string', e: [value.arg]}] + ), + ]; +} + +export function fnSqlTimestamp(): DialectFunctionOverloadDef[] { + const value = makeParam('value', literal(maxScalar('string'))); + return [ + overload( + minScalar('timestamp'), + [value.param], + [{type: 'sql-string', e: [value.arg]}] + ), + ]; +} + +export function fnSqlBoolean(): DialectFunctionOverloadDef[] { + const value = makeParam('value', literal(maxScalar('string'))); + return [ + overload( + minScalar('boolean'), + [value.param], + [{type: 'sql-string', e: [value.arg]}] + ), + ]; +} diff --git a/packages/malloy/src/lang/ast/expressions/expr-aggregate-function.ts b/packages/malloy/src/lang/ast/expressions/expr-aggregate-function.ts index f9c945c63b..17086f16a5 100644 --- a/packages/malloy/src/lang/ast/expressions/expr-aggregate-function.ts +++ b/packages/malloy/src/lang/ast/expressions/expr-aggregate-function.ts @@ -261,6 +261,7 @@ function getJoinUsage( }; exprWalk(expr, frag => { if (typeof frag !== 'string') { + // TODO make this work for field references inside sql_* functions if (frag.type === 'field') { const def = lookup(fs, frag.path); if (def.def.type !== 'struct' && def.def.type !== 'turtle') { diff --git a/packages/malloy/src/lang/ast/expressions/expr-func.ts b/packages/malloy/src/lang/ast/expressions/expr-func.ts index 95e9876cbd..a2b997c6f5 100644 --- a/packages/malloy/src/lang/ast/expressions/expr-func.ts +++ b/packages/malloy/src/lang/ast/expressions/expr-func.ts @@ -207,7 +207,7 @@ export class ExprFunc extends ExpressionDef { ); return errorFor('cannot call with source'); } - const funcCall: Expr = [ + let funcCall: Expr = [ { type: 'function_call', overload, @@ -216,6 +216,72 @@ export class ExprFunc extends ExpressionDef { structPath, }, ]; + if ( + [ + 'sql_number', + 'sql_string', + 'sql_date', + 'sql_timestamp', + 'sql_boolean', + ].includes(func.name) + ) { + if (!this.inExperiment('sql_functions', true)) { + return errorFor( + `Cannot use sql_function \`${func.name}\`; use \`sql_functions\` experiment to enable this behavior` + ); + } + + const str = argExprs[0].value; + if ( + str.length !== 1 || + typeof str[0] === 'string' || + str[0].type !== 'dialect' || + str[0].function !== 'stringLiteral' + ) { + this.log(`Invalid string literal for \`${func.name}\``); + } else { + const literal = str[0].literal; + const parts = parseSQLInterpolation(literal); + const unsupportedInterpolations = parts + .filter( + part => part.type === 'interpolation' && part.name.includes('.') + ) + .map(unsupportedPart => + unsupportedPart.type === 'interpolation' + ? `\${${unsupportedPart.name}}` + : `\${${unsupportedPart.value}}` + ); + + if (unsupportedInterpolations.length > 0) { + const unsupportedInterpolationMsg = + unsupportedInterpolations.length === 1 + ? `'.' paths are not yet supported in sql interpolations, found ${unsupportedInterpolations.at( + 0 + )}` + : `'.' paths are not yet supported in sql interpolations, found [${unsupportedInterpolations.join( + ', ' + )}]`; + this.log(unsupportedInterpolationMsg); + + return errorFor( + `${unsupportedInterpolationMsg}. See LookML \${...} documentation at https://cloud.google.com/looker/docs/reference/param-field-sql#sql_for_dimensions` + ); + } + + funcCall = [ + { + type: 'sql-string', + e: parts.map(part => + part.type === 'string' + ? part.value + : part.name === 'TABLE' + ? {type: 'source-reference'} + : {type: 'field-reference', path: part.name} + ), + }, + ]; + } + } if (type.dataType === 'any') { this.log( `Invalid return type ${type.dataType} for function '${this.name}'` @@ -369,3 +435,34 @@ function findOverload( } } } + +type InterpolationPart = + | {type: 'string'; value: string} + | {type: 'interpolation'; name: string}; + +function parseSQLInterpolation(template: string): InterpolationPart[] { + const parts: InterpolationPart[] = []; + let remaining = template; + while (remaining.length) { + const nextInterp = remaining.indexOf('${'); + if (nextInterp === -1) { + parts.push({type: 'string', value: remaining}); + break; + } else { + const interpEnd = remaining.slice(nextInterp).indexOf('}'); + if (interpEnd === -1) { + parts.push({type: 'string', value: remaining}); + break; + } + if (nextInterp > 0) { + parts.push({type: 'string', value: remaining.slice(0, nextInterp)}); + } + parts.push({ + type: 'interpolation', + name: remaining.slice(nextInterp + 2, interpEnd + nextInterp), + }); + remaining = remaining.slice(interpEnd + nextInterp + 1); + } + } + return parts; +} diff --git a/packages/malloy/src/model/malloy_query.ts b/packages/malloy/src/model/malloy_query.ts index d836ec3a71..82b7f263ba 100644 --- a/packages/malloy/src/model/malloy_query.ts +++ b/packages/malloy/src/model/malloy_query.ts @@ -37,6 +37,7 @@ import { FieldDef, FieldFragment, FieldRef, + FieldReferenceFragment, FieldTimestampDef, Filtered, FilterExpression, @@ -81,8 +82,10 @@ import { ResultMetadataDef, ResultStructMetadataDef, SearchIndexResult, + SourceReferenceFragment, SpreadFragment, SQLExpressionFragment, + SqlStringFragment, StructDef, StructRef, TurtleDef, @@ -782,6 +785,47 @@ class QueryField extends QueryNode { ); } + generateFieldReference( + resultSet: FieldInstanceResult, + context: QueryStruct, + expr: FieldReferenceFragment, + state: GenerateState + ): string { + return this.generateFieldFragment( + resultSet, + context, + {type: 'field', path: expr.path}, + state + ); + } + + generateSqlString( + resultSet: FieldInstanceResult, + context: QueryStruct, + expr: SqlStringFragment, + state: GenerateState + ): string { + return expr.e + .map(part => + typeof part === 'string' + ? part + : this.generateExpressionFromExpr(resultSet, context, [part], state) + ) + .join(''); + } + + generateSourceReference( + resultSet: FieldInstanceResult, + context: QueryStruct, + expr: SourceReferenceFragment + ): string { + if (expr.path === undefined) { + return context.getSQLIdentifier(); + } else { + return context.getFieldByName(expr.path).getIdentifier(); + } + } + getAnalyticPartitions(resultStruct: FieldInstanceResult) { const ret: string[] = []; let p = resultStruct.parent; @@ -980,6 +1024,12 @@ class QueryField extends QueryNode { s += this.generateSpread(resultSet, context, expr, state); } else if (expr.type === 'dialect') { s += this.generateDialect(resultSet, context, expr, state); + } else if (expr.type === 'sql-string') { + s += this.generateSqlString(resultSet, context, expr, state); + } else if (expr.type === 'source-reference') { + s += this.generateSourceReference(resultSet, context, expr); + } else if (expr.type === 'field-reference') { + s += this.generateFieldReference(resultSet, context, expr, state); } else { throw new Error( `Internal Error: Unknown expression fragment ${JSON.stringify( diff --git a/packages/malloy/src/model/malloy_types.ts b/packages/malloy/src/model/malloy_types.ts index ac1dab4cce..10a9947e8e 100644 --- a/packages/malloy/src/model/malloy_types.ts +++ b/packages/malloy/src/model/malloy_types.ts @@ -261,6 +261,34 @@ export function isFieldFragment(f: Fragment): f is FieldFragment { return (f as FieldFragment)?.type === 'field'; } +export interface FieldReferenceFragment { + type: 'field-reference'; + path: string; +} +export function isFieldReferenceFragment( + f: Fragment +): f is FieldReferenceFragment { + return (f as FieldReferenceFragment)?.type === 'field-reference'; +} + +export interface SqlStringFragment { + type: 'sql-string'; + e: Expr; +} +export function isSqlStringFragment(f: Fragment): f is SqlStringFragment { + return (f as SqlStringFragment)?.type === 'sql-string'; +} + +export interface SourceReferenceFragment { + type: 'source-reference'; + path?: string; +} +export function isSourceReferenceFragment( + f: Fragment +): f is SourceReferenceFragment { + return (f as SourceReferenceFragment)?.type === 'source-reference'; +} + export interface ParameterFragment { type: 'parameter'; path: string; @@ -382,6 +410,9 @@ export type Fragment = | ApplyFragment | ApplyValueFragment | FieldFragment + | FieldReferenceFragment + | SourceReferenceFragment + | SqlStringFragment | ParameterFragment | FilterFragment | OutputFieldFragment diff --git a/packages/malloy/src/model/utils.ts b/packages/malloy/src/model/utils.ts index 09bae46f8b..4cbafe5ad4 100644 --- a/packages/malloy/src/model/utils.ts +++ b/packages/malloy/src/model/utils.ts @@ -134,6 +134,11 @@ export function exprMap(expr: Expr, func: (fragment: Fragment) => Expr): Expr { case 'parameter': case 'outputField': return fragment; + case 'sql-string': + return { + ...fragment, + e: exprMap(fragment.e, func), + }; case 'function_call': return { ...fragment, @@ -240,6 +245,11 @@ export function exprWalk(expr: Expr, func: (fragment: Fragment) => void): void { ...fragment, args: fragment.args.map(arg => exprWalk(arg, func)), }; + case 'sql-string': + return { + ...fragment, + e: exprWalk(fragment.e, func), + }; case 'filterExpression': return { ...fragment, diff --git a/test/src/databases/all/expr.spec.ts b/test/src/databases/all/expr.spec.ts index 1e26967c4e..6e9eaf95a6 100644 --- a/test/src/databases/all/expr.spec.ts +++ b/test/src/databases/all/expr.spec.ts @@ -340,6 +340,170 @@ describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => { }); }); + describe('sql expr functions', () => { + it('sql_string', async () => { + await expect(` + ##! experimental { sql_functions } + source: a is ${databaseName}.table('malloytest.aircraft_models') extend { where: aircraft_model_code ? '0270202' } + + run: a -> { + group_by: string_1 is sql_string("UPPER(\${manufacturer})") + } + `).malloyResultMatches(expressionModel, { + string_1: 'AHRENS AIRCRAFT CORP.', + }); + }); + + it('sql_number', async () => { + await expect(` + ##! experimental { sql_functions } + source: a is ${databaseName}.table('malloytest.aircraft_models') extend { where: aircraft_model_code ? '0270202' } + + run: a -> { + group_by: seats + group_by: number_1 is sql_number("\${seats} * 2") + } + `).malloyResultMatches(expressionModel, { + seats: 29, + number_1: 58, + }); + }); + + it('sql_boolean', async () => { + await expect(` + ##! experimental { sql_functions } + source: a is ${databaseName}.table('malloytest.aircraft_models') extend { where: aircraft_model_code ? '0270202' } + + run: a -> { + group_by: boolean_1 is sql_boolean("\${seats} > 20") + group_by: boolean_2 is sql_boolean("\${engines} = 2") + } + `).malloyResultMatches(expressionModel, { + boolean_1: true, + boolean_2: false, + }); + }); + + it('sql_date', async () => { + await expect(` + ##! experimental { sql_functions } + source: a is ${databaseName}.table('malloytest.aircraft') extend { where: tail_num ? 'N110WL' } + + run: a -> { + group_by: date_1 is sql_date("\${last_action_date}") + } + `).malloyResultMatches(expressionModel, { + date_1: new Date('2000-01-04T00:00:00.000Z'), + }); + }); + + it('sql_timestamp', async () => { + await expect(` + ##! experimental { sql_functions } + source: a is ${databaseName}.table('malloytest.aircraft') extend { where: tail_num ? 'N110WL' } + + run: a -> { + group_by: timestamp_1 is sql_timestamp("\${last_action_date}") + } + `).malloyResultMatches(expressionModel, { + timestamp_1: new Date('2000-01-04T00:00:00.000Z'), + }); + }); + + it('with ${TABLE}.field', async () => { + await expect(` + ##! experimental { sql_functions } + source: a is ${databaseName}.table('malloytest.aircraft_models') extend { where: aircraft_model_code ? '0270202' } + + run: a -> { + group_by: string_1 is sql_string("UPPER(\${TABLE}.manufacturer)") + } + `).malloyResultMatches(expressionModel, { + string_1: 'AHRENS AIRCRAFT CORP.', + }); + }); + + it('with ${field}', async () => { + await expect(` + ##! experimental { sql_functions } + source: a is ${databaseName}.table('malloytest.aircraft_models') extend { where: aircraft_model_code ? '0270202' } + + run: a -> { + group_by: string_1 is sql_string("UPPER(\${manufacturer})") + } + `).malloyResultMatches(expressionModel, { + string_1: 'AHRENS AIRCRAFT CORP.', + }); + }); + + it('sql_functions - experimental flag check', async () => { + const query = await expressionModel.loadQuery( + ` + source: a is ${databaseName}.table('malloytest.aircraft_models') extend { where: aircraft_model_code ? '0270202' } + + run: a -> { + group_by: string_1 is sql_string("UPPER(\${manufacturer})") + } + ` + ); + await expect(query.run()).rejects.toThrow( + "Experimental flag 'sql_functions' is not set, feature not available" + ); + }); + + describe('[not yet supported]', () => { + // See ${...} documentation for lookml here for guidance on remaining work: + // https://cloud.google.com/looker/docs/reference/param-field-sql#sql_for_dimensions + it('${view_name.dimension_name} - one path', async () => { + const query = await expressionModel.loadQuery( + ` + ##! experimental { sql_functions } + source: a is ${databaseName}.table('malloytest.aircraft_models') extend { where: aircraft_model_code ? '0270202' } + + run: a -> { + group_by: string_1 is sql_string("UPPER(\${a.manufacturer})") + } + ` + ); + await expect(query.run()).rejects.toThrow( + "'.' paths are not yet supported in sql interpolations, found ${a.manufacturer}" + ); + }); + + it('${view_name.dimension_name} - multiple paths', async () => { + const query = await expressionModel.loadQuery( + ` + ##! experimental { sql_functions } + source: a is ${databaseName}.table('malloytest.aircraft_models') extend { where: aircraft_model_code ? '0270202' } + + run: a -> { + group_by: number_1 is sql_number("\${a.seats} * \${a.seats} + \${a.total_seats}") + } + ` + ); + await expect(query.run()).rejects.toThrow( + "'.' paths are not yet supported in sql interpolations, found [${a.seats}, ${a.seats}, ${a.total_seats}]" + ); + }); + + it('${view_name.SQL_TABLE_NAME}', async () => { + const query = await expressionModel.loadQuery( + ` + ##! experimental { sql_functions } + source: a is ${databaseName}.table('malloytest.aircraft_models') extend { where: aircraft_model_code ? '0270202' } + + run: a -> { + group_by: number_1 is sql_number("\${a.SQL_TABLE_NAME}.seats") + } + ` + ); + await expect(query.run()).rejects.toThrow( + "'.' paths are not yet supported in sql interpolations, found ${a.SQL_TABLE_NAME}" + ); + }); + }); + }); + testIf(runtime.supportsNesting)( 'query with aliasname used twice', async () => {