From 3e9b0d605bd02e2d6c08134a9c6ede8b3445de09 Mon Sep 17 00:00:00 2001 From: lloyd tabb Date: Tue, 2 Apr 2024 04:07:21 -0700 Subject: [PATCH 1/3] start of convert --- .../malloy-db-trino/src/trino_connection.ts | 18 ++++++++++++++++-- test/src/databases/all/expr.spec.ts | 4 ++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/malloy-db-trino/src/trino_connection.ts b/packages/malloy-db-trino/src/trino_connection.ts index d4dd5c1af..97c7031d7 100644 --- a/packages/malloy-db-trino/src/trino_connection.ts +++ b/packages/malloy-db-trino/src/trino_connection.ts @@ -250,6 +250,20 @@ export class TrinoConnection implements Connection, PersistSQLResults { } }*/ + convertNest(column: string, dataRows: unknown[][]) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const ret: any[] = []; + for (const row of dataRows) { + const col = {}; + for (const element of row) { + const e: string[] = element as string[]; + col[e[0] as string] = e[1]; + } + ret.push(col); + } + return ret; + } + public async runSQL( sqlCommand: string, options: RunSQLOptions = {}, @@ -276,8 +290,8 @@ export class TrinoConnection implements Connection, PersistSQLResults { for (let i = 0; i < queryResult.value.columns.length; i++) { const column = queryResult.value.columns[i]; // TODO: handle arrays etc. - if (column.type === 'json') { - malloyRow[column.name] = JSON.parse(row[i]) as QueryValue; + if (column.type.startsWith('xarray(row')) { + malloyRow[column.name] = this.convertNest(column.type, row[i]); } else { malloyRow[column.name] = row[i] as QueryValue; } diff --git a/test/src/databases/all/expr.spec.ts b/test/src/databases/all/expr.spec.ts index 7da920337..675e5ea81 100644 --- a/test/src/databases/all/expr.spec.ts +++ b/test/src/databases/all/expr.spec.ts @@ -114,6 +114,7 @@ describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => { it(' simple turtle', async () => { await expect(` run: ${databaseName}.table('malloytest.state_facts') -> { + --debug group_by: popular_name aggregate: airport_count.sum() nest: by_state is { @@ -121,6 +122,9 @@ describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => { aggregate: airport_count.sum() limit: 2 } + nest: a is { + aggregate: foo is airport_count.sum(), bar is max(airport_count) + } limit: 3 } `).malloyResultMatches(expressionModel, { From c79562aebae04a66159a1af7a7732f5af1e48bb5 Mon Sep 17 00:00:00 2001 From: lloyd tabb Date: Tue, 2 Apr 2024 08:52:10 -0700 Subject: [PATCH 2/3] Convert rows to structs. --- .../malloy-db-trino/src/trino_connection.ts | 66 +++++++++++++------ test/src/databases/all/expr.spec.ts | 4 -- 2 files changed, 47 insertions(+), 23 deletions(-) diff --git a/packages/malloy-db-trino/src/trino_connection.ts b/packages/malloy-db-trino/src/trino_connection.ts index 6dfaa9923..5a2ffb135 100644 --- a/packages/malloy-db-trino/src/trino_connection.ts +++ b/packages/malloy-db-trino/src/trino_connection.ts @@ -253,16 +253,25 @@ export class TrinoConnection implements Connection, PersistSQLResults { } }*/ - convertNest(column: string, dataRows: unknown[][]) { + convertRow(structDef: StructDef, row: unknown[]) { + const col = {}; + for (let i = 0; i < structDef.fields.length; i++) { + col[structDef.fields[i].name] = row[i]; + } + return col; + } + + convertNest(structDef: StructDef, dataRows: unknown[][]) { // eslint-disable-next-line @typescript-eslint/no-explicit-any const ret: any[] = []; + if ( + structDef.structRelationship.type === 'nested' && + !structDef.structRelationship.isArray + ) { + return this.convertRow(structDef, dataRows); + } for (const row of dataRows) { - const col = {}; - for (const element of row) { - const e: string[] = element as string[]; - col[e[0] as string] = e[1]; - } - ret.push(col); + ret.push(this.convertRow(structDef, row)); } return ret; } @@ -284,6 +293,14 @@ export class TrinoConnection implements Connection, PersistSQLResults { ); } + const malloyColumns = queryResult.value.columns.map(c => + this.malloyTypeFromTrinoType(c.name, c.type) + ); + + // Debugging types + // const _x = queryResult.value.columns.map(c => console.log(c.type)); + // console.log(JSON.stringify(malloyColumns, null, 2)); + let maxRows = options.rowLimit ?? 50; const malloyRows: QueryDataRow[] = []; while (queryResult !== null && maxRows--) { @@ -292,9 +309,11 @@ export class TrinoConnection implements Connection, PersistSQLResults { const malloyRow: QueryDataRow = {}; for (let i = 0; i < queryResult.value.columns.length; i++) { const column = queryResult.value.columns[i]; - // TODO: handle arrays etc. - if (column.type.startsWith('xarray(row')) { - malloyRow[column.name] = this.convertNest(column.type, row[i]); + if (malloyColumns[i].type === 'struct') { + malloyRow[column.name] = this.convertNest( + malloyColumns[i] as StructDef, + row[i] + ) as QueryValue; } else { malloyRow[column.name] = row[i] as QueryValue; } @@ -482,18 +501,27 @@ export class TrinoConnection implements Connection, PersistSQLResults { if (arrayMatch) { const arrayType = arrayMatch[1]; const innerType = this.malloyTypeFromTrinoType(name, arrayType); - malloyType = { - type: 'struct', - name, - dialect: this.dialectName, - structSource: {type: 'nested'}, - structRelationship: { + if (innerType.type === 'struct') { + malloyType = innerType; + malloyType.structRelationship = { type: 'nested', fieldName: name, isArray: true, - }, - fields: [{...innerType, name: 'value'} as FieldTypeDef], - }; + }; + } else { + malloyType = { + type: 'struct', + name, + dialect: this.dialectName, + structSource: {type: 'nested'}, + structRelationship: { + type: 'nested', + fieldName: name, + isArray: true, + }, + fields: [{...innerType, name: 'value'} as FieldTypeDef], + }; + } } else if (structMatch) { // TODO: Trino doesn't quote or escape commas in field names, // so some magic is going to need to be applied before we get here diff --git a/test/src/databases/all/expr.spec.ts b/test/src/databases/all/expr.spec.ts index 675e5ea81..7da920337 100644 --- a/test/src/databases/all/expr.spec.ts +++ b/test/src/databases/all/expr.spec.ts @@ -114,7 +114,6 @@ describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => { it(' simple turtle', async () => { await expect(` run: ${databaseName}.table('malloytest.state_facts') -> { - --debug group_by: popular_name aggregate: airport_count.sum() nest: by_state is { @@ -122,9 +121,6 @@ describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => { aggregate: airport_count.sum() limit: 2 } - nest: a is { - aggregate: foo is airport_count.sum(), bar is max(airport_count) - } limit: 3 } `).malloyResultMatches(expressionModel, { From 9b1e9196340a750396af103cd01ae446d0bc8aca Mon Sep 17 00:00:00 2001 From: lloyd tabb Date: Tue, 2 Apr 2024 09:32:00 -0700 Subject: [PATCH 3/3] nested array test. --- .../src/trino_connection.spec.ts | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/packages/malloy-db-trino/src/trino_connection.spec.ts b/packages/malloy-db-trino/src/trino_connection.spec.ts index b28f8e053..d984ee140 100644 --- a/packages/malloy-db-trino/src/trino_connection.spec.ts +++ b/packages/malloy-db-trino/src/trino_connection.spec.ts @@ -113,22 +113,9 @@ describe('Trino connection', () => { }, 'structSource': {'type': 'nested'}, 'fields': [ - { - 'name': 'value', - 'type': 'struct', - 'dialect': 'trino', - 'structRelationship': { - 'fieldName': 'test', - 'isArray': false, - 'type': 'nested', - }, - 'structSource': {'type': 'nested'}, - 'fields': [ - {'name': 'a', 'numberType': 'float', 'type': 'number'}, - {'name': 'b', 'numberType': 'integer', 'type': 'number'}, - {'name': 'c', 'type': 'string'}, - ], - }, + {'name': 'a', 'numberType': 'float', 'type': 'number'}, + {'name': 'b', 'numberType': 'integer', 'type': 'number'}, + {'name': 'c', 'type': 'string'}, ], }); });