Fixes to nested data, read more rows until done.

malloydata · Mar 18, 2024 · c5d75ba · c5d75ba
1 parent 5e2fe65
commit c5d75ba
Show file tree

Hide file tree

Showing 6 changed files with 57 additions and 185 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/malloy-db-trino/package.json b/packages/malloy-db-trino/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-trino",
-  "version": "0.0.131",
+  "version": "0.0.132",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

diff --git a/packages/malloy-db-trino/src/trino_connection.ts b/packages/malloy-db-trino/src/trino_connection.ts
@@ -76,17 +76,16 @@ export class TrinoConnection implements Connection, PersistSQLResults {
     'string': {type: 'string'},
     'date': {type: 'date'},
 
-    'STRING': {type: 'string'},
-    'INTEGER': {type: 'number', numberType: 'integer'},
-    'INT64': {type: 'number', numberType: 'integer'},
+    // TODO: cleanup.
+    /* 'INT64': {type: 'number', numberType: 'integer'},
     'FLOAT': {type: 'number', numberType: 'float'},
     'FLOAT64': {type: 'number', numberType: 'float'},
     'NUMERIC': {type: 'number', numberType: 'float'},
     'BIGNUMERIC': {type: 'number', numberType: 'float'},
     'TIMESTAMP': {type: 'timestamp'},
     'BOOLEAN': {type: 'boolean'},
     'BOOL': {type: 'boolean'},
-    'JSON': {type: 'json'},
+    'JSON': {type: 'json'},*/
     // TODO (https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#tablefieldschema):
     // BYTES
     // DATETIME
@@ -265,7 +264,9 @@ export class TrinoConnection implements Connection, PersistSQLResults {
     // TODO: fill in with options.
     const result = await this.trino.query(sqlCommand);
 
-    const queryResult = await result.next();
+    console.log(`THE SQL ===> ${sqlCommand}`);
+
+    let queryResult = await result.next();
     if (queryResult.value.error) {
       // TODO: handle.
       throw new Error(
@@ -275,21 +276,35 @@ export class TrinoConnection implements Connection, PersistSQLResults {
       );
     }
 
-    const rows = queryResult.value.data ?? [];
-
     const malloyRows: QueryDataRow[] = [];
-    for (const row of rows) {
-      const malloyRow: QueryDataRow = {};
-      for (let i = 0; i < queryResult.value.columns.length; i++) {
-        const column = queryResult.value.columns[i];
-        // TODO: handle arrays etc.
-        malloyRow[column.name] = row[i] as QueryValue;
+    while (queryResult !== null) {
+      const rows = queryResult.value.data ?? [];
+      for (const row of rows) {
+        const malloyRow: QueryDataRow = {};
+        for (let i = 0; i < queryResult.value.columns.length; i++) {
+          console.log(
+            `COLUMNI ${JSON.stringify(queryResult.value.columns[i])}`
+          );
+          const column = queryResult.value.columns[i];
+          // TODO: handle arrays etc.
+          if (column.type === 'json') {
+            malloyRow[column.name] = JSON.parse(row[i]) as QueryValue;
+          } else {
+            malloyRow[column.name] = row[i] as QueryValue;
+          }
+        }
+
+        malloyRows.push(malloyRow);
       }
 
-      malloyRows.push(malloyRow);
+      if (!queryResult.done) {
+        queryResult = await result.next();
+      } else {
+        break;
+      }
     }
 
-    console.log(`ROWS: ${JSON.stringify(malloyRows)}`);
+    console.log(`ROWS: ${JSON.stringify(malloyRows)} ${malloyRows.length}`);
     // TODO: handle totalrows.
     return {rows: malloyRows, totalRows: malloyRows.length};
   }
@@ -325,7 +340,10 @@ export class TrinoConnection implements Connection, PersistSQLResults {
 
     for (const tableKey in missing) {
       let inCache = this.schemaCache.get(tableKey);
-      const tablePath = missing[tableKey].replace(/malloytest/g, "malloy_demo.faa");
+      const tablePath = missing[tableKey].replace(
+        /malloytest/g,
+        'malloy_demo.faa'
+      );
       if (
         !inCache ||
         (refreshTimestamp && refreshTimestamp > inCache.timestamp)
@@ -384,7 +402,9 @@ export class TrinoConnection implements Connection, PersistSQLResults {
       if (queryResult.value.error) {
         // TODO: handle.
         throw new Error(
-          `Failed to grab schema for table ${tablePath}: ${JSON.stringify(queryResult.value.error)}`
+          `Failed to grab schema for table ${tablePath}: ${JSON.stringify(
+            queryResult.value.error
+          )}`
         );
       }
 

diff --git a/packages/malloy/src/dialect/trino/trino.ts b/packages/malloy/src/dialect/trino/trino.ts
@@ -140,7 +140,7 @@ export class TrinoDialect extends Dialect {
     const fields = fieldList
       .map(f => `\n '${f.sqlOutputName}' VALUE ${f.sqlExpression}`)
       .join(', ');
-    return `COALESCE(SLICE(COALESCE(ARRAY_AGG(CASE WHEN group_set=${groupSet} THEN JSON_OBJECT(${fields}) END \n ${orderBy} -- ${tail}\n), ARRAY[]), 1, ${limit}), ARRAY[])`;
+    return `cast(COALESCE(SLICE(COALESCE(ARRAY_AGG(CASE WHEN group_set=${groupSet} THEN JSON_PARSE(JSON_OBJECT(${fields})) END \n ${orderBy} -- ${tail}\n), ARRAY[]), 1, ${limit}), ARRAY[]) as json)`;
   }
 
   sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string {

diff --git a/test/package.json b/test/package.json
@@ -27,7 +27,7 @@
     "@malloydata/db-snowflake": "^0.0.132",
     "@malloydata/malloy": "^0.0.132",
     "@malloydata/render": "^0.0.132",
-    "@malloydata/db-trino": "^0.0.1",
+    "@malloydata/db-trino": "^0.0.132",
     "jsdom": "^22.1.0",
     "luxon": "^2.4.0",
     "madge": "^6.0.0"

diff --git a/test/src/databases/trino/trino.spec.ts b/test/src/databases/trino/trino.spec.ts
@@ -40,18 +40,9 @@ describe('Trino tests', () => {
     // Issue: #151
     it(`Basic trino  - ${databaseName}`, async () => {
       await expect(`
-        run: trino.table('sample.burstbank.customer') -> {
-          group_by: city
-          aggregate: avgIncome is avg(estimated_income)
-          aggregate: counti is count()
-          nest: foo is {
-            select: estimated_income
-            order_by: estimated_income
-            limit: 5
-          }
-          order_by: counti desc
-          limit: 5
-        }
+      run: trino.table('malloy_demo.faa.aircraft') -> {
+            select: *
+    }
       `).malloyResultMatches(runtime, {custkey: '1000001'});
     });
   });