Initial implementation of HyperLogLog in Snowflake (#2066)

* Initial implementation of HyperLogLog in Snowflake

Signed-off-by: Jason Smart <jlsmart@meta.com>

* Added a custom HyperLogLog test for Snowflake.  This test is similar to BigQuery, but the two estimators return different values, reflecting the probabalistic nature of the algorithm.  Thus we provide standalone tests for each.

Signed-off-by: Jason Smart <jlsmart@meta.com>

* Fixed formatting and added a comment explaining why we have a dialect-specific test here.

Signed-off-by: Jason Smart <jlsmart@meta.com>

* Cleaned up lint errors.  Why does the linter care about EOF new lines?

Signed-off-by: Jason Smart <jlsmart@meta.com>

* More lint fixes

Signed-off-by: Jason Smart <jlsmart@meta.com>

* Added afterAll() to properly close databases after tests.

Signed-off-by: Jason Smart <jlsmart@meta.com>

* Added misisng EOL

Signed-off-by: Jason Smart <jlsmart@meta.com>

* Implemented a better example query for HLL as a test, and fixed a bug in the db name specified in the test.

Signed-off-by: Jason Smart <jlsmart@meta.com>

* Removed dialect-specific HLL tests, as they are redundant and potentially brittle if the underlying implementations ever change.

Signed-off-by: Jason Smart <jlsmart@meta.com>

---------

Signed-off-by: Jason Smart <jlsmart@meta.com>

Author: anagrd-ai

packages/malloy/src/dialect/snowflake/dialect_functions.ts

@@ -10,10 +10,14 @@ import {
   def,
   DefinitionBlueprintMap,
   OverloadedDefinitionBlueprint,
+  TypeDescBlueprint,
   arg as a,
   sql,
 } from '../functions/util';
 
+// Cute shortcut So you can write things like: {array: T} and {dimension: T}
+const T: TypeDescBlueprint = {generic: 'T'};
+
 const order_by: AggregateOrderByNode = {
   node: 'aggregate_order_by',
   prefix: ' WITHIN GROUP(',
@@ -62,6 +66,38 @@ const string_agg_distinct: OverloadedDefinitionBlueprint = {
 export const SNOWFLAKE_DIALECT_FUNCTIONS: DefinitionBlueprintMap = {
   string_agg,
   string_agg_distinct,
+  hll_accumulate: {
+    default: {
+      takes: {'value': {dimension: T}},
+      returns: {measure: {sql_native: 'hyperloglog'}},
+      generic: {
+        'T': ['string', 'number'],
+      },
+      isSymmetric: true,
+      impl: {function: 'hll_accumulate'},
+    },
+  },
+  hll_combine: {
+    takes: {'value': {sql_native: 'hyperloglog'}},
+    returns: {measure: {sql_native: 'hyperloglog'}},
+    impl: {function: 'hll_combine'},
+    isSymmetric: true,
+  },
+  hll_estimate: {
+    takes: {'value': {sql_native: 'hyperloglog'}},
+    returns: {dimension: 'number'},
+    impl: {function: 'hll_estimate'},
+  },
+  hll_export: {
+    takes: {'value': {sql_native: 'hyperloglog'}},
+    returns: {dimension: {sql_native: 'bytes'}},
+    impl: {function: 'hll_export'},
+  },
+  hll_import: {
+    takes: {'value': {sql_native: 'bytes'}},
+    returns: {dimension: {sql_native: 'hyperloglog'}},
+    impl: {function: 'hll_import'},
+  },
   ...def('repeat', {'str': 'string', 'n': 'number'}, 'string'),
   ...def('reverse', {'str': 'string'}, 'string'),
 };

packages/malloy/src/dialect/snowflake/snowflake.ts

@@ -116,6 +116,7 @@ export class SnowflakeDialect extends Dialect {
   supportsSafeCast = true;
   supportsNesting = true;
   defaultSampling = {rows: 50000};
+  supportsHyperLogLog = true;
 
   // NOTE: safely setting all these to false for now
   // more many be implemented in future

test/src/databases/bigquery/malloy_query.spec.ts

@@ -750,15 +750,6 @@ describe('airport_tests', () => {
     `).malloyResultMatches(model, {'pipe_turtle.total_airports': 1845});
   });
 
-  it('hyperloglog combine', async () => {
-    await expect(`run: bigquery.table('malloytest.airports')->{
-        aggregate: code_hll is hll_accumulate(code)
-    } -> {
-        aggregate: code_count is hll_estimate(hll_combine(code_hll))
-    }
-    `).malloyResultMatches(runtime, {code_count: 19799});
-  });
-
   it.skip('crossjoined turtles', async () => {
     // const result = await runQuery(model,`
     //     explore airports