support all comparison and aggregation functions on extended json fie…

…lds (#99)

Since Extended JSON fields may contain any data users may reasonably want access to all comparison and aggregation operations. MongoDB is designed to accommodate functions that don't make sense on all values they are applied to. I included a native query that produces a collection of mixed data types, and some integration tests on that data.

I also updated the dev shell to pull the cli from the [nix flake](https://github.com/hasura/ddn-cli-nix) that @TheInnerLight helpfully set up.

CHANGELOG.md

@@ -4,6 +4,14 @@ This changelog documents the changes between release versions.
 
 ## [Unreleased]
 
+### Added
+
+- Extended JSON fields now support all comparison and aggregation functions ([#99](https://github.com/hasura/ndc-mongodb/pull/99))
+
+### Fixed
+
+### Changed
+
 ## [1.1.0] - 2024-08-16
 
 - Accept predicate arguments in native mutations and native queries ([#92](https://github.com/hasura/ndc-mongodb/pull/92))

crates/integration-tests/src/tests/aggregation.rs

@@ -37,3 +37,89 @@ async fn runs_aggregation_over_top_level_fields() -> anyhow::Result<()> {
     );
     Ok(())
 }
+
+#[tokio::test]
+async fn aggregates_extended_json_representing_mixture_of_numeric_types() -> anyhow::Result<()> {
+    // Skip this test in MongoDB 5 because the example fails there. We're getting an error:
+    //
+    // > Kind: Command failed: Error code 5491300 (Location5491300): $documents' is not allowed in user requests, labels: {}
+    //
+    // This doesn't affect native queries that don't use the $documents stage.
+    if let Ok(image) = std::env::var("MONGODB_IMAGE") {
+        if image == "mongo:5" {
+            return Ok(());
+        }
+    }
+
+    assert_yaml_snapshot!(
+        graphql_query(
+            r#"
+                query ($types: String!) {
+                  extendedJsonTestDataAggregate(
+                    filter_input: { where: { type: { _regex: $types } } }
+                  ) {
+                    value {
+                      _avg
+                      _count
+                      _max
+                      _min
+                      _sum
+                      _count_distinct
+                    }
+                  }
+                  extendedJsonTestData(where: { type: { _regex: $types } }) {
+                    type
+                    value
+                  }
+                }
+            "#
+        )
+        .variables(json!({ "types": "decimal|double|int|long" }))
+        .run()
+        .await?
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn aggregates_mixture_of_numeric_and_null_values() -> anyhow::Result<()> {
+    // Skip this test in MongoDB 5 because the example fails there. We're getting an error:
+    //
+    // > Kind: Command failed: Error code 5491300 (Location5491300): $documents' is not allowed in user requests, labels: {}
+    //
+    // This doesn't affect native queries that don't use the $documents stage.
+    if let Ok(image) = std::env::var("MONGODB_IMAGE") {
+        if image == "mongo:5" {
+            return Ok(());
+        }
+    }
+
+    assert_yaml_snapshot!(
+        graphql_query(
+            r#"
+                query ($types: String!) {
+                  extendedJsonTestDataAggregate(
+                    filter_input: { where: { type: { _regex: $types } } }
+                  ) {
+                    value {
+                      _avg
+                      _count
+                      _max
+                      _min
+                      _sum
+                      _count_distinct
+                    }
+                  }
+                  extendedJsonTestData(where: { type: { _regex: $types } }) {
+                    type
+                    value
+                  }
+                }
+            "#
+        )
+        .variables(json!({ "types": "double|null" }))
+        .run()
+        .await?
+    );
+    Ok(())
+}

crates/integration-tests/src/tests/filtering.rs

@@ -0,0 +1,33 @@
+use insta::assert_yaml_snapshot;
+
+use crate::graphql_query;
+
+#[tokio::test]
+async fn filters_on_extended_json_using_string_comparison() -> anyhow::Result<()> {
+    // Skip this test in MongoDB 5 because the example fails there. We're getting an error:
+    //
+    // > Kind: Command failed: Error code 5491300 (Location5491300): $documents' is not allowed in user requests, labels: {}
+    //
+    // This doesn't affect native queries that don't use the $documents stage.
+    if let Ok(image) = std::env::var("MONGODB_IMAGE") {
+        if image == "mongo:5" {
+            return Ok(());
+        }
+    }
+
+    assert_yaml_snapshot!(
+        graphql_query(
+            r#"
+                query Filtering {
+                  extendedJsonTestData(where: { value: { _regex: "hello" } }) {
+                    type
+                    value
+                  }
+                }
+            "#
+        )
+        .run()
+        .await?
+    );
+    Ok(())
+}

crates/integration-tests/src/tests/mod.rs

@@ -9,8 +9,10 @@
 
 mod aggregation;
 mod basic;
+mod filtering;
 mod local_relationship;
 mod native_mutation;
 mod native_query;
 mod permissions;
 mod remote_relationship;
+mod sorting;

crates/integration-tests/src/tests/snapshots/integration_tests__tests__aggregation__aggregates_extended_json_representing_mixture_of_numeric_types.snap

@@ -0,0 +1,43 @@
+---
+source: crates/integration-tests/src/tests/aggregation.rs
+expression: "graphql_query(r#\"\n                query ($types: String!) {\n                  extendedJsonTestDataAggregate(\n                    filter_input: { where: { type: { _regex: $types } } }\n                  ) {\n                    value {\n                      _avg\n                      _count\n                      _max\n                      _min\n                      _sum\n                      _count_distinct\n                    }\n                  }\n                  extendedJsonTestData(where: { type: { _regex: $types } }) {\n                    type\n                    value\n                  }\n                }\n            \"#).variables(json!({\n                        \"types\": \"decimal|double|int|long\"\n                    })).run().await?"
+---
+data:
+  extendedJsonTestDataAggregate:
+    value:
+      _avg:
+        $numberDecimal: "4.5"
+      _count: 8
+      _max:
+        $numberLong: "8"
+      _min:
+        $numberDecimal: "1"
+      _sum:
+        $numberDecimal: "36"
+      _count_distinct: 8
+  extendedJsonTestData:
+    - type: decimal
+      value:
+        $numberDecimal: "1"
+    - type: decimal
+      value:
+        $numberDecimal: "2"
+    - type: double
+      value:
+        $numberDouble: "3.0"
+    - type: double
+      value:
+        $numberDouble: "4.0"
+    - type: int
+      value:
+        $numberInt: "5"
+    - type: int
+      value:
+        $numberInt: "6"
+    - type: long
+      value:
+        $numberLong: "7"
+    - type: long
+      value:
+        $numberLong: "8"
+errors: ~

crates/integration-tests/src/tests/snapshots/integration_tests__tests__aggregation__aggregates_mixture_of_numeric_and_null_values.snap

@@ -0,0 +1,27 @@
+---
+source: crates/integration-tests/src/tests/aggregation.rs
+expression: "graphql_query(r#\"\n                query ($types: String!) {\n                  extendedJsonTestDataAggregate(\n                    filter_input: { where: { type: { _regex: $types } } }\n                  ) {\n                    value {\n                      _avg\n                      _count\n                      _max\n                      _min\n                      _sum\n                      _count_distinct\n                    }\n                  }\n                  extendedJsonTestData(where: { type: { _regex: $types } }) {\n                    type\n                    value\n                  }\n                }\n            \"#).variables(json!({\n                        \"types\": \"double|null\"\n                    })).run().await?"
+---
+data:
+  extendedJsonTestDataAggregate:
+    value:
+      _avg:
+        $numberDouble: "3.5"
+      _count: 2
+      _max:
+        $numberDouble: "4.0"
+      _min:
+        $numberDouble: "3.0"
+      _sum:
+        $numberDouble: "7.0"
+      _count_distinct: 2
+  extendedJsonTestData:
+    - type: double
+      value:
+        $numberDouble: "3.0"
+    - type: double
+      value:
+        $numberDouble: "4.0"
+    - type: "null"
+      value: ~
+errors: ~

crates/integration-tests/src/tests/snapshots/integration_tests__tests__filtering__filters_on_extended_json_using_string_comparison.snap

@@ -0,0 +1,9 @@
+---
+source: crates/integration-tests/src/tests/filtering.rs
+expression: "graphql_query(r#\"\n                query Filtering {\n                  extendedJsonTestData(where: { value: { _regex: \"hello\" } }) {\n                    type\n                    value\n                  }\n                }\n            \"#).variables(json!({\n                        \"types\": \"double|null\"\n                    })).run().await?"
+---
+data:
+  extendedJsonTestData:
+    - type: string
+      value: "hello, world!"
+errors: ~

crates/integration-tests/src/tests/snapshots/integration_tests__tests__sorting__sorts_on_extended_json.snap

@@ -0,0 +1,45 @@
+---
+source: crates/integration-tests/src/tests/sorting.rs
+expression: "graphql_query(r#\"\n                query Sorting {\n                  extendedJsonTestData(order_by: { value: Desc }) {\n                    type\n                    value\n                  }\n                }\n            \"#).run().await?"
+---
+data:
+  extendedJsonTestData:
+    - type: date
+      value:
+        $date:
+          $numberLong: "1724164680000"
+    - type: date
+      value:
+        $date:
+          $numberLong: "1637571600000"
+    - type: string
+      value: "hello, world!"
+    - type: string
+      value: foo
+    - type: long
+      value:
+        $numberLong: "8"
+    - type: long
+      value:
+        $numberLong: "7"
+    - type: int
+      value:
+        $numberInt: "6"
+    - type: int
+      value:
+        $numberInt: "5"
+    - type: double
+      value:
+        $numberDouble: "4.0"
+    - type: double
+      value:
+        $numberDouble: "3.0"
+    - type: decimal
+      value:
+        $numberDecimal: "2"
+    - type: decimal
+      value:
+        $numberDecimal: "1"
+    - type: "null"
+      value: ~
+errors: ~

crates/integration-tests/src/tests/sorting.rs

@@ -0,0 +1,33 @@
+use insta::assert_yaml_snapshot;
+
+use crate::graphql_query;
+
+#[tokio::test]
+async fn sorts_on_extended_json() -> anyhow::Result<()> {
+    // Skip this test in MongoDB 5 because the example fails there. We're getting an error:
+    //
+    // > Kind: Command failed: Error code 5491300 (Location5491300): $documents' is not allowed in user requests, labels: {}
+    //
+    // This doesn't affect native queries that don't use the $documents stage.
+    if let Ok(image) = std::env::var("MONGODB_IMAGE") {
+        if image == "mongo:5" {
+            return Ok(());
+        }
+    }
+
+    assert_yaml_snapshot!(
+        graphql_query(
+            r#"
+                query Sorting {
+                  extendedJsonTestData(order_by: { value: Desc }) {
+                    type
+                    value
+                  }
+                }
+            "#
+        )
+        .run()
+        .await?
+    );
+    Ok(())
+}

crates/mongodb-agent-common/src/scalar_types_capabilities.rs

@@ -25,12 +25,51 @@ pub fn scalar_types() -> BTreeMap<ndc_models::ScalarTypeName, ScalarType> {
 }
 
 fn extended_json_scalar_type() -> (ndc_models::ScalarTypeName, ScalarType) {
+    // Extended JSON could be anything, so allow all aggregation functions
+    let aggregation_functions = enum_iterator::all::<AggregationFunction>();
+
+    // Extended JSON could be anything, so allow all comparison operators
+    let comparison_operators = enum_iterator::all::<ComparisonFunction>();
+
+    let ext_json_type = Type::Named {
+        name: mongodb_support::EXTENDED_JSON_TYPE_NAME.into(),
+    };
+
     (
         mongodb_support::EXTENDED_JSON_TYPE_NAME.into(),
         ScalarType {
             representation: Some(TypeRepresentation::JSON),
-            aggregate_functions: BTreeMap::new(),
-            comparison_operators: BTreeMap::new(),
+            aggregate_functions: aggregation_functions
+                .into_iter()
+                .map(|aggregation_function| {
+                    let name = aggregation_function.graphql_name().into();
+                    let result_type = match aggregation_function {
+                        AggregationFunction::Avg => ext_json_type.clone(),
+                        AggregationFunction::Count => bson_to_named_type(S::Int),
+                        AggregationFunction::Min => ext_json_type.clone(),
+                        AggregationFunction::Max => ext_json_type.clone(),
+                        AggregationFunction::Sum => ext_json_type.clone(),
+                    };
+                    let definition = AggregateFunctionDefinition { result_type };
+                    (name, definition)
+                })
+                .collect(),
+            comparison_operators: comparison_operators
+                .into_iter()
+                .map(|comparison_fn| {
+                    let name = comparison_fn.graphql_name().into();
+                    let definition = match comparison_fn {
+                        C::Equal => ComparisonOperatorDefinition::Equal,
+                        C::Regex | C::IRegex => ComparisonOperatorDefinition::Custom {
+                            argument_type: bson_to_named_type(S::String),
+                        },
+                        _ => ComparisonOperatorDefinition::Custom {
+                            argument_type: ext_json_type.clone(),
+                        },
+                    };
+                    (name, definition)
+                })
+                .collect(),
         },
     )
 }