update to ndc-spec v0.1.6 to implement support for querying nested co…

…llections (#101) I added some support for more ad-hoc data for test cases that I worked on recently. I also moved the fixture connector configuration files because twice now I've run into situations where I was wondering why something wasn't working, and then I realized that I wrote configuration to one directory, but the connectors were reading configuration from a subdirectory.
hasura · Sep 9, 2024 · fa22741 · fa22741
1 parent 8df94e9
commit fa22741
Show file tree

Hide file tree

Showing 90 changed files with 1,976 additions and 218 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,113 @@ This changelog documents the changes between release versions.
 ### Added
 
 - Extended JSON fields now support all comparison and aggregation functions ([#99](https://github.com/hasura/ndc-mongodb/pull/99))
+- Update to ndc-spec v0.1.6 which allows filtering by object values in array fields ([#101](https://github.com/hasura/ndc-mongodb/pull/101))
+
+#### Filtering by values in arrays
+
+In this update you can filter by making comparisons to object values inside
+arrays. For example consider a MongoDB database with these three documents:
+
+```json
+{ "institution": "Black Mesa", "staff": [{ "name": "Freeman" }, { "name": "Calhoun" }] }
+{ "institution": "Aperture Science", "staff": [{ "name": "GLaDOS" }, { "name": "Chell" }] }
+{ "institution": "City 17", "staff": [{ "name": "Alyx" }, { "name": "Freeman" }, { "name": "Breen" }] }
+```
+
+You can now write a GraphQL query with a `where` clause that checks individual
+entries in the `staff` arrays:
+
+```graphql
+query {
+  institutions(where: { staff: { name: { _eq: "Freeman" } } }) {
+    institution
+  }
+}
+```
+
+Which produces the result:
+
+```json
+{ "data": { "institutions": [
+  { "institution": "Black Mesa" },
+  { "institution": "City 17" } 
+] } }
+```
+
+The filter selects documents where **any** element in the array passes the
+condition. If you want to select only documents where _every_ array element
+passes then negate the comparison on array element values, and also negate the
+entire predicate like this:
+
+```graphql
+query EveryElementMustMatch {
+  institutions(
+    where: { _not: { staff: { name: { _neq: "Freeman" } } } }
+  ) {
+    institution
+  }
+}
+```
+
+**Note:** It is currently only possible to filter on arrays that contain
+objects. Filtering on arrays that contain scalar values or nested arrays will
+come later.
+
+To configure DDN metadata to filter on array fields configure the
+`BooleanExpressionType` for the containing document object type to use an
+**object** boolean expression type for comparisons on the array field. The
+GraphQL Engine will transparently distribute object comparisons over array
+elements. For example the above example is configured with this boolean
+expression type for documents:
+
+```yaml
+---
+kind: BooleanExpressionType
+version: v1
+definition:
+  name: InstitutionComparisonExp
+  operand:
+    object:
+      type: Institution
+      comparableFields:
+        - fieldName: id
+          booleanExpressionType: ObjectIdComparisonExp
+        - fieldName: institution
+          booleanExpressionType: StringComparisonExp
+        - fieldName: staff
+          booleanExpressionType: InstitutionStaffComparisonExp
+      comparableRelationships: []
+  logicalOperators:
+    enable: true
+  isNull:
+    enable: true
+  graphql:
+    typeName: InstitutionComparisonExp
+```
+
+`InstitutionStaffComparisonExp` is the boolean expression type for objects
+inside the `staff` array. It looks like this:
+
+```yaml
+---
+kind: BooleanExpressionType
+version: v1
+definition:
+  name: InstitutionStaffComparisonExp
+  operand:
+    object:
+      type: InstitutionStaff
+      comparableFields:
+        - fieldName: name
+          booleanExpressionType: StringComparisonExp
+      comparableRelationships: []
+  logicalOperators:
+    enable: true
+  isNull:
+    enable: true
+  graphql:
+    typeName: InstitutionStaffComparisonExp
+```
 
 ### Fixed
 

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -18,8 +18,8 @@ resolver = "2"
 # The tag or rev of ndc-models must match the locked tag or rev of the
 # ndc-models dependency of ndc-sdk
 [workspace.dependencies]
-ndc-sdk = { git = "https://github.com/hasura/ndc-sdk-rs.git", tag = "v0.2.1" }
-ndc-models = { git = "http://github.com/hasura/ndc-spec.git", tag = "v0.1.5" }
+ndc-sdk = { git = "https://github.com/hasura/ndc-sdk-rs.git", tag = "v0.4.0" }
+ndc-models = { git = "http://github.com/hasura/ndc-spec.git", tag = "v0.1.6" }
 
 indexmap = { version = "2", features = [
   "serde",

diff --git a/arion-compose/e2e-testing.nix b/arion-compose/e2e-testing.nix
@@ -20,7 +20,7 @@ in
 
     connector = import ./services/connector.nix {
       inherit pkgs;
-      configuration-dir = ../fixtures/hasura/chinook/connector/chinook;
+      configuration-dir = ../fixtures/hasura/chinook/connector;
       database-uri = "mongodb://mongodb/chinook";
       port = connector-port;
       service.depends_on.mongodb.condition = "service_healthy";

diff --git a/arion-compose/integration-test-services.nix b/arion-compose/integration-test-services.nix
@@ -12,6 +12,7 @@
 , otlp-endpoint ? null
 , connector-port ? "7130"
 , connector-chinook-port ? "7131"
+, connector-test-cases-port ? "7132"
 , engine-port ? "7100"
 , mongodb-port ? "27017"
 }:
@@ -21,7 +22,7 @@ in
 {
   connector = import ./services/connector.nix {
     inherit pkgs otlp-endpoint;
-    configuration-dir = ../fixtures/hasura/sample_mflix/connector/sample_mflix;
+    configuration-dir = ../fixtures/hasura/sample_mflix/connector;
     database-uri = "mongodb://mongodb/sample_mflix";
     port = connector-port;
     hostPort = hostPort connector-port;
@@ -32,7 +33,7 @@ in
 
   connector-chinook = import ./services/connector.nix {
     inherit pkgs otlp-endpoint;
-    configuration-dir = ../fixtures/hasura/chinook/connector/chinook;
+    configuration-dir = ../fixtures/hasura/chinook/connector;
     database-uri = "mongodb://mongodb/chinook";
     port = connector-chinook-port;
     hostPort = hostPort connector-chinook-port;
@@ -41,6 +42,17 @@ in
     };
   };
 
+  connector-test-cases = import ./services/connector.nix {
+    inherit pkgs otlp-endpoint;
+    configuration-dir = ../fixtures/hasura/test_cases/connector;
+    database-uri = "mongodb://mongodb/test_cases";
+    port = connector-test-cases-port;
+    hostPort = hostPort connector-test-cases-port;
+    service.depends_on = {
+      mongodb.condition = "service_healthy";
+    };
+  };
+
   mongodb = import ./services/mongodb.nix {
     inherit pkgs;
     port = mongodb-port;
@@ -60,10 +72,12 @@ in
     connectors = {
       chinook = "http://connector-chinook:${connector-chinook-port}";
       sample_mflix = "http://connector:${connector-port}";
+      test_cases = "http://connector-test-cases:${connector-test-cases-port}";
     };
     ddn-dirs = [
       ../fixtures/hasura/chinook/metadata
       ../fixtures/hasura/sample_mflix/metadata
+      ../fixtures/hasura/test_cases/metadata
       ../fixtures/hasura/common/metadata
     ];
     service.depends_on = {

diff --git a/arion-compose/integration-tests.nix b/arion-compose/integration-tests.nix
@@ -11,6 +11,7 @@
 let
   connector-port = "7130";
   connector-chinook-port = "7131";
+  connector-test-cases-port = "7132";
   engine-port = "7100";
 
   services = import ./integration-test-services.nix {
@@ -26,10 +27,12 @@ in
       inherit pkgs;
       connector-url = "http://connector:${connector-port}/";
       connector-chinook-url = "http://connector-chinook:${connector-chinook-port}/";
+      connector-test-cases-url = "http://connector-test-cases:${connector-test-cases-port}/";
       engine-graphql-url = "http://engine:${engine-port}/graphql";
       service.depends_on = {
         connector.condition = "service_healthy";
         connector-chinook.condition = "service_healthy";
+        connector-test-cases.condition = "service_healthy";
         engine.condition = "service_healthy";
       };
       # Run the container as the current user so when it writes to the snapshots

diff --git a/arion-compose/ndc-test.nix b/arion-compose/ndc-test.nix
@@ -14,7 +14,7 @@ in
       # command = ["test" "--snapshots-dir" "/snapshots" "--seed" "1337_1337_1337_1337_1337_1337_13"];
       # Replay and test the recorded snapshots
       # command = ["replay" "--snapshots-dir" "/snapshots"];
-      configuration-dir = ../fixtures/hasura/chinook/connector/chinook;
+      configuration-dir = ../fixtures/hasura/chinook/connector;
       database-uri = "mongodb://mongodb:${mongodb-port}/chinook";
       service.depends_on.mongodb.condition = "service_healthy";
       # Run the container as the current user so when it writes to the snapshots directory it doesn't write as root

diff --git a/arion-compose/services/connector.nix b/arion-compose/services/connector.nix
@@ -12,7 +12,7 @@
 , profile ? "dev" # Rust crate profile, usually either "dev" or "release"
 , hostPort ? null
 , command ? ["serve"]
-, configuration-dir ? ../../fixtures/hasura/sample_mflix/connector/sample_mflix
+, configuration-dir ? ../../fixtures/hasura/sample_mflix/connector
 , database-uri ? "mongodb://mongodb/sample_mflix"
 , service ? { } # additional options to customize this service configuration
 , otlp-endpoint ? null
@@ -32,16 +32,14 @@ let
       "${hostPort}:${port}" # host:container
     ];
     environment = pkgs.lib.filterAttrs (_: v: v != null) {
-      HASURA_CONFIGURATION_DIRECTORY = "/configuration";
+      HASURA_CONFIGURATION_DIRECTORY = (pkgs.lib.sources.cleanSource configuration-dir).outPath;
       HASURA_CONNECTOR_PORT = port;
       MONGODB_DATABASE_URI = database-uri;
       OTEL_SERVICE_NAME = "mongodb-connector";
       OTEL_EXPORTER_OTLP_ENDPOINT = otlp-endpoint;
       RUST_LOG = "configuration=debug,mongodb_agent_common=debug,mongodb_connector=debug,mongodb_support=debug,ndc_query_plan=debug";
     };
-    volumes = [
-      "${configuration-dir}:/configuration:ro"
-    ] ++ extra-volumes;
+    volumes = extra-volumes;
     healthcheck = {
       test = [ "CMD" "${pkgs.pkgsCross.linux.curl}/bin/curl" "-f" "http://localhost:${port}/health" ];
       start_period = "5s";

diff --git a/arion-compose/services/integration-tests.nix b/arion-compose/services/integration-tests.nix
@@ -1,6 +1,7 @@
 { pkgs
 , connector-url
 , connector-chinook-url
+, connector-test-cases-url
 , engine-graphql-url
 , service ? { } # additional options to customize this service configuration
 }:
@@ -16,6 +17,7 @@ let
     environment = {
       CONNECTOR_URL = connector-url;
       CONNECTOR_CHINOOK_URL = connector-chinook-url;
+      CONNECTOR_TEST_CASES_URL = connector-test-cases-url;
       ENGINE_GRAPHQL_URL = engine-graphql-url;
       INSTA_WORKSPACE_ROOT = repo-source-mount-point;
       MONGODB_IMAGE = builtins.getEnv "MONGODB_IMAGE";

diff --git a/crates/integration-tests/src/tests/filtering.rs b/crates/integration-tests/src/tests/filtering.rs
@@ -31,3 +31,24 @@ async fn filters_on_extended_json_using_string_comparison() -> anyhow::Result<()
     );
     Ok(())
 }
+
+#[tokio::test]
+async fn filters_by_comparisons_on_elements_of_array_field() -> anyhow::Result<()> {
+    assert_yaml_snapshot!(
+        graphql_query(
+            r#"
+            query {
+              testCases_nestedCollection(
+                where: { staff: { name: { _eq: "Freeman" } } }
+                order_by: { institution: Asc }
+              ) {
+                institution
+              }
+            }
+            "#
+        )
+        .run()
+        .await?
+    );
+    Ok(())
+}
diff --git a/...tegration_tests__tests__filtering__filters_by_comparisons_on_elements_of_array_field.snap b/...tegration_tests__tests__filtering__filters_by_comparisons_on_elements_of_array_field.snap
@@ -0,0 +1,9 @@
+---
+source: crates/integration-tests/src/tests/filtering.rs
+expression: "graphql_query(r#\"\n            query {\n              testCases_nestedCollection(\n                where: { staff: { name: { _eq: \"Freeman\" } } }\n                order_by: { institution: Asc }\n              ) {\n                institution\n              }\n            }\n            \"#).run().await?"
+---
+data:
+  testCases_nestedCollection:
+    - institution: Black Mesa
+    - institution: City 17
+errors: ~
diff --git a/crates/mongodb-agent-common/src/health.rs b/crates/mongodb-agent-common/src/health.rs
diff --git a/crates/mongodb-agent-common/src/interface_types/mongo_agent_error.rs b/crates/mongodb-agent-common/src/interface_types/mongo_agent_error.rs
@@ -1,4 +1,7 @@
-use std::fmt::{self, Display};
+use std::{
+    borrow::Cow,
+    fmt::{self, Display},
+};
 
 use http::StatusCode;
 use mongodb::bson;
@@ -19,7 +22,7 @@ pub enum MongoAgentError {
     MongoDBDeserialization(#[from] mongodb::bson::de::Error),
     MongoDBSerialization(#[from] mongodb::bson::ser::Error),
     MongoDBSupport(#[from] mongodb_support::error::Error),
-    NotImplemented(&'static str),
+    NotImplemented(Cow<'static, str>),
     Procedure(#[from] ProcedureError),
     QueryPlan(#[from] QueryPlanError),
     ResponseSerialization(#[from] QueryResponseError),

diff --git a/crates/mongodb-agent-common/src/lib.rs b/crates/mongodb-agent-common/src/lib.rs
@@ -1,7 +1,6 @@
 pub mod aggregation_function;
 pub mod comparison_function;
 pub mod explain;
-pub mod health;
 pub mod interface_types;
 pub mod mongo_query_plan;
 pub mod mongodb;