From c84dd634784b8a49fef26301834b653a68ecb6f0 Mon Sep 17 00:00:00 2001 From: David Overton Date: Fri, 15 Mar 2024 13:54:55 +1100 Subject: [PATCH] CLI plugin (#1) * Move branch dmoverton/cli-plugin from the old repository * Fix build after patch apply * nix configuration for CLI plugin * Finish implementing v2_schema_response_to_configuration * Move state module to common crate to avoid cli dependency on connector * Remove ndc-sdk dependency from cli * Generate metadata directly instead of going via v2 schema response * REmove configuration type conversions * Use ValidatorSchema from common lib * Clippy suggestion * Use schema.json instead of metadata.json for schema file name * Remove unused lib.rs file --------- Co-authored-by: Jesse Hallett --- Cargo.lock | 155 ++++++++------- Cargo.toml | 3 +- connector-definition/Makefile | 21 ++ connector-definition/connector-metadata.yaml | 15 ++ crates/cli/Cargo.toml | 23 +++ crates/cli/src/introspection.rs | 180 ++++++++++++++++++ crates/cli/src/lib.rs | 40 ++++ crates/cli/src/main.rs | 54 ++++++ crates/configuration/src/configuration.rs | 4 +- .../src/{read_directory.rs => directory.rs} | 25 ++- crates/configuration/src/lib.rs | 5 +- crates/configuration/src/metadata/database.rs | 10 +- crates/configuration/src/metadata/mod.rs | 4 +- crates/mongodb-agent-common/src/lib.rs | 1 + crates/mongodb-agent-common/src/schema.rs | 14 +- .../src/state.rs | 8 +- crates/mongodb-connector/Cargo.toml | 1 + crates/mongodb-connector/src/main.rs | 1 - .../mongodb-connector/src/mongo_connector.rs | 2 +- crates/mongodb-support/src/bson_type.rs | 4 +- flake.nix | 11 ++ nix/docker-cli-plugin.nix | 12 ++ 22 files changed, 502 insertions(+), 91 deletions(-) create mode 100644 connector-definition/Makefile create mode 100644 connector-definition/connector-metadata.yaml create mode 100644 crates/cli/Cargo.toml create mode 100644 crates/cli/src/introspection.rs create mode 100644 crates/cli/src/lib.rs create mode 100644 crates/cli/src/main.rs rename crates/configuration/src/{read_directory.rs => directory.rs} (75%) rename crates/{mongodb-connector => mongodb-agent-common}/src/state.rs (70%) create mode 100644 nix/docker-cli-plugin.nix diff --git a/Cargo.lock b/Cargo.lock index dbce74e4..6cb0c498 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -56,9 +56,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.4" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" dependencies = [ "anstyle", "anstyle-parse", @@ -104,9 +104,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" [[package]] name = "async-trait" @@ -116,7 +116,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.52", ] [[package]] @@ -351,9 +351,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.10" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fffed7514f420abec6d183b1d3acfd9099c79c3a10a06ade4f8203f1411272" +checksum = "b230ab84b0ffdf890d5a10abdbc8b83ae1c4918275daea1ab8801f71536b2651" dependencies = [ "clap_builder", "clap_derive", @@ -361,33 +361,33 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.9" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63361bae7eef3771745f02d8d892bec2fee5f6e34af316ba556e7f97a7069ff1" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.0", ] [[package]] name = "clap_derive" -version = "4.4.7" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.52", ] [[package]] name = "clap_lex" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "colorchoice" @@ -519,7 +519,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.10.0", "syn 1.0.109", ] @@ -546,8 +546,8 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", - "syn 2.0.38", + "strsim 0.10.0", + "syn 2.0.52", ] [[package]] @@ -580,7 +580,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.38", + "syn 2.0.52", ] [[package]] @@ -744,7 +744,7 @@ checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.52", ] [[package]] @@ -882,7 +882,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.52", ] [[package]] @@ -920,7 +920,7 @@ name = "gdc_rust_types" version = "1.0.2" source = "git+https://github.com/hasura/gdc_rust_types.git?rev=3273434#3273434068400f836cf12ea08c514505446821cb" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.5", "openapiv3", "serde", "serde-enum-str", @@ -967,7 +967,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 2.1.0", + "indexmap 2.2.5", "slab", "tokio", "tokio-util", @@ -1200,9 +1200,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ "equivalent", "hashbrown 0.14.1", @@ -1479,7 +1479,7 @@ dependencies = [ "sha2", "socket2 0.4.9", "stringprep", - "strsim", + "strsim 0.10.0", "take_mut", "thiserror", "tokio", @@ -1524,6 +1524,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "mongodb-cli-plugin" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "configuration", + "futures-util", + "indexmap 1.9.3", + "mongodb", + "mongodb-agent-common", + "mongodb-support", + "serde", + "serde_json", + "thiserror", + "tokio", +] + [[package]] name = "mongodb-connector" version = "0.1.0" @@ -1536,10 +1554,11 @@ dependencies = [ "dc-api-types", "enum-iterator", "http", - "indexmap 2.1.0", + "indexmap 2.2.5", "itertools 0.10.5", "mongodb", "mongodb-agent-common", + "mongodb-support", "ndc-sdk", "ndc-test-helpers", "pretty_assertions", @@ -1588,7 +1607,7 @@ version = "0.1.0" source = "git+http://github.com/hasura/ndc-spec.git?tag=v0.1.0-rc.18#46ef35891198840a21653738cb386f97b069f56f" dependencies = [ "async-trait", - "indexmap 2.1.0", + "indexmap 2.2.5", "opentelemetry", "reqwest", "schemars", @@ -1611,7 +1630,7 @@ dependencies = [ "clap", "gdc_rust_types", "http", - "indexmap 2.1.0", + "indexmap 2.2.5", "mime", "ndc-client", "ndc-test", @@ -1642,7 +1661,7 @@ dependencies = [ "async-trait", "clap", "colored", - "indexmap 2.1.0", + "indexmap 2.2.5", "ndc-client", "proptest", "reqwest", @@ -1657,7 +1676,7 @@ dependencies = [ name = "ndc-test-helpers" version = "0.1.0" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.5", "itertools 0.10.5", "ndc-sdk", "serde_json", @@ -1749,7 +1768,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75e56d5c441965b6425165b7e3223cc933ca469834f4a8b4786817a1f9dc4f13" dependencies = [ - "indexmap 2.1.0", + "indexmap 1.9.3", "serde", "serde_json", ] @@ -1777,7 +1796,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.52", ] [[package]] @@ -1981,7 +2000,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.52", ] [[package]] @@ -2056,9 +2075,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] @@ -2135,9 +2154,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -2440,7 +2459,7 @@ checksum = "45a28f4c49489add4ce10783f7911893516f15afe45d015608d41faca6bc4d29" dependencies = [ "dyn-clone", "indexmap 1.9.3", - "indexmap 2.1.0", + "indexmap 2.2.5", "schemars_derive", "serde", "serde_json", @@ -2521,9 +2540,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.193" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] @@ -2569,13 +2588,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.52", ] [[package]] @@ -2591,11 +2610,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.5", "itoa", "ryu", "serde", @@ -2659,7 +2678,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.1.0", + "indexmap 2.2.5", "serde", "serde_json", "serde_with_macros 3.4.0", @@ -2687,7 +2706,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.52", ] [[package]] @@ -2699,7 +2718,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.52", ] [[package]] @@ -2708,7 +2727,7 @@ version = "0.9.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a15e0ef66bf939a7c890a0bf6d5a733c70202225f9888a89ed5c62298b019129" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.5", "itoa", "ryu", "serde", @@ -2845,6 +2864,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strsim" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" + [[package]] name = "subtle" version = "2.5.0" @@ -2864,9 +2889,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.38" +version = "2.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" dependencies = [ "proc-macro2", "quote", @@ -2933,22 +2958,22 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "thiserror" -version = "1.0.50" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.50" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.52", ] [[package]] @@ -3007,9 +3032,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.34.0" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes", @@ -3042,7 +3067,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.52", ] [[package]] @@ -3191,7 +3216,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.52", ] [[package]] @@ -3470,7 +3495,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.52", "wasm-bindgen-shared", ] @@ -3504,7 +3529,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.52", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3689,5 +3714,5 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.52", ] diff --git a/Cargo.toml b/Cargo.toml index fd9e7429..5c9e06e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,8 @@ members = [ "crates/dc-api", "crates/dc-api-types", "crates/dc-api-test-helpers", - "crates/ndc-test-helpers" + "crates/ndc-test-helpers", + "crates/cli" ] resolver = "2" diff --git a/connector-definition/Makefile b/connector-definition/Makefile new file mode 100644 index 00000000..d6744a88 --- /dev/null +++ b/connector-definition/Makefile @@ -0,0 +1,21 @@ +.DEFAULT_GOAL := build +SHELL = /usr/bin/env bash + +.PHONY: build +build: dist/connector-definition.tgz + +.PHONY: clean +clean: + rm -rf dist + +dist dist/.hasura-connector: + mkdir dist + mkdir dist/.hasura-connector + +dist/.hasura-connector/connector-metadata.yaml: DOCKER_IMAGE ?= $(error The DOCKER_IMAGE variable must be defined) +dist/.hasura-connector/connector-metadata.yaml: connector-metadata.yaml dist/.hasura-connector + cp -f connector-metadata.yaml dist/.hasura-connector/ + yq -i '.packagingDefinition.dockerImage = "$(DOCKER_IMAGE)"' dist/.hasura-connector/connector-metadata.yaml + +dist/connector-definition.tgz: dist/.hasura-connector/connector-metadata.yaml + shopt -s dotglob && cd dist && tar -czvf connector-definition.tgz * \ No newline at end of file diff --git a/connector-definition/connector-metadata.yaml b/connector-definition/connector-metadata.yaml new file mode 100644 index 00000000..833db913 --- /dev/null +++ b/connector-definition/connector-metadata.yaml @@ -0,0 +1,15 @@ +packagingDefinition: + type: PrebuiltDockerImage + dockerImage: +supportedEnvironmentVariables: + - name: MONGODB_DATABASE_URI + description: The URI for the MongoDB database +commands: + update: hasura-mongodb update +cliPlugin: + name: hasura-mongodb + version: "0.0.1" +dockerComposeWatch: + - path: ./ + target: /etc/connector + action: sync+restart \ No newline at end of file diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml new file mode 100644 index 00000000..a4564c46 --- /dev/null +++ b/crates/cli/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "mongodb-cli-plugin" +edition = "2021" +version.workspace = true + +[[bin]] +name = "hasura-mongodb" +path = "./src/main.rs" + +[dependencies] +configuration = { path = "../configuration" } +mongodb-agent-common = { path = "../mongodb-agent-common" } +mongodb = "2.8" +mongodb-support = { path = "../mongodb-support" } + +anyhow = "1.0.80" +clap = { version = "4.5.1", features = ["derive", "env"] } +futures-util = "0.3.28" +indexmap = { version = "1", features = ["serde"] } # must match the version that ndc-client uses +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0.113", features = ["raw_value"] } +thiserror = "1.0.57" +tokio = { version = "1.36.0", features = ["full"] } diff --git a/crates/cli/src/introspection.rs b/crates/cli/src/introspection.rs new file mode 100644 index 00000000..7a8a60d4 --- /dev/null +++ b/crates/cli/src/introspection.rs @@ -0,0 +1,180 @@ +use configuration::{ + metadata::{Collection, ObjectField, ObjectType, Type}, + Metadata, +}; +use futures_util::{StreamExt, TryStreamExt}; +use indexmap::IndexMap; +use mongodb::bson::from_bson; +use mongodb_agent_common::schema::{get_property_description, Property, ValidatorSchema}; +use mongodb_support::{BsonScalarType, BsonType}; + +use mongodb_agent_common::interface_types::{MongoAgentError, MongoConfig}; + +pub async fn get_metadata_from_validation_schema( + config: &MongoConfig, +) -> Result { + let db = config.client.database(&config.database); + let collections_cursor = db.list_collections(None, None).await?; + + let (object_types, collections) = collections_cursor + .into_stream() + .map( + |collection_spec| -> Result<(Vec, Collection), MongoAgentError> { + let collection_spec_value = collection_spec?; + let name = &collection_spec_value.name; + let schema_bson_option = collection_spec_value + .options + .validator + .as_ref() + .and_then(|x| x.get("$jsonSchema")); + + match schema_bson_option { + Some(schema_bson) => { + from_bson::(schema_bson.clone()).map_err(|err| { + MongoAgentError::BadCollectionSchema( + name.to_owned(), + schema_bson.clone(), + err, + ) + }) + } + None => Ok(ValidatorSchema { + bson_type: BsonType::Object, + description: None, + required: Vec::new(), + properties: IndexMap::new(), + }), + } + .map(|validator_schema| make_collection(name, &validator_schema)) + }, + ) + .try_collect::<(Vec>, Vec)>() + .await?; + + Ok(Metadata { + collections, + object_types: object_types.concat(), + }) +} + +fn make_collection( + collection_name: &str, + validator_schema: &ValidatorSchema, +) -> (Vec, Collection) { + let properties = &validator_schema.properties; + let required_labels = &validator_schema.required; + + let (mut object_type_defs, object_fields) = { + let type_prefix = format!("{collection_name}_"); + let id_field = ObjectField { + name: "_id".to_string(), + description: Some("primary key _id".to_string()), + r#type: Type::Scalar(BsonScalarType::ObjectId), + }; + let (object_type_defs, mut object_fields): (Vec>, Vec) = + properties + .iter() + .map(|prop| make_object_field(&type_prefix, required_labels, prop)) + .unzip(); + if !object_fields.iter().any(|info| info.name == "_id") { + // There should always be an _id field, so add it unless it was already specified in + // the validator. + object_fields.push(id_field); + } + (object_type_defs.concat(), object_fields) + }; + + let collection_type = ObjectType { + name: collection_name.to_string(), + description: Some(format!("Object type for collection {collection_name}")), + fields: object_fields, + }; + + object_type_defs.push(collection_type); + + let collection_info = Collection { + name: collection_name.to_string(), + description: validator_schema.description.clone(), + r#type: collection_name.to_string(), + }; + + (object_type_defs, collection_info) +} + +fn make_object_field( + type_prefix: &str, + required_labels: &[String], + (prop_name, prop_schema): (&String, &Property), +) -> (Vec, ObjectField) { + let description = get_property_description(prop_schema); + + let object_type_name = format!("{type_prefix}{prop_name}"); + let (collected_otds, field_type) = make_field_type(&object_type_name, prop_schema); + + let object_field = ObjectField { + name: prop_name.clone(), + description, + r#type: maybe_nullable(field_type, !required_labels.contains(prop_name)), + }; + + (collected_otds, object_field) +} + +fn maybe_nullable( + t: configuration::metadata::Type, + is_nullable: bool, +) -> configuration::metadata::Type { + if is_nullable { + configuration::metadata::Type::Nullable(Box::new(t)) + } else { + t + } +} + +fn make_field_type(object_type_name: &str, prop_schema: &Property) -> (Vec, Type) { + let mut collected_otds: Vec = vec![]; + + match prop_schema { + Property::Object { + bson_type: _, + description: _, + required, + properties, + } => { + let type_prefix = format!("{object_type_name}_"); + let (otds, otd_fields): (Vec>, Vec) = properties + .iter() + .map(|prop| make_object_field(&type_prefix, required, prop)) + .unzip(); + + let object_type_definition = ObjectType { + name: object_type_name.to_string(), + description: Some("generated from MongoDB validation schema".to_string()), + fields: otd_fields, + }; + + collected_otds.append(&mut otds.concat()); + collected_otds.push(object_type_definition); + + (collected_otds, Type::Object(object_type_name.to_string())) + } + Property::Array { + bson_type: _, + description: _, + items, + } => { + let item_schemas = *items.clone(); + + let (mut otds, element_type) = make_field_type(object_type_name, &item_schemas); + let field_type = Type::ArrayOf(Box::new(element_type)); + + collected_otds.append(&mut otds); + + (collected_otds, field_type) + } + Property::Scalar { + bson_type, + description: _, + } => (collected_otds, Type::Scalar(bson_type.to_owned())), + } +} diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs new file mode 100644 index 00000000..40cc2697 --- /dev/null +++ b/crates/cli/src/lib.rs @@ -0,0 +1,40 @@ +//! The interpretation of the commands that the CLI can handle. + +mod introspection; + +use std::path::PathBuf; + +use clap::Subcommand; + +use configuration::Configuration; +use mongodb_agent_common::interface_types::MongoConfig; + +/// The command invoked by the user. +#[derive(Debug, Clone, Subcommand)] +pub enum Command { + /// Update the configuration by introspecting the database, using the configuration options. + Update, +} + +pub struct Context { + pub path: PathBuf, + pub mongo_config: MongoConfig, +} + +/// Run a command in a given directory. +pub async fn run(command: Command, context: &Context) -> anyhow::Result<()> { + match command { + Command::Update => update(context).await?, + }; + Ok(()) +} + +/// Update the configuration in the current directory by introspecting the database. +async fn update(context: &Context) -> anyhow::Result<()> { + let metadata = introspection::get_metadata_from_validation_schema(&context.mongo_config).await?; + let configuration = Configuration { metadata }; + + configuration::write_directory(&context.path, &configuration).await?; + + Ok(()) +} diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs new file mode 100644 index 00000000..8d3d40ba --- /dev/null +++ b/crates/cli/src/main.rs @@ -0,0 +1,54 @@ +//! The CLI application. This is used to configure a deployment of mongo-agent-v3. +//! +//! This is intended to be automatically downloaded and invoked via the Hasura CLI, as a plugin. +//! It is unlikely that end-users will use it directly. + +use anyhow::anyhow; +use std::env; +use std::path::PathBuf; + +use clap::Parser; +use mongodb_agent_common::state::{try_init_state_from_uri, DATABASE_URI_ENV_VAR}; +use mongodb_cli_plugin::{run, Command, Context}; + +/// The command-line arguments. +#[derive(Debug, Parser)] +pub struct Args { + /// The path to the configuration. Defaults to the current directory. + #[arg( + long = "context-path", + env = "HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH", + value_name = "DIRECTORY" + )] + pub context_path: Option, + + #[arg( + long = "connection-uri", + env = DATABASE_URI_ENV_VAR, + required = true, + value_name = "URI" + )] + pub connection_uri: String, + + /// The command to invoke. + #[command(subcommand)] + pub subcommand: Command, +} + +/// The application entrypoint. It pulls information from the environment and then calls the [run] +/// function. The library remains unaware of the environment, so that we can more easily test it. +#[tokio::main] +pub async fn main() -> anyhow::Result<()> { + let args = Args::parse(); + // Default the context path to the current directory. + let path = match args.context_path { + Some(path) => path, + None => env::current_dir()?, + }; + let mongo_config = try_init_state_from_uri(&args.connection_uri) + .await + .map_err(|e| anyhow!("Error initializing MongoDB state {}", e))?; + let context = Context { path, mongo_config }; + run(args.subcommand, &context).await?; + Ok(()) +} diff --git a/crates/configuration/src/configuration.rs b/crates/configuration/src/configuration.rs index c38671b1..625ec509 100644 --- a/crates/configuration/src/configuration.rs +++ b/crates/configuration/src/configuration.rs @@ -1,11 +1,11 @@ use std::{io, path::Path}; use schemars::JsonSchema; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::{read_directory, Metadata}; -#[derive(Clone, Debug, Default, Deserialize, JsonSchema)] +#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct Configuration { pub metadata: Metadata, diff --git a/crates/configuration/src/read_directory.rs b/crates/configuration/src/directory.rs similarity index 75% rename from crates/configuration/src/read_directory.rs rename to crates/configuration/src/directory.rs index 3a72c5a6..2a7739eb 100644 --- a/crates/configuration/src/read_directory.rs +++ b/crates/configuration/src/directory.rs @@ -8,9 +8,10 @@ use tokio::fs; use crate::Configuration; -pub const CONFIGURATION_FILENAME: &str = "configuration"; +pub const CONFIGURATION_FILENAME: &str = "schema"; pub const CONFIGURATION_EXTENSIONS: [(&str, FileFormat); 3] = [("json", JSON), ("yaml", YAML), ("yml", YAML)]; +pub const DEFAULT_EXTENSION: &str = "json"; #[derive(Clone, Copy, Debug)] pub enum FileFormat { @@ -80,3 +81,25 @@ where }; Ok(value) } + +pub async fn write_directory( + configuration_dir: impl AsRef, + configuration: &Configuration, +) -> io::Result<()> { + write_file(configuration_dir, CONFIGURATION_FILENAME, configuration).await +} + +fn default_file_path(configuration_dir: impl AsRef, basename: &str) -> PathBuf { + let dir = configuration_dir.as_ref(); + dir.join(format!("{basename}.{DEFAULT_EXTENSION}")) +} + +async fn write_file( + configuration_dir: impl AsRef, + basename: &str, + configuration: &Configuration, +) -> io::Result<()> { + let path = default_file_path(configuration_dir, basename); + let bytes = serde_json::to_vec_pretty(configuration)?; + fs::write(path, bytes).await +} diff --git a/crates/configuration/src/lib.rs b/crates/configuration/src/lib.rs index ba88399d..b4a239ce 100644 --- a/crates/configuration/src/lib.rs +++ b/crates/configuration/src/lib.rs @@ -1,7 +1,8 @@ mod configuration; pub mod metadata; -mod read_directory; +mod directory; pub use crate::configuration::Configuration; pub use crate::metadata::Metadata; -pub use crate::read_directory::read_directory; +pub use crate::directory::read_directory; +pub use crate::directory::write_directory; diff --git a/crates/configuration/src/metadata/database.rs b/crates/configuration/src/metadata/database.rs index 8ea09ef4..c82942e5 100644 --- a/crates/configuration/src/metadata/database.rs +++ b/crates/configuration/src/metadata/database.rs @@ -1,9 +1,9 @@ use schemars::JsonSchema; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use mongodb_support::BsonScalarType; -#[derive(Debug, Clone, PartialEq, Eq, Deserialize, JsonSchema)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct Collection { pub name: String, @@ -15,7 +15,7 @@ pub struct Collection { } /// The type of values that a column, field, or argument may take. -#[derive(Debug, Clone, PartialEq, Eq, Deserialize, JsonSchema)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum Type { /// One of the predefined BSON scalar types @@ -27,7 +27,7 @@ pub enum Type { Nullable(Box), } -#[derive(Debug, Clone, PartialEq, Eq, Deserialize, JsonSchema)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct ObjectType { pub name: String, @@ -37,7 +37,7 @@ pub struct ObjectType { } /// Information about an object type field. -#[derive(Debug, Clone, PartialEq, Eq, Deserialize, JsonSchema)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct ObjectField { pub name: String, diff --git a/crates/configuration/src/metadata/mod.rs b/crates/configuration/src/metadata/mod.rs index 28751944..6326d2e9 100644 --- a/crates/configuration/src/metadata/mod.rs +++ b/crates/configuration/src/metadata/mod.rs @@ -1,11 +1,11 @@ mod database; use schemars::JsonSchema; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; pub use self::database::{Collection, ObjectField, ObjectType, Type}; -#[derive(Clone, Debug, Default, Deserialize, JsonSchema)] +#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct Metadata { #[serde(default)] diff --git a/crates/mongodb-agent-common/src/lib.rs b/crates/mongodb-agent-common/src/lib.rs index 57972fab..ab1585eb 100644 --- a/crates/mongodb-agent-common/src/lib.rs +++ b/crates/mongodb-agent-common/src/lib.rs @@ -8,3 +8,4 @@ pub mod mongodb_connection; pub mod query; pub mod scalar_types_capabilities; pub mod schema; +pub mod state; diff --git a/crates/mongodb-agent-common/src/schema.rs b/crates/mongodb-agent-common/src/schema.rs index 790d691d..a1acd963 100644 --- a/crates/mongodb-agent-common/src/schema.rs +++ b/crates/mongodb-agent-common/src/schema.rs @@ -202,22 +202,22 @@ fn make_column_type( #[derive(Debug, Deserialize)] #[cfg_attr(test, derive(PartialEq))] -struct ValidatorSchema { +pub struct ValidatorSchema { #[serde(rename = "bsonType", alias = "type", default = "default_bson_type")] #[allow(dead_code)] - bson_type: BsonType, + pub bson_type: BsonType, #[serde(skip_serializing_if = "Option::is_none")] - description: Option, + pub description: Option, #[serde(default)] - required: Vec, + pub required: Vec, #[serde(default)] - properties: IndexMap, + pub properties: IndexMap, } #[derive(Clone, Debug, Deserialize)] #[cfg_attr(test, derive(PartialEq))] #[serde(untagged)] -enum Property { +pub enum Property { Object { #[serde(rename = "bsonType", default = "default_bson_type")] #[allow(dead_code)] @@ -244,7 +244,7 @@ enum Property { }, } -fn get_property_description(p: &Property) -> Option { +pub fn get_property_description(p: &Property) -> Option { match p { Property::Object { bson_type: _, diff --git a/crates/mongodb-connector/src/state.rs b/crates/mongodb-agent-common/src/state.rs similarity index 70% rename from crates/mongodb-connector/src/state.rs rename to crates/mongodb-agent-common/src/state.rs index 912bcd96..4ace391b 100644 --- a/crates/mongodb-connector/src/state.rs +++ b/crates/mongodb-agent-common/src/state.rs @@ -1,7 +1,7 @@ use std::{env, error::Error}; use anyhow::anyhow; -use mongodb_agent_common::{interface_types::MongoConfig, mongodb_connection::get_mongodb_client}; +use crate::{interface_types::MongoConfig, mongodb_connection::get_mongodb_client}; pub const DATABASE_URI_ENV_VAR: &str = "MONGODB_DATABASE_URI"; @@ -9,7 +9,11 @@ pub const DATABASE_URI_ENV_VAR: &str = "MONGODB_DATABASE_URI"; pub async fn try_init_state() -> Result> { // Splitting this out of the `Connector` impl makes error translation easier let database_uri = env::var(DATABASE_URI_ENV_VAR)?; - let client = get_mongodb_client(&database_uri).await?; + try_init_state_from_uri(&database_uri).await +} + +pub async fn try_init_state_from_uri(database_uri: &str) -> Result> { + let client = get_mongodb_client(database_uri).await?; let database_name = match client.default_database() { Some(database) => Ok(database.name().to_owned()), None => Err(anyhow!( diff --git a/crates/mongodb-connector/Cargo.toml b/crates/mongodb-connector/Cargo.toml index 8c3b63cc..0632b67c 100644 --- a/crates/mongodb-connector/Cargo.toml +++ b/crates/mongodb-connector/Cargo.toml @@ -15,6 +15,7 @@ indexmap = { version = "2.1.0", features = ["serde"] } itertools = "^0.10" mongodb = "2.8" mongodb-agent-common = { path = "../mongodb-agent-common" } +mongodb-support = { path = "../mongodb-support" } ndc-sdk = { git = "https://github.com/hasura/ndc-hub.git" } prometheus = "*" # share version from ndc-sdk serde = { version = "1.0", features = ["derive"] } diff --git a/crates/mongodb-connector/src/main.rs b/crates/mongodb-connector/src/main.rs index d38f7ce1..26c46d0b 100644 --- a/crates/mongodb-connector/src/main.rs +++ b/crates/mongodb-connector/src/main.rs @@ -3,7 +3,6 @@ mod capabilities; mod error_mapping; mod mongo_connector; mod schema; -mod state; use std::error::Error; diff --git a/crates/mongodb-connector/src/mongo_connector.rs b/crates/mongodb-connector/src/mongo_connector.rs index 6a15e319..9edf4709 100644 --- a/crates/mongodb-connector/src/mongo_connector.rs +++ b/crates/mongodb-connector/src/mongo_connector.rs @@ -48,7 +48,7 @@ impl Connector for MongoConnector { _configuration: &Self::Configuration, _metrics: &mut prometheus::Registry, ) -> Result { - let state = crate::state::try_init_state().await?; + let state = mongodb_agent_common::state::try_init_state().await?; Ok(state) } diff --git a/crates/mongodb-support/src/bson_type.rs b/crates/mongodb-support/src/bson_type.rs index efeae0ac..5f948553 100644 --- a/crates/mongodb-support/src/bson_type.rs +++ b/crates/mongodb-support/src/bson_type.rs @@ -1,7 +1,7 @@ use dc_api_types::GraphQlType; use enum_iterator::{all, Sequence}; use schemars::JsonSchema; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::error::Error; @@ -80,7 +80,7 @@ impl<'de> Deserialize<'de> for BsonType { } } -#[derive(Copy, Clone, Debug, PartialEq, Eq, Sequence, Deserialize, JsonSchema)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Sequence, Serialize, Deserialize, JsonSchema)] #[serde(try_from = "BsonType", rename_all = "camelCase")] pub enum BsonScalarType { // numeric diff --git a/flake.nix b/flake.nix index 4f35db9b..04b064e7 100644 --- a/flake.nix +++ b/flake.nix @@ -89,6 +89,7 @@ # arion-compose.nix. mongodb-connector-workspace = final.callPackage ./nix/mongodb-connector-workspace.nix { }; # builds all packages in this repo mongodb-connector = final.mongodb-connector-workspace.override { package = "mongodb-connector"; }; # override `package` to build one specific crate + mongodb-cli-plugin = final.mongodb-connector-workspace.override { package = "mongodb-cli-plugin"; }; v3-engine = final.callPackage ./nix/v3-engine.nix { src = v3-engine-source; }; v3-e2e-testing = final.callPackage ./nix/v3-e2e-testing.nix { src = v3-e2e-testing-source; database-to-test = "mongodb"; }; inherit v3-e2e-testing-source; # include this source so we can read files from it in arion-compose configs @@ -170,6 +171,16 @@ architecture = "arm64"; }; + # CLI plugin packages with cross-compilation options + mongodb-cli-plugin = pkgs.mongodb-cli-plugin.override { staticallyLinked = true; }; + mongodb-cli-plugin-x86_64-linux = pkgs.pkgsCross.x86_64-linux.mongodb-cli-plugin.override { staticallyLinked = true; }; + mongodb-cli-plugin-aarch64-linux = pkgs.pkgsCross.aarch64-linux.mongodb-cli-plugin.override { staticallyLinked = true; }; + + # CLI plugin docker images + mongodb-cli-plugin-docker = pkgs.callPackage ./nix/docker-cli-plugin.nix { }; + mongodb-cli-plugin-docker-x86_64-linux = pkgs.pkgsCross.x86_64-linux.callPackage ./nix/docker-cli-plugin.nix { }; + mongodb-cli-plugin-docker-aarch64-linux = pkgs.pkgsCross.aarch64-linux.callPackage ./nix/docker-cli-plugin.nix { }; + publish-docker-image = pkgs.writeShellApplication { name = "publish-docker-image"; runtimeInputs = with pkgs; [ coreutils skopeo ]; diff --git a/nix/docker-cli-plugin.nix b/nix/docker-cli-plugin.nix new file mode 100644 index 00000000..e28ae550 --- /dev/null +++ b/nix/docker-cli-plugin.nix @@ -0,0 +1,12 @@ +{ name ? "ghcr.io/hasura/mongodb-cli-plugin" +, mongodb-cli-plugin +, dockerTools +}: + +dockerTools.buildLayeredImage { + inherit name; + created = "now"; + config = { + Entrypoint = [ "${mongodb-cli-plugin}/bin/hasura-mongodb" ]; + }; +}