diff --git a/MODULE.bazel b/MODULE.bazel index aba1c63c607..5a9faffa387 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -31,7 +31,7 @@ bazel_dep( ) bazel_dep( name = "rules_proto", - version = "6.0.0-rc1", + version = "6.0.0-rc2", ) bazel_dep( name = "rules_cc", @@ -323,6 +323,18 @@ http_archive( url = "https://github.com/world-federation-of-advertisers/uk-pilot-event-templates/archive/refs/tags/v0.2.0.tar.gz", ) +# TODO(bazelbuild/rules_proto#203): Remove when fixed. +archive_override( + module_name = "rules_proto", + integrity = "sha256-cf2+0AoHCVIa0hIFjGDROZe5IqXQHb/Zl/DVfWiee2c=", + patch_strip = 0, + patches = ["//build/rules_proto:module_dot_bazel.patch"], + strip_prefix = "rules_proto-6.0.0-rc2", + urls = [ + "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc2/rules_proto-6.0.0-rc2.tar.gz", + ], +) + # Force use of newer version of boringssl. single_version_override( module_name = "boringssl", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index b532bebf389..4a321801c4e 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,6 +1,6 @@ { "lockFileVersion": 3, - "moduleFileHash": "68bf6f2e79e06113d8915d2cdb2ea8f02cc664eb5d03a363eac0a05bc8a42229", + "moduleFileHash": "9749e8e9681fc64b3129c96fa8d80d023d03f74ced682e30a424278e0d6ea3dd", "flags": { "cmdRegistries": [ "https://raw.githubusercontent.com/world-federation-of-advertisers/bazel-registry/main", @@ -354,7 +354,7 @@ "deps": { "platforms": "platforms@0.0.8", "bazel_skylib": "bazel_skylib@1.5.0", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "rules_cc": "rules_cc@0.0.9", "rules_pkg": "rules_pkg@0.9.1", "io_bazel_rules_go": "rules_go@0.43.0", @@ -450,10 +450,10 @@ } } }, - "rules_proto@6.0.0-rc1": { + "rules_proto@_": { "name": "rules_proto", - "version": "6.0.0-rc1", - "key": "rules_proto@6.0.0-rc1", + "version": "0.0.0", + "key": "rules_proto@_", "repoName": "rules_proto", "executionPlatformsToRegister": [], "toolchainsToRegister": [], @@ -461,24 +461,10 @@ "deps": { "rules_license": "rules_license@0.0.7", "bazel_skylib": "bazel_skylib@1.5.0", + "bazel_features": "bazel_features@1.2.0", + "rules_cc": "rules_cc@0.0.9", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_proto~6.0.0-rc1", - "urls": [ - "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc1/rules_proto-6.0.0-rc1.tar.gz" - ], - "integrity": "sha256-kEqAl/rkKmkMjgjYBSEOQMzLBp9fmg9nJ89PqnvtLJw=", - "strip_prefix": "rules_proto-6.0.0-rc1", - "remote_patches": { - "https://bcr.bazel.build/modules/rules_proto/6.0.0-rc1/patches/module_dot_bazel_version.patch": "sha256-OepRECTunWiz3WREvMK/pOvpt/HRZ6g87fKfokcnQGE=" - }, - "remote_patch_strip": 1 - } } }, "rules_cc@0.0.9": { @@ -671,10 +657,10 @@ } ], "deps": { - "bazel_features": "bazel_features@1.1.1", + "bazel_features": "bazel_features@1.2.0", "bazel_skylib": "bazel_skylib@1.5.0", "platforms": "platforms@0.0.8", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_protobuf": "protobuf@23.1", "gazelle": "gazelle@0.34.0", "bazel_tools": "bazel_tools@_", @@ -797,7 +783,7 @@ "bazel_skylib": "bazel_skylib@1.5.0", "com_google_protobuf": "protobuf@23.1", "io_bazel_rules_go": "rules_go@0.43.0", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" }, @@ -897,7 +883,7 @@ "platforms": "platforms@0.0.8", "rules_cc": "rules_cc@0.0.9", "bazel_skylib": "bazel_skylib@1.5.0", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "rules_license": "rules_license@0.0.7", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" @@ -1325,10 +1311,10 @@ } ], "deps": { - "bazel_features": "bazel_features@1.1.1", + "bazel_features": "bazel_features@1.2.0", "bazel_skylib": "bazel_skylib@1.5.0", "platforms": "platforms@0.0.8", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_protobuf": "protobuf@23.1", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" @@ -1421,7 +1407,7 @@ "bazel_skylib": "bazel_skylib@1.5.0", "rules_python": "rules_python@0.29.0", "rules_cc": "rules_cc@0.0.9", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "rules_java": "rules_java@7.3.2", "rules_pkg": "rules_pkg@0.9.1", "platforms": "platforms@0.0.8", @@ -1573,7 +1559,7 @@ "deps": { "aspect_bazel_lib": "aspect_bazel_lib@1.39.0", "aspect_rules_lint": "aspect_rules_lint@0.7.0", - "bazel_features": "bazel_features@1.1.1", + "bazel_features": "bazel_features@1.2.0", "bazel_skylib": "bazel_skylib@1.5.0", "rules_nodejs": "rules_nodejs@5.8.2", "platforms": "platforms@0.0.8", @@ -1955,7 +1941,7 @@ ], "deps": { "bazel_skylib": "bazel_skylib@1.5.0", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "rules_java": "rules_java@7.3.2", "rules_kotlin": "rules_kotlin@1.9.0", "rules_jvm_external": "rules_jvm_external@6.0", @@ -2477,7 +2463,7 @@ "deps": { "platforms": "platforms@0.0.8", "bazel_skylib": "bazel_skylib@1.5.0", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "rules_pkg": "rules_pkg@0.9.1", "rules_java": "rules_java@7.3.2", "rules_multirun": "rules_multirun@0.6.1", @@ -2538,7 +2524,7 @@ ], "deps": { "rules_cc": "rules_cc@0.0.9", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_absl": "abseil-cpp@20230802.1", "com_google_googletest": "googletest@1.14.0.bcr.1", "com_google_protobuf": "protobuf@23.1", @@ -2591,7 +2577,7 @@ } ], "deps": { - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_protobuf": "protobuf@23.1", "com_google_googleapis": "googleapis@0.0.0-bzlmod.1", "bazel_tools": "bazel_tools@_", @@ -2656,7 +2642,7 @@ "extensionUsages": [], "deps": { "rules_cc": "rules_cc@0.0.9", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_absl": "abseil-cpp@20230802.1", "com_google_googletest": "googletest@1.14.0.bcr.1", "com_github_google_glog": "glog@0.6.0", @@ -2784,7 +2770,7 @@ } ], "deps": { - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_protobuf": "protobuf@23.1", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" @@ -2818,7 +2804,7 @@ "com_github_grpc_grpc": "grpc@1.48.1.bcr.2", "com_google_protobuf": "protobuf@23.1", "rules_cc": "rules_cc@0.0.9", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_absl": "abseil-cpp@20230802.1", "com_github_google_googletest": "googletest@1.14.0.bcr.1", "boringssl": "boringssl@0.0.0-20230215-5c22014", @@ -2891,7 +2877,7 @@ } ], "deps": { - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_protobuf": "protobuf@23.1", "rules_cc": "rules_cc@0.0.9", "com_google_absl": "abseil-cpp@20230802.1", @@ -3002,7 +2988,7 @@ } ], "deps": { - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "io_bazel_rules_go": "rules_go@0.43.0", "bazel_gazelle": "gazelle@0.34.0", "bazel_skylib": "bazel_skylib@1.5.0", @@ -3163,7 +3149,7 @@ ], "deps": { "platforms": "platforms@0.0.8", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_protobuf": "protobuf@23.1", "com_google_absl": "abseil-cpp@20230802.1", "bazel_tools": "bazel_tools@_", @@ -3311,7 +3297,7 @@ "rules_cc": "rules_cc@0.0.9", "rules_java": "rules_java@7.3.2", "rules_license": "rules_license@0.0.7", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "rules_python": "rules_python@0.29.0", "platforms": "platforms@0.0.8", "com_google_protobuf": "protobuf@23.1", @@ -3360,10 +3346,10 @@ } } }, - "bazel_features@1.1.1": { + "bazel_features@1.2.0": { "name": "bazel_features", - "version": "1.1.1", - "key": "bazel_features@1.1.1", + "version": "1.2.0", + "key": "bazel_features@1.2.0", "repoName": "bazel_features", "executionPlatformsToRegister": [], "toolchainsToRegister": [], @@ -3371,9 +3357,9 @@ { "extensionBzlFile": "@bazel_features//private:extensions.bzl", "extensionName": "version_extension", - "usingModule": "bazel_features@1.1.1", + "usingModule": "bazel_features@1.2.0", "location": { - "file": "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel", + "file": "https://bcr.bazel.build/modules/bazel_features/1.2.0/MODULE.bazel", "line": 6, "column": 24 }, @@ -3395,14 +3381,14 @@ "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "name": "bazel_features~1.1.1", + "name": "bazel_features~1.2.0", "urls": [ - "https://github.com/bazel-contrib/bazel_features/releases/download/v1.1.1/bazel_features-v1.1.1.tar.gz" + "https://github.com/bazel-contrib/bazel_features/releases/download/v1.2.0/bazel_features-v1.2.0.tar.gz" ], - "integrity": "sha256-YsJuQn5cvHUQJERpJ2IuOYqdzfMsZDJSOIFXCdEcEag=", - "strip_prefix": "bazel_features-1.1.1", + "integrity": "sha256-uHicg8iT1+8wQdPyeVd0k2sn/2FwGnBd9S/UHW3b9pI=", + "strip_prefix": "bazel_features-1.2.0", "remote_patches": { - "https://bcr.bazel.build/modules/bazel_features/1.1.1/patches/module_dot_bazel_version.patch": "sha256-+56MAEsc7bYN/Pzhn252ZQUxiRzZg9bynXj1qpsmCYs=" + "https://bcr.bazel.build/modules/bazel_features/1.2.0/patches/module_dot_bazel_version.patch": "sha256-B0rNFSA7VB65YY95ORkzTgsn21RkUwlAb1Xy2y2PyXA=" }, "remote_patch_strip": 1 } @@ -3524,7 +3510,7 @@ "rules_jvm_external": "rules_jvm_external@6.0", "rules_pkg": "rules_pkg@0.9.1", "io_bazel_stardoc": "stardoc@0.5.6", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" }, @@ -3635,7 +3621,7 @@ ], "deps": { "bazel_skylib": "bazel_skylib@1.5.0", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_protobuf": "protobuf@23.1", "com_google_absl": "abseil-cpp@20230802.1", "rules_pkg": "rules_pkg@0.9.1", @@ -3799,7 +3785,7 @@ "aspect_rules_js": "aspect_rules_js@1.35.0", "bazel_skylib": "bazel_skylib@1.5.0", "platforms": "platforms@0.0.8", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "rules_buf": "rules_buf@0.1.1", "com_google_protobuf": "protobuf@23.1", "bazel_tools": "bazel_tools@_", @@ -4132,7 +4118,7 @@ "com_google_absl": "abseil-cpp@20230802.1", "com_google_protobuf": "protobuf@23.1", "com_googlesource_code_re2": "re2@2023-09-01", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "upb": "upb@0.0.0-20230516-61a97ef", "zlib": "zlib@1.3", "rules_java": "rules_java@7.3.2", @@ -4239,7 +4225,7 @@ "deps": { "platforms": "platforms@0.0.8", "com_google_protobuf": "protobuf@23.1", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" }, @@ -135737,7 +135723,7 @@ [ "aspect_rules_js~1.35.0", "bazel_features", - "bazel_features~1.1.1" + "bazel_features~1.2.0" ], [ "aspect_rules_js~1.35.0", @@ -135844,24 +135830,24 @@ ] } }, - "@@bazel_features~1.1.1//private:extensions.bzl%version_extension": { + "@@bazel_features~1.2.0//private:extensions.bzl%version_extension": { "general": { "bzlTransitiveDigest": "xm7Skm1Las5saxzFWt2hbS+e68BWi+MXyt6+lKIhjPA=", "accumulatedFileDigests": {}, "envVariables": {}, "generatedRepoSpecs": { "bazel_features_version": { - "bzlFile": "@@bazel_features~1.1.1//private:version_repo.bzl", + "bzlFile": "@@bazel_features~1.2.0//private:version_repo.bzl", "ruleClassName": "version_repo", "attributes": { - "name": "bazel_features~1.1.1~version_extension~bazel_features_version" + "name": "bazel_features~1.2.0~version_extension~bazel_features_version" } }, "bazel_features_globals": { - "bzlFile": "@@bazel_features~1.1.1//private:globals_repo.bzl", + "bzlFile": "@@bazel_features~1.2.0//private:globals_repo.bzl", "ruleClassName": "globals_repo", "attributes": { - "name": "bazel_features~1.1.1~version_extension~bazel_features_globals", + "name": "bazel_features~1.2.0~version_extension~bazel_features_globals", "globals": { "RunEnvironmentInfo": "5.3.0", "DefaultInfo": "0.0.1", @@ -137608,7 +137594,7 @@ }, "@@rules_go~0.43.0//go:extensions.bzl%go_sdk": { "os:linux,arch:amd64": { - "bzlTransitiveDigest": "vnmAJ4XgQqHHLFcBK8FIw6dj/TSctEBQAwaVeDRGyiI=", + "bzlTransitiveDigest": "rXV9vbWpSmIyZo4jRaNZj4wcI9IJZWc3um7z5mmTu18=", "accumulatedFileDigests": {}, "envVariables": {}, "generatedRepoSpecs": { @@ -137666,19 +137652,19 @@ }, "recordedRepoMappingEntries": [ [ - "bazel_features~1.1.1", + "bazel_features~1.2.0", "bazel_features_globals", - "bazel_features~1.1.1~version_extension~bazel_features_globals" + "bazel_features~1.2.0~version_extension~bazel_features_globals" ], [ - "bazel_features~1.1.1", + "bazel_features~1.2.0", "bazel_features_version", - "bazel_features~1.1.1~version_extension~bazel_features_version" + "bazel_features~1.2.0~version_extension~bazel_features_version" ], [ "rules_go~0.43.0", "bazel_features", - "bazel_features~1.1.1" + "bazel_features~1.2.0" ], [ "rules_go~0.43.0", @@ -137690,7 +137676,7 @@ }, "@@rules_go~0.43.0//go/private:extensions.bzl%non_module_dependencies": { "general": { - "bzlTransitiveDigest": "ZKrgcEEVppo3fWRms84VMXw08TJubOdDD/oJsStb68g=", + "bzlTransitiveDigest": "XNuPeWvAtP8chNDlXrCCWQVfBVQS2OQmdvYGeDJKsmI=", "accumulatedFileDigests": {}, "envVariables": {}, "generatedRepoSpecs": { @@ -137910,19 +137896,19 @@ }, "recordedRepoMappingEntries": [ [ - "bazel_features~1.1.1", + "bazel_features~1.2.0", "bazel_features_globals", - "bazel_features~1.1.1~version_extension~bazel_features_globals" + "bazel_features~1.2.0~version_extension~bazel_features_globals" ], [ - "bazel_features~1.1.1", + "bazel_features~1.2.0", "bazel_features_version", - "bazel_features~1.1.1~version_extension~bazel_features_version" + "bazel_features~1.2.0~version_extension~bazel_features_version" ], [ "rules_go~0.43.0", "bazel_features", - "bazel_features~1.1.1" + "bazel_features~1.2.0" ], [ "rules_go~0.43.0", diff --git a/build/rules_proto/BUILD.bazel b/build/rules_proto/BUILD.bazel new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build/rules_proto/module_dot_bazel.patch b/build/rules_proto/module_dot_bazel.patch new file mode 100644 index 00000000000..f9c278a9827 --- /dev/null +++ b/build/rules_proto/module_dot_bazel.patch @@ -0,0 +1,14 @@ +--- MODULE.bazel ++++ MODULE.bazel +@@ -2,10 +2,10 @@ + bazel_dep(name = "rules_license", version = "0.0.7") + bazel_dep(name = "bazel_skylib", version = "1.3.0") + bazel_dep(name = "bazel_features", version = "1.2.0") ++bazel_dep(name = "rules_cc", version = "0.0.1") + + # Dependencies needed in tests + bazel_dep(name = "stardoc", version = "0.5.6", dev_dependency = True, repo_name = "io_bazel_stardoc") +-bazel_dep(name = "rules_cc", version = "0.0.1", dev_dependency = True) + bazel_dep(name = "googletest", version = "1.11.0", dev_dependency = True, repo_name = "com_google_googletest") + bazel_dep(name = "protobuf", version = "23.1", dev_dependency = True, repo_name = "com_google_protobuf") + bazel_dep(name = "platforms", version = "0.0.8", dev_dependency = True) diff --git a/docs/gke/kingdom-deployment.md b/docs/gke/kingdom-deployment.md index 8b1576b242a..195a25a9ec2 100644 --- a/docs/gke/kingdom-deployment.md +++ b/docs/gke/kingdom-deployment.md @@ -214,14 +214,16 @@ within the Kustomization directory. ## Customize the K8s configMap Configuration that may frequently change is stored in a K8s configMap. The `dev` -configuration uses one named `config-files` containing the file -`authority_key_identifier_to_principal_map.textproto`. +configuration uses one named `config-files` containing the following files: -Place this file in the `src/main/k8s/dev/config_files/` path within the -Kustomization directory. +* `authority_key_identifier_to_principal_map.textproto` + * See [Creating Resources](../operations/creating-resources.md) +* `known_event_group_metadata_type_set.pb` + * Protobuf `FileDescriptorSet` containing known `EventGroup` metadata + types. -See [Creating Resources](../operations/creating-resources.md) for information on -this file format. +Place these files in the `src/main/k8s/dev/config_files/` path within the +Kustomization directory. ## Apply the K8s Kustomization diff --git a/docs/gke/reporting-server-deployment.md b/docs/gke/reporting-server-deployment.md index 0badde25784..68d8d0e7b0c 100644 --- a/docs/gke/reporting-server-deployment.md +++ b/docs/gke/reporting-server-deployment.md @@ -294,16 +294,17 @@ secretGenerator: ## Customize the K8s ConfigMap Configuration that may frequently change is stored in a K8s configMap. The `dev` -configuration uses one named `config-files`, which contains configuration files -in -[protobuf text format](https://developers.google.com/protocol-buffers/docs/text-format-spec). - -* `authority_key_identifier_to_principal_map.textproto` - - [`AuthorityKeyToPrincipalMap`](../../src/main/proto/wfa/measurement/config/authority_key_to_principal_map.proto) -* `encryption_key_pair_config.textproto` - - [`EncryptionKeyPairConfig`](../../src/main/proto/wfa/measurement/config/reporting/encryption_key_pair_config.proto) -* `measurement_spec_config.textproto` - - [`MeasurementSpecConfig`](../../src/main/proto/wfa/measurement/config/reporting/measurement_spec_config.proto) +configuration uses one named `config-files`. + +* `authority_key_identifier_to_principal_map.textproto` + * [`AuthorityKeyToPrincipalMap`](../../src/main/proto/wfa/measurement/config/authority_key_to_principal_map.proto) +* `encryption_key_pair_config.textproto` + * [`EncryptionKeyPairConfig`](../../src/main/proto/wfa/measurement/config/reporting/encryption_key_pair_config.proto) +* `measurement_spec_config.textproto` + * [`MeasurementSpecConfig`](../../src/main/proto/wfa/measurement/config/reporting/measurement_spec_config.proto) +* `known_event_group_metadata_type_set.pb` + * Protobuf `FileDescriptorSet` containing known `EventGroup` metadata + types. Place these files into the `src/main/k8s/dev/reporting_config_files/` path within the Kustomization directory. diff --git a/docs/gke/reporting-v2-server-deployment.md b/docs/gke/reporting-v2-server-deployment.md index 226d1bd24b0..6242b853a94 100644 --- a/docs/gke/reporting-v2-server-deployment.md +++ b/docs/gke/reporting-v2-server-deployment.md @@ -24,7 +24,7 @@ free to use whichever you prefer. - 1 Kubernetes configmap - `config-files` - 2 Kubernetes services - - `postgres-internal-reporting-server` (Cluster IP) + - `postgres-internal-reporting-server` (Cluster IP) - `reporting-v2alpha-public-api-server` (External load balancer) - 2 Kubernetes deployments - `postgres-internal-reporting-server-deployment` @@ -153,16 +153,16 @@ In order to use the IAM service account that we created earlier from our cluster, we need to create a K8s ServiceAccount and give it access to that IAM service account. -For example, to create a K8s ServiceAccount named `internal-reporting-v2-server`, -run +For example, to create a K8s ServiceAccount named +`internal-reporting-v2-server`, run ```shell kubectl create serviceaccount internal-reporting-v2-server ``` Supposing the IAM service account you created in a previous step is named -`reporting-v2-internal` within the `halo-cmm-dev` project. You'll need to allow the -K8s service account to impersonate it +`reporting-v2-internal` within the `halo-cmm-dev` project. You'll need to allow +the K8s service account to impersonate it ```shell gcloud iam service-accounts add-iam-policy-binding \ @@ -259,8 +259,8 @@ Generate the archive: bazel build //src/main/k8s/testing/secretfiles:archive ``` -Extract the generated archive to the `src/main/k8s/dev/reporting_v2_secrets/` path -within the Kustomization directory. +Extract the generated archive to the `src/main/k8s/dev/reporting_v2_secrets/` +path within the Kustomization directory. ### Measurement Consumer config @@ -273,8 +273,8 @@ Contents: ### Generator -Place the above files into the `src/main/k8s/dev/reporting_v2_secrets/` path within -the Kustomization directory. +Place the above files into the `src/main/k8s/dev/reporting_v2_secrets/` path +within the Kustomization directory. Create a `kustomization.yaml` file in that path with the following content, substituting the names of your own keys: @@ -297,17 +297,17 @@ secretGenerator: ## Customize the K8s ConfigMap Configuration that may frequently change is stored in a K8s configMap. The `dev` -configuration uses one named `config-files`, which contains configuration files -in -[protobuf text format](https://developers.google.com/protocol-buffers/docs/text-format-spec). - -* `authority_key_identifier_to_principal_map.textproto` - - [`AuthorityKeyToPrincipalMap`](../../src/main/proto/wfa/measurement/config/authority_key_to_principal_map.proto) -* `encryption_key_pair_config.textproto` - - [`EncryptionKeyPairConfig`](../../src/main/proto/wfa/measurement/config/reporting/encryption_key_pair_config.proto) -* `metric_spec_config.textproto` - - [`MetricSpecConfig`](../../src/main/proto/wfa/measurement/config/reporting/metric_spec_config.proto) - +configuration uses one named `config-files`. + +* `authority_key_identifier_to_principal_map.textproto` + * [`AuthorityKeyToPrincipalMap`](../../src/main/proto/wfa/measurement/config/authority_key_to_principal_map.proto) +* `encryption_key_pair_config.textproto` + * [`EncryptionKeyPairConfig`](../../src/main/proto/wfa/measurement/config/reporting/encryption_key_pair_config.proto) +* `metric_spec_config.textproto` + * [`MetricSpecConfig`](../../src/main/proto/wfa/measurement/config/reporting/metric_spec_config.proto) +* `known_event_group_metadata_type_set.pb` + * Protobuf `FileDescriptorSet` containing known `EventGroup` metadata + types. Place these files into the `src/main/k8s/dev/reporting_v2_config_files/` path within the Kustomization directory. diff --git a/src/main/k8s/dev/BUILD.bazel b/src/main/k8s/dev/BUILD.bazel index bac8ecf9606..e29320e0f23 100644 --- a/src/main/k8s/dev/BUILD.bazel +++ b/src/main/k8s/dev/BUILD.bazel @@ -82,12 +82,17 @@ kustomization_dir( kustomization_dir( name = "config_files", - srcs = ["config_files_kustomization.yaml"], + testonly = True, + srcs = [ + "config_files_kustomization.yaml", + "//src/main/k8s/testing/secretfiles:known_event_group_metadata_type_set", + ], renames = {"config_files_kustomization.yaml": "kustomization.yaml"}, ) kustomization_dir( name = "kingdom", + testonly = True, srcs = [ "resource_requirements.yaml", ":kingdom_gke", @@ -426,7 +431,11 @@ cue_dump( kustomization_dir( name = "reporting_config_files", - srcs = ["reporting_config_files_kustomization.yaml"], + testonly = True, + srcs = [ + "reporting_config_files_kustomization.yaml", + "//src/main/k8s/testing/secretfiles:known_event_group_metadata_type_set", + ], renames = {"reporting_config_files_kustomization.yaml": "kustomization.yaml"}, ) @@ -436,6 +445,7 @@ kustomization_dir( kustomization_dir( name = "reporting", + testonly = True, srcs = [ "resource_requirements.yaml", ":reporting_gke", @@ -472,7 +482,11 @@ cue_dump( kustomization_dir( name = "reporting_v2_config_files", - srcs = ["reporting_v2_config_files_kustomization.yaml"], + testonly = True, + srcs = [ + "reporting_v2_config_files_kustomization.yaml", + "//src/main/k8s/testing/secretfiles:known_event_group_metadata_type_set", + ], renames = {"reporting_v2_config_files_kustomization.yaml": "kustomization.yaml"}, ) @@ -482,6 +496,7 @@ kustomization_dir( kustomization_dir( name = "reporting_v2", + testonly = True, srcs = [ "resource_requirements.yaml", ":reporting_v2_gke", diff --git a/src/main/k8s/dev/config_files_kustomization.yaml b/src/main/k8s/dev/config_files_kustomization.yaml index 2ffb149570a..fb191e02f49 100644 --- a/src/main/k8s/dev/config_files_kustomization.yaml +++ b/src/main/k8s/dev/config_files_kustomization.yaml @@ -16,3 +16,4 @@ configMapGenerator: - name: config-files files: - authority_key_identifier_to_principal_map.textproto + - known_event_group_metadata_type_set.pb diff --git a/src/main/k8s/dev/reporting_config_files_kustomization.yaml b/src/main/k8s/dev/reporting_config_files_kustomization.yaml index 269aca98fd7..a15c366fe23 100644 --- a/src/main/k8s/dev/reporting_config_files_kustomization.yaml +++ b/src/main/k8s/dev/reporting_config_files_kustomization.yaml @@ -18,3 +18,4 @@ configMapGenerator: - authority_key_identifier_to_principal_map.textproto - encryption_key_pair_config.textproto - measurement_spec_config.textproto + - known_event_group_metadata_type_set.pb diff --git a/src/main/k8s/dev/reporting_v2_config_files_kustomization.yaml b/src/main/k8s/dev/reporting_v2_config_files_kustomization.yaml index d58b0bf5559..07593c9aa07 100644 --- a/src/main/k8s/dev/reporting_v2_config_files_kustomization.yaml +++ b/src/main/k8s/dev/reporting_v2_config_files_kustomization.yaml @@ -18,3 +18,4 @@ configMapGenerator: - authority_key_identifier_to_principal_map.textproto - encryption_key_pair_config.textproto - metric_spec_config.textproto + - known_event_group_metadata_type_set.pb diff --git a/src/main/k8s/kingdom.cue b/src/main/k8s/kingdom.cue index 16b2a790335..17f0a84f53f 100644 --- a/src/main/k8s/kingdom.cue +++ b/src/main/k8s/kingdom.cue @@ -65,6 +65,7 @@ import ("strings") _kingdom_cert_collection_file_flag: "--cert-collection-file=/var/run/secrets/files/all_root_certs.pem" _kingdom_root_cert_file_flag: "--cert-collection-file=/var/run/secrets/files/kingdom_root.pem" _akid_to_principal_map_file_flag: "--authority-key-identifier-to-principal-map-file=/etc/\(#AppName)/config-files/authority_key_identifier_to_principal_map.textproto" + _knownEventGroupMetadataTypeFlag: "--known-event-group-metadata-type=/etc/\(#AppName)/config-files/known_event_group_metadata_type_set.pb" _debug_verbose_grpc_client_logging_flag: "--debug-verbose-grpc-client-logging=\(_verboseGrpcClientLogging)" _debug_verbose_grpc_server_logging_flag: "--debug-verbose-grpc-server-logging=\(_verboseGrpcServerLogging)" @@ -131,6 +132,7 @@ import ("strings") _kingdom_tls_key_file_flag, // Internal Kingdom API server should only trust Kingdom certs. _kingdom_root_cert_file_flag, + _knownEventGroupMetadataTypeFlag, _debug_verbose_grpc_server_logging_flag, ] + Container._commonServerFlags + _spannerConfig.flags } @@ -140,6 +142,7 @@ import ("strings") args: _spannerConfig.flags } spec: template: spec: { + _mounts: "config-files": #ConfigMapMount _initContainers: { "update-kingdom-schema": _updateSchemaContainer } diff --git a/src/main/k8s/local/BUILD.bazel b/src/main/k8s/local/BUILD.bazel index fe27c8e4b5b..0a16b1a8a2d 100644 --- a/src/main/k8s/local/BUILD.bazel +++ b/src/main/k8s/local/BUILD.bazel @@ -291,6 +291,7 @@ kustomization_dir( "empty_akid_mapping_config.textproto", "empty_encryption_key_pair_config.textproto", "//src/main/k8s/testing/data:synthetic_generation_specs", + "//src/main/k8s/testing/secretfiles:known_event_group_metadata_type_set", "//src/main/k8s/testing/secretfiles:measurement_spec_config.textproto", "//src/main/k8s/testing/secretfiles:metric_spec_config.textproto", ], @@ -307,6 +308,7 @@ kustomization_dir( "config_files_kustomization.yaml", ":encryption_key_pair_config.textproto", "//src/main/k8s/testing/data:synthetic_generation_specs", + "//src/main/k8s/testing/secretfiles:known_event_group_metadata_type_set", "//src/main/k8s/testing/secretfiles:measurement_spec_config.textproto", "//src/main/k8s/testing/secretfiles:metric_spec_config.textproto", ], diff --git a/src/main/k8s/local/config_files_kustomization.yaml b/src/main/k8s/local/config_files_kustomization.yaml index 7d90a168c8b..fe114f0f15f 100644 --- a/src/main/k8s/local/config_files_kustomization.yaml +++ b/src/main/k8s/local/config_files_kustomization.yaml @@ -22,3 +22,4 @@ configMapGenerator: - synthetic_population_spec.textproto - synthetic_event_group_spec_1.textproto - synthetic_event_group_spec_2.textproto + - known_event_group_metadata_type_set.pb diff --git a/src/main/k8s/local/testing/BUILD.bazel b/src/main/k8s/local/testing/BUILD.bazel index ab97696cee6..d46ac15b4fa 100644 --- a/src/main/k8s/local/testing/BUILD.bazel +++ b/src/main/k8s/local/testing/BUILD.bazel @@ -62,6 +62,7 @@ kustomization_dir( srcs = [ "config_files_kustomization.yaml", "//src/main/k8s/testing/data:synthetic_generation_specs", + "//src/main/k8s/testing/secretfiles:known_event_group_metadata_type_set", ], renames = {"config_files_kustomization.yaml": "kustomization.yaml"}, ) diff --git a/src/main/k8s/local/testing/config_files_kustomization.yaml b/src/main/k8s/local/testing/config_files_kustomization.yaml index 361437132f9..be06e62c888 100644 --- a/src/main/k8s/local/testing/config_files_kustomization.yaml +++ b/src/main/k8s/local/testing/config_files_kustomization.yaml @@ -19,3 +19,4 @@ configMapGenerator: - synthetic_population_spec.textproto - synthetic_event_group_spec_1.textproto - synthetic_event_group_spec_2.textproto + - known_event_group_metadata_type_set.pb diff --git a/src/main/k8s/panelmatch/local/BUILD.bazel b/src/main/k8s/panelmatch/local/BUILD.bazel index 25c1fe135ea..cf16a2fd291 100644 --- a/src/main/k8s/panelmatch/local/BUILD.bazel +++ b/src/main/k8s/panelmatch/local/BUILD.bazel @@ -92,6 +92,7 @@ kustomization_dir( name = "config_files", srcs = [ "config_files_kustomization.yaml", + "//src/main/k8s/testing/secretfiles:known_event_group_metadata_type_set", ], renames = { "config_files_kustomization.yaml": "kustomization.yaml", diff --git a/src/main/k8s/panelmatch/local/config_files_kustomization.yaml b/src/main/k8s/panelmatch/local/config_files_kustomization.yaml index cf8653ff11a..811b72be0a7 100644 --- a/src/main/k8s/panelmatch/local/config_files_kustomization.yaml +++ b/src/main/k8s/panelmatch/local/config_files_kustomization.yaml @@ -15,4 +15,5 @@ configMapGenerator: - name: config-files files: - - authority_key_identifier_to_principal_map.textproto \ No newline at end of file + - authority_key_identifier_to_principal_map.textproto + - known_event_group_metadata_type_set.pb diff --git a/src/main/k8s/reporting.cue b/src/main/k8s/reporting.cue index 209409c494a..e9b5eefb136 100644 --- a/src/main/k8s/reporting.cue +++ b/src/main/k8s/reporting.cue @@ -61,6 +61,7 @@ package k8s _encryptionKeyPairDirFlag: "--key-pair-dir=/var/run/secrets/files" _encryptionKeyPairConfigFileFlag: "--key-pair-config-file=/etc/\(#AppName)/config-files/encryption_key_pair_config.textproto" _measurementSpecConfigFileFlag: "--measurement-spec-config-file=/etc/\(#AppName)/config-files/measurement_spec_config.textproto" + _knownEventGroupMetadataTypeFlag: "--known-event-group-metadata-type=/etc/\(#AppName)/config-files/known_event_group_metadata_type_set.pb" _debugVerboseGrpcClientLoggingFlag: "--debug-verbose-grpc-client-logging=\(_verboseGrpcClientLogging)" _debugVerboseGrpcServerLoggingFlag: "--debug-verbose-grpc-server-logging=\(_verboseGrpcServerLogging)" @@ -116,6 +117,7 @@ package k8s _encryptionKeyPairDirFlag, _encryptionKeyPairConfigFileFlag, _measurementSpecConfigFileFlag, + _knownEventGroupMetadataTypeFlag, "--port=8443", "--health-port=8080", "--event-group-metadata-descriptor-cache-duration=1h", diff --git a/src/main/k8s/reporting_v2.cue b/src/main/k8s/reporting_v2.cue index 68f8b207d60..a63768bdec1 100644 --- a/src/main/k8s/reporting_v2.cue +++ b/src/main/k8s/reporting_v2.cue @@ -67,6 +67,7 @@ package k8s _encryptionKeyPairDirFlag: "--key-pair-dir=/var/run/secrets/files" _encryptionKeyPairConfigFileFlag: "--key-pair-config-file=/etc/\(#AppName)/config-files/encryption_key_pair_config.textproto" _metricSpecConfigFileFlag: "--metric-spec-config-file=/etc/\(#AppName)/config-files/metric_spec_config.textproto" + _knownEventGroupMetadataTypeFlag: "--known-event-group-metadata-type=/etc/\(#AppName)/config-files/known_event_group_metadata_type_set.pb" _debugVerboseGrpcClientLoggingFlag: "--debug-verbose-grpc-client-logging=\(_verboseGrpcClientLogging)" _debugVerboseGrpcServerLoggingFlag: "--debug-verbose-grpc-server-logging=\(_verboseGrpcServerLogging)" @@ -122,6 +123,7 @@ package k8s _encryptionKeyPairDirFlag, _encryptionKeyPairConfigFileFlag, _metricSpecConfigFileFlag, + _knownEventGroupMetadataTypeFlag, "--port=8443", "--health-port=8080", "--event-group-metadata-descriptor-cache-duration=1h", diff --git a/src/main/k8s/testing/secretfiles/BUILD.bazel b/src/main/k8s/testing/secretfiles/BUILD.bazel index 2c7527642c2..81460542db3 100644 --- a/src/main/k8s/testing/secretfiles/BUILD.bazel +++ b/src/main/k8s/testing/secretfiles/BUILD.bazel @@ -1,6 +1,7 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file") -load("//build/k8s:defs.bzl", "k8s_apply", "kustomization_dir") load("@rules_pkg//pkg:pkg.bzl", "pkg_tar") +load("@rules_proto//proto:defs.bzl", "proto_descriptor_set") +load("//build/k8s:defs.bzl", "k8s_apply", "kustomization_dir") package( default_testonly = True, @@ -123,6 +124,14 @@ genrule( tools = ["//src/main/kotlin/org/wfanet/measurement/api/v2alpha/tools:EncryptionPublicKeys"], ) +proto_descriptor_set( + name = "known_event_group_metadata_type_set", + deps = [ + "//src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing:simulator_synthetic_data_spec_proto", + "//src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing:test_metadata_message_proto", + ], +) + SECRET_FILES = [ "all_root_certs.pem", "edp_trusted_certs.pem", diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessCmmsComponents.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessCmmsComponents.kt index 6956eaabae8..b9bd4c32dd3 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessCmmsComponents.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessCmmsComponents.kt @@ -62,8 +62,8 @@ class InProcessCmmsComponents( val kingdom: InProcessKingdom = InProcessKingdom( dataServicesProvider = { kingdomDataServices }, - verboseGrpcLogging = false, REDIRECT_URI, + verboseGrpcLogging = false, ) private val duchies: List by lazy { diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessEdpSimulator.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessEdpSimulator.kt index 1f6209ec69a..d5923d60ff3 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessEdpSimulator.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessEdpSimulator.kt @@ -110,6 +110,7 @@ class InProcessEdpSimulator( 100.0f, ), trustedCertificates = trustedCertificates, + knownEventGroupMetadataTypes = listOf(SyntheticEventGroupSpec.getDescriptor().file), random = random, ) diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessKingdom.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessKingdom.kt index d516bb0003b..385427fb988 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessKingdom.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessKingdom.kt @@ -14,6 +14,7 @@ package org.wfanet.measurement.integration.common +import com.google.protobuf.Descriptors import io.grpc.Channel import java.util.concurrent.TimeUnit import java.util.logging.Logger @@ -68,12 +69,15 @@ import org.wfanet.measurement.loadtest.panelmatchresourcesetup.PanelMatchResourc /** TestRule that starts and stops all Kingdom gRPC services. */ class InProcessKingdom( dataServicesProvider: () -> DataServices, - val verboseGrpcLogging: Boolean = true, /** The open id client redirect uri when creating the authentication uri. */ private val redirectUri: String, + val verboseGrpcLogging: Boolean = true, ) : TestRule { private val kingdomDataServices by lazy { dataServicesProvider() } + val knownEventGroupMetadataTypes: Iterable + get() = kingdomDataServices.knownEventGroupMetadataTypes + private val internalApiChannel by lazy { internalDataServer.channel.withDefaultDeadline( DEFAULT_INTERNAL_DEADLINE_MILLIS, diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessLifeOfAnEventGroupIntegrationTest.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessLifeOfAnEventGroupIntegrationTest.kt index d8d3322bb9c..30a4704ee59 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessLifeOfAnEventGroupIntegrationTest.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessLifeOfAnEventGroupIntegrationTest.kt @@ -59,8 +59,8 @@ abstract class InProcessLifeOfAnEventGroupIntegrationTest { private val kingdom: InProcessKingdom = InProcessKingdom( dataServicesProvider = { kingdomDataServices }, - verboseGrpcLogging = false, REDIRECT_URI, + verboseGrpcLogging = false, ) @get:Rule diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/InProcessReportingServer.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/InProcessReportingServer.kt index 43df27aa538..2f931a8e5cb 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/InProcessReportingServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/InProcessReportingServer.kt @@ -21,7 +21,6 @@ import java.security.SecureRandom import java.security.cert.X509Certificate import java.time.Duration import java.util.logging.Logger -import kotlinx.coroutines.Dispatchers import org.junit.rules.TestRule import org.junit.runner.Description import org.junit.runners.model.Statement @@ -126,7 +125,7 @@ class InProcessReportingServer( ), EventGroup.getDescriptor(), Duration.ofSeconds(5), - Dispatchers.Default, + knownMetadataTypes = emptyList(), ) listOf( diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessLifeOfAReportIntegrationTest.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessLifeOfAReportIntegrationTest.kt index 1a749b2b3c3..1d38b1e5a88 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessLifeOfAReportIntegrationTest.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessLifeOfAReportIntegrationTest.kt @@ -37,6 +37,7 @@ import org.junit.BeforeClass import org.junit.Rule import org.junit.Test import org.junit.rules.TestRule +import org.junit.runner.Description import org.junit.runners.model.Statement import org.wfanet.measurement.api.v2alpha.DataProviderCertificateKey import org.wfanet.measurement.api.v2alpha.DataProvidersGrpcKt.DataProvidersCoroutineStub @@ -60,10 +61,8 @@ import org.wfanet.measurement.common.getRuntimePath import org.wfanet.measurement.common.testing.ProviderRule import org.wfanet.measurement.common.testing.chainRulesSequentially import org.wfanet.measurement.common.toInterval -import org.wfanet.measurement.config.reporting.EncryptionKeyPairConfig import org.wfanet.measurement.config.reporting.EncryptionKeyPairConfigKt.keyPair import org.wfanet.measurement.config.reporting.EncryptionKeyPairConfigKt.principalKeyPairs -import org.wfanet.measurement.config.reporting.MeasurementConsumerConfig import org.wfanet.measurement.config.reporting.encryptionKeyPairConfig import org.wfanet.measurement.config.reporting.measurementConsumerConfig import org.wfanet.measurement.consent.client.dataprovider.encryptMetadata @@ -143,53 +142,68 @@ abstract class InProcessLifeOfAReportIntegrationTest( abstract val internalReportingServerServices: InternalReportingServer.Services - private val reportingServer: InProcessReportingServer by lazy { - val encryptionKeyPairConfigGenerator: () -> EncryptionKeyPairConfig = { - val measurementConsumerData = inProcessCmmsComponents.getMeasurementConsumerData() + private val reportingServerRule = + object : TestRule { + lateinit var reportingServer: InProcessReportingServer + private set + + private fun buildReportingServer(): InProcessReportingServer { + val measurementConsumerData = inProcessCmmsComponents.getMeasurementConsumerData() + val measurementConsumer = runBlocking { + publicKingdomMeasurementConsumersClient + .withAuthenticationKey(measurementConsumerData.apiAuthenticationKey) + .getMeasurementConsumer( + getMeasurementConsumerRequest { name = measurementConsumerData.name } + ) + } - encryptionKeyPairConfig { - principalKeyPairs += principalKeyPairs { - principal = measurementConsumerData.name - keyPairs += keyPair { - publicKeyFile = "mc_enc_public.tink" - privateKeyFile = "mc_enc_private.tink" + val encryptionKeyPairConfig = encryptionKeyPairConfig { + principalKeyPairs += principalKeyPairs { + principal = measurementConsumerData.name + keyPairs += keyPair { + publicKeyFile = "mc_enc_public.tink" + privateKeyFile = "mc_enc_private.tink" + } } } - } - } - - val measurementConsumerConfigGenerator: suspend () -> MeasurementConsumerConfig = { - val measurementConsumerData = inProcessCmmsComponents.getMeasurementConsumerData() + val measurementConsumerConfig = measurementConsumerConfig { + apiKey = measurementConsumerData.apiAuthenticationKey + signingCertificateName = measurementConsumer.certificate + signingPrivateKeyPath = MC_SIGNING_PRIVATE_KEY_PATH + } - val measurementConsumer = - publicKingdomMeasurementConsumersClient - .withAuthenticationKey(measurementConsumerData.apiAuthenticationKey) - .getMeasurementConsumer( - getMeasurementConsumerRequest { name = measurementConsumerData.name } - ) + return InProcessReportingServer( + internalReportingServerServices, + inProcessCmmsComponents.kingdom.publicApiChannel, + encryptionKeyPairConfig, + SECRETS_DIR, + measurementConsumerConfig, + TRUSTED_CERTIFICATES, + inProcessCmmsComponents.kingdom.knownEventGroupMetadataTypes, + verboseGrpcLogging = false, + ) + } - measurementConsumerConfig { - apiKey = measurementConsumerData.apiAuthenticationKey - signingCertificateName = measurementConsumer.certificate - signingPrivateKeyPath = MC_SIGNING_PRIVATE_KEY_PATH + override fun apply(base: Statement, description: Description): Statement { + return object : Statement() { + override fun evaluate() { + reportingServer = buildReportingServer() + reportingServer.apply(base, description) + } + } } } - InProcessReportingServer( - internalReportingServerServices, - { inProcessCmmsComponents.kingdom.publicApiChannel }, - encryptionKeyPairConfigGenerator, - SECRETS_DIR, - measurementConsumerConfigGenerator, - TRUSTED_CERTIFICATES, - verboseGrpcLogging = false, - ) - } + private val reportingServer: InProcessReportingServer + get() = reportingServerRule.reportingServer @get:Rule - val ruleChain: TestRule by lazy { - chainRulesSequentially(inProcessCmmsComponents, inProcessCmmsComponentsStartup, reportingServer) - } + val ruleChain: TestRule = + chainRulesSequentially( + inProcessCmmsComponents, + inProcessCmmsComponentsStartup, + reportingServerRule, + ) private val publicKingdomMeasurementConsumersClient by lazy { MeasurementConsumersCoroutineStub(inProcessCmmsComponents.kingdom.publicApiChannel) diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessReportingServer.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessReportingServer.kt index fac39d73fef..42e4c4d3141 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessReportingServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessReportingServer.kt @@ -17,6 +17,7 @@ package org.wfanet.measurement.integration.common.reporting.v2 import com.google.protobuf.ByteString +import com.google.protobuf.Descriptors import com.google.protobuf.util.Durations import io.grpc.Channel import io.grpc.Status @@ -77,31 +78,26 @@ import org.wfanet.measurement.reporting.v2alpha.MetricsGrpcKt.MetricsCoroutineSt /** TestRule that starts and stops all Reporting Server gRPC services. */ class InProcessReportingServer( private val internalReportingServerServices: InternalReportingServer.Services, - private val publicKingdomChannelGenerator: () -> Channel, - private val encryptionKeyPairConfigGenerator: () -> EncryptionKeyPairConfig, + private val kingdomPublicApiChannel: Channel, + private val encryptionKeyPairConfig: EncryptionKeyPairConfig, private val signingPrivateKeyDir: File, - private val measurementConsumerConfigGenerator: suspend () -> MeasurementConsumerConfig, + private val measurementConsumerConfig: MeasurementConsumerConfig, private val trustedCertificates: Map, + private val knownEventGroupMetadataTypes: Iterable, private val verboseGrpcLogging: Boolean = true, ) : TestRule { - private val publicKingdomMeasurementConsumersClient by lazy { - PublicKingdomMeasurementConsumersCoroutineStub(publicKingdomChannelGenerator()) - } - private val publicKingdomMeasurementsClient by lazy { - PublicKingdomMeasurementsCoroutineStub(publicKingdomChannelGenerator()) - } - private val publicKingdomCertificatesClient by lazy { - PublicKingdomCertificatesCoroutineStub(publicKingdomChannelGenerator()) - } - private val publicKingdomDataProvidersClient by lazy { - PublicKingdomDataProvidersCoroutineStub(publicKingdomChannelGenerator()) - } - private val publicKingdomEventGroupMetadataDescriptorsClient by lazy { - PublicKingdomEventGroupMetadataDescriptorsCoroutineStub(publicKingdomChannelGenerator()) - } - private val publicKingdomEventGroupsClient by lazy { - PublicKingdomEventGroupsCoroutineStub(publicKingdomChannelGenerator()) - } + private val publicKingdomMeasurementConsumersClient = + PublicKingdomMeasurementConsumersCoroutineStub(kingdomPublicApiChannel) + private val publicKingdomMeasurementsClient = + PublicKingdomMeasurementsCoroutineStub(kingdomPublicApiChannel) + private val publicKingdomCertificatesClient = + PublicKingdomCertificatesCoroutineStub(kingdomPublicApiChannel) + private val publicKingdomDataProvidersClient = + PublicKingdomDataProvidersCoroutineStub(kingdomPublicApiChannel) + private val publicKingdomEventGroupMetadataDescriptorsClient = + PublicKingdomEventGroupMetadataDescriptorsCoroutineStub(kingdomPublicApiChannel) + private val publicKingdomEventGroupsClient = + PublicKingdomEventGroupsCoroutineStub(kingdomPublicApiChannel) private val internalApiChannel by lazy { internalReportingServer.channel } private val internalMeasurementConsumersClient by lazy { @@ -136,8 +132,6 @@ class InProcessReportingServer( runBlocking { logger.info("Building Reporting Server's public API services") - val encryptionKeyPairConfig = encryptionKeyPairConfigGenerator() - val encryptionKeyPairStore = InMemoryEncryptionKeyPairStore( encryptionKeyPairConfig.principalKeyPairsList.associateBy( @@ -153,7 +147,6 @@ class InProcessReportingServer( ) ) - val measurementConsumerConfig = measurementConsumerConfigGenerator() val measurementConsumerName = MeasurementConsumerKey( MeasurementConsumerCertificateKey.fromName( @@ -187,7 +180,7 @@ class InProcessReportingServer( ), EventGroup.getDescriptor(), Duration.ofSeconds(5), - Dispatchers.Default, + knownEventGroupMetadataTypes, ) metricSpecConfig = METRIC_SPEC_CONFIG diff --git a/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/BUILD.bazel index 88985804cd4..470892f1dd1 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/BUILD.bazel @@ -6,14 +6,14 @@ package( ) kt_jvm_library( - name = "in_process_kingdom", - srcs = [ - "InProcessKingdom.kt", - ], + name = "kingdom_data_services_provider_rule", + srcs = ["KingdomDataServicesProviderRule.kt"], deps = [ - "//src/main/kotlin/org/wfanet/measurement/integration/common:in_process_kingdom", + "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/service:data_services", "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner:services", "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing", + "//src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing:simulator_synthetic_data_spec_kt_jvm_proto", + "//src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing:test_metadata_message_kt_jvm_proto", "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common/testing", "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/spanner/testing", ], diff --git a/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/InProcessKingdom.kt b/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/InProcessKingdom.kt deleted file mode 100644 index bbf07dba4cc..00000000000 --- a/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/InProcessKingdom.kt +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2021 The Cross-Media Measurement Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.wfanet.measurement.integration.deploy.gcloud - -import java.time.Clock -import org.wfanet.measurement.common.identity.RandomIdGenerator -import org.wfanet.measurement.gcloud.spanner.testing.SpannerEmulatorDatabaseRule -import org.wfanet.measurement.integration.common.InProcessKingdom -import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.SpannerDataServices -import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.testing.Schemata - -private const val REDIRECT_URI = "https://localhost:2048" - -fun buildKingdomSpannerEmulatorDatabaseRule(): SpannerEmulatorDatabaseRule { - return SpannerEmulatorDatabaseRule(Schemata.KINGDOM_CHANGELOG_PATH) -} - -fun buildSpannerInProcessKingdom( - databaseRule: SpannerEmulatorDatabaseRule, - clock: Clock = Clock.systemUTC(), - verboseGrpcLogging: Boolean = false, -): InProcessKingdom { - return InProcessKingdom( - dataServicesProvider = { - SpannerDataServices(clock, RandomIdGenerator(clock), databaseRule.databaseClient) - }, - verboseGrpcLogging = verboseGrpcLogging, - REDIRECT_URI, - ) -} diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing/KingdomDataServicesProviderRule.kt b/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/KingdomDataServicesProviderRule.kt similarity index 70% rename from src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing/KingdomDataServicesProviderRule.kt rename to src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/KingdomDataServicesProviderRule.kt index 169673e56e2..e76e9b782e8 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing/KingdomDataServicesProviderRule.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/KingdomDataServicesProviderRule.kt @@ -12,16 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.wfanet.measurement.kingdom.deploy.gcloud.spanner.testing +package org.wfanet.measurement.integration.deploy.gcloud import java.time.Clock import org.junit.runner.Description import org.junit.runners.model.Statement +import org.wfanet.measurement.api.v2alpha.event_group_metadata.testing.SyntheticEventGroupSpec +import org.wfanet.measurement.api.v2alpha.event_group_metadata.testing.TestMetadataMessage import org.wfanet.measurement.common.identity.RandomIdGenerator import org.wfanet.measurement.common.testing.ProviderRule import org.wfanet.measurement.gcloud.spanner.testing.SpannerEmulatorDatabaseRule import org.wfanet.measurement.kingdom.deploy.common.service.DataServices import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.SpannerDataServices +import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.testing.Schemata class KingdomDataServicesProviderRule : ProviderRule { private val spannerDatabase = SpannerEmulatorDatabaseRule(Schemata.KINGDOM_CHANGELOG_PATH) @@ -37,10 +40,20 @@ class KingdomDataServicesProviderRule : ProviderRule { override fun evaluate() { val clock = Clock.systemUTC() dataServices = - SpannerDataServices(clock, RandomIdGenerator(clock), spannerDatabase.databaseClient) + SpannerDataServices( + clock, + RandomIdGenerator(clock), + spannerDatabase.databaseClient, + KNOWN_EVENT_GROUP_METADATA_TYPES, + ) base.evaluate() } } return spannerDatabase.apply(statement, description) } + + companion object { + private val KNOWN_EVENT_GROUP_METADATA_TYPES = + listOf(TestMetadataMessage.getDescriptor().file, SyntheticEventGroupSpec.getDescriptor().file) + } } diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/KingdomDataServer.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/KingdomDataServer.kt index 1dc07e09073..f3fc867fb7f 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/KingdomDataServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/KingdomDataServer.kt @@ -14,7 +14,11 @@ package org.wfanet.measurement.kingdom.deploy.common.server +import com.google.protobuf.DescriptorProtos +import com.google.protobuf.Descriptors +import java.io.File import kotlinx.coroutines.runInterruptible +import org.wfanet.measurement.common.ProtoReflection import org.wfanet.measurement.common.grpc.CommonServer import org.wfanet.measurement.common.identity.DuchyInfo import org.wfanet.measurement.common.identity.DuchyInfoFlags @@ -39,6 +43,28 @@ abstract class KingdomDataServer : Runnable { @CommandLine.Mixin private lateinit var roLlv2ProtocolConfigFlags: RoLlv2ProtocolConfigFlags + @CommandLine.Option( + names = ["--known-event-group-metadata-type"], + description = + [ + "File path to FileDescriptorSet containing known EventGroup metadata types.", + "This is in addition to standard protobuf well-known types.", + "Can be specified multiple times.", + ], + required = false, + defaultValue = "", + ) + private fun setKnownEventGroupMetadataTypes(fileDescriptorSetFiles: List) { + val fileDescriptorSets = + fileDescriptorSetFiles.map { file -> + file.inputStream().use { input -> DescriptorProtos.FileDescriptorSet.parseFrom(input) } + } + knownEventGroupMetadataTypes = ProtoReflection.buildFileDescriptors(fileDescriptorSets) + } + + protected lateinit var knownEventGroupMetadataTypes: List + private set + protected suspend fun run(dataServices: DataServices) { DuchyInfo.initializeFromFlags(duchyInfoFlags) DuchyIds.initializeFromFlags(duchyIdsFlags) diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/service/DataServices.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/service/DataServices.kt index 1cb7afeace5..03ccf1f87cd 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/service/DataServices.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/service/DataServices.kt @@ -14,6 +14,7 @@ package org.wfanet.measurement.kingdom.deploy.common.service +import com.google.protobuf.Descriptors import io.grpc.BindableService import kotlin.reflect.full.declaredMemberProperties import org.wfanet.measurement.internal.kingdom.AccountsGrpcKt.AccountsCoroutineImplBase @@ -42,9 +43,15 @@ import org.wfanet.measurement.internal.kingdom.RecurringExchangesGrpcKt.Recurrin import org.wfanet.measurement.internal.kingdom.RequisitionsGrpcKt.RequisitionsCoroutineImplBase interface DataServices { - /** Builds a list of all the Kingdom's internal data-layer services. */ fun buildDataServices(): KingdomDataServices + + /** + * Known types for EventGroup metadata. + * + * This is in addition to standard protobuf well-known types. + */ + val knownEventGroupMetadataTypes: Iterable } data class KingdomDataServices( diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/server/SpannerKingdomDataServer.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/server/SpannerKingdomDataServer.kt index f6ed3413cd5..9bd00eff4d4 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/server/SpannerKingdomDataServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/server/SpannerKingdomDataServer.kt @@ -40,7 +40,7 @@ class SpannerKingdomDataServer : KingdomDataServer() { val idGenerator = RandomIdGenerator(clock) val client = spanner.databaseClient - run(SpannerDataServices(clock, idGenerator, client)) + run(SpannerDataServices(clock, idGenerator, client, knownEventGroupMetadataTypes)) } } } diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/SpannerDataServices.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/SpannerDataServices.kt index f1fe5d3f1bc..2e7c0e8040d 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/SpannerDataServices.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/SpannerDataServices.kt @@ -14,6 +14,7 @@ package org.wfanet.measurement.kingdom.deploy.gcloud.spanner +import com.google.protobuf.Descriptors import java.time.Clock import org.wfanet.measurement.common.identity.IdGenerator import org.wfanet.measurement.gcloud.spanner.AsyncDatabaseClient @@ -24,6 +25,7 @@ class SpannerDataServices( private val clock: Clock, private val idGenerator: IdGenerator, private val client: AsyncDatabaseClient, + override val knownEventGroupMetadataTypes: Iterable = emptyList(), ) : DataServices { override fun buildDataServices(): KingdomDataServices { return KingdomDataServices( @@ -32,7 +34,11 @@ class SpannerDataServices( SpannerCertificatesService(idGenerator, client), SpannerDataProvidersService(idGenerator, client), SpannerModelProvidersService(idGenerator, client), - SpannerEventGroupMetadataDescriptorsService(idGenerator, client), + SpannerEventGroupMetadataDescriptorsService( + idGenerator, + client, + knownEventGroupMetadataTypes, + ), SpannerEventGroupsService(idGenerator, client), SpannerMeasurementConsumersService(idGenerator, client), SpannerMeasurementsService(idGenerator, client), diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/SpannerEventGroupMetadataDescriptorsService.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/SpannerEventGroupMetadataDescriptorsService.kt index 09f94f6a559..c3aba6a6703 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/SpannerEventGroupMetadataDescriptorsService.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/SpannerEventGroupMetadataDescriptorsService.kt @@ -14,9 +14,12 @@ package org.wfanet.measurement.kingdom.deploy.gcloud.spanner +import com.google.protobuf.DescriptorProtos +import com.google.protobuf.Descriptors import io.grpc.Status import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.map +import org.wfanet.measurement.common.ProtoReflection import org.wfanet.measurement.common.grpc.failGrpc import org.wfanet.measurement.common.grpc.grpcRequire import org.wfanet.measurement.common.identity.IdGenerator @@ -27,6 +30,7 @@ import org.wfanet.measurement.internal.kingdom.GetEventGroupMetadataDescriptorRe import org.wfanet.measurement.internal.kingdom.StreamEventGroupMetadataDescriptorsRequest import org.wfanet.measurement.internal.kingdom.UpdateEventGroupMetadataDescriptorRequest import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.DataProviderNotFoundException +import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.EventGroupMetadataDescriptorAlreadyExistsWithTypeException import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.EventGroupMetadataDescriptorNotFoundException import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.KingdomInternalException import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.queries.StreamEventGroupMetadataDescriptors @@ -37,14 +41,24 @@ import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.writers.UpdateEventG class SpannerEventGroupMetadataDescriptorsService( private val idGenerator: IdGenerator, private val client: AsyncDatabaseClient, + knownMetadataTypes: Iterable, ) : EventGroupMetadataDescriptorsCoroutineImplBase() { + private val allKnownMetadataTypes: Set = + knownMetadataTypes.asSequence().plus(ProtoReflection.WELL_KNOWN_TYPES).toSet() + override suspend fun createEventGroupMetadataDescriptor( request: EventGroupMetadataDescriptor ): EventGroupMetadataDescriptor { try { - return CreateEventGroupMetadataDescriptor(request).execute(client, idGenerator) + return CreateEventGroupMetadataDescriptor( + request, + getProtobufTypeNames(request.details.descriptorSet), + ) + .execute(client, idGenerator) } catch (e: DataProviderNotFoundException) { throw e.asStatusRuntimeException(Status.Code.NOT_FOUND, "DataProvider not found.") + } catch (e: EventGroupMetadataDescriptorAlreadyExistsWithTypeException) { + throw e.asStatusRuntimeException(Status.Code.ALREADY_EXISTS) } catch (e: KingdomInternalException) { throw e.asStatusRuntimeException(Status.Code.INTERNAL, "Unexpected internal error") } @@ -73,13 +87,18 @@ class SpannerEventGroupMetadataDescriptorsService( "ExternalEventGroupMetadataDescriptorId unspecified" } try { - return UpdateEventGroupMetadataDescriptor(request.eventGroupMetadataDescriptor) + return UpdateEventGroupMetadataDescriptor( + request.eventGroupMetadataDescriptor, + getProtobufTypeNames(request.eventGroupMetadataDescriptor.details.descriptorSet), + ) .execute(client, idGenerator) } catch (e: EventGroupMetadataDescriptorNotFoundException) { throw e.asStatusRuntimeException( Status.Code.NOT_FOUND, "EventGroupMetadataDescriptor not found.", ) + } catch (e: EventGroupMetadataDescriptorAlreadyExistsWithTypeException) { + throw e.asStatusRuntimeException(Status.Code.ALREADY_EXISTS) } catch (e: KingdomInternalException) { throw e.asStatusRuntimeException(Status.Code.INTERNAL, "Unexpected internal error") } @@ -94,4 +113,16 @@ class SpannerEventGroupMetadataDescriptorsService( .execute(client.singleUse()) .map { it.eventGroupMetadataDescriptor } } + + /** + * Returns the protobuf type names of types defined in [fileDescriptorSet], excluding those + * defined in [allKnownMetadataTypes]. + */ + private fun getProtobufTypeNames( + fileDescriptorSet: DescriptorProtos.FileDescriptorSet + ): List { + return ProtoReflection.buildDescriptors(listOf(fileDescriptorSet), allKnownMetadataTypes).map { + it.fullName + } + } } diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/common/KingdomInternalException.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/common/KingdomInternalException.kt index 9d05c83ae38..33ef66434a7 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/common/KingdomInternalException.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/common/KingdomInternalException.kt @@ -580,6 +580,19 @@ class EventGroupMetadataDescriptorNotFoundException( ) } +class EventGroupMetadataDescriptorAlreadyExistsWithTypeException( + message: String = "EventGroupMetadataDescriptor with same protobuf type already exists", + cause: Throwable? = null, +) : + KingdomInternalException( + ErrorCode.EVENT_GROUP_METADATA_DESCRIPTOR_ALREADY_EXISTS_WITH_TYPE, + message, + cause, + ) { + override val context: Map + get() = emptyMap() +} + class RecurringExchangeNotFoundException( val externalRecurringExchangeId: ExternalId, provideDescription: () -> String = { "RecurringExchange not found" }, diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing/BUILD.bazel index 7a387821b93..302c1160240 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing/BUILD.bazel @@ -14,7 +14,6 @@ kt_jvm_library( srcs = glob(["*.kt"]), resources = ["//src/main/resources/kingdom/spanner"], deps = [ - "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner:services", "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common", "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/spanner/testing", ], diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/writers/CreateEventGroupMetadataDescriptor.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/writers/CreateEventGroupMetadataDescriptor.kt index 424885ddf00..fb6ca2335c0 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/writers/CreateEventGroupMetadataDescriptor.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/writers/CreateEventGroupMetadataDescriptor.kt @@ -14,6 +14,8 @@ package org.wfanet.measurement.kingdom.deploy.gcloud.spanner.writers +import com.google.cloud.spanner.ErrorCode +import com.google.cloud.spanner.SpannerException import org.wfanet.measurement.common.identity.ExternalId import org.wfanet.measurement.common.identity.InternalId import org.wfanet.measurement.gcloud.spanner.bufferInsertMutation @@ -21,6 +23,7 @@ import org.wfanet.measurement.gcloud.spanner.set import org.wfanet.measurement.gcloud.spanner.setJson import org.wfanet.measurement.internal.kingdom.EventGroupMetadataDescriptor import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.DataProviderNotFoundException +import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.EventGroupMetadataDescriptorAlreadyExistsWithTypeException import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.KingdomInternalException import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.readers.DataProviderReader import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.readers.EventGroupMetadataDescriptorReader @@ -28,12 +31,13 @@ import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.readers.EventGroupMe /** * Creates a EventGroupMetadataDescriptor in the database. * - * Throws a subclass of [KingdomInternalException] on [execute]. - * - * @throws [DataProviderNotFoundException] DataProvider not found + * Throws one of the following subclasses of [KingdomInternalException] on [execute]: + * * [DataProviderNotFoundException] + * * [EventGroupMetadataDescriptorAlreadyExistsWithTypeException] */ class CreateEventGroupMetadataDescriptor( - private val eventGroupMetadataDescriptor: EventGroupMetadataDescriptor + private val eventGroupMetadataDescriptor: EventGroupMetadataDescriptor, + private val protobufTypeNames: List, ) : SpannerWriter() { init { require(eventGroupMetadataDescriptor.externalEventGroupMetadataDescriptorId == 0L) @@ -58,6 +62,14 @@ class CreateEventGroupMetadataDescriptor( return createNewEventGroupMetadataDescriptor(dataProviderId) } + override suspend fun handleSpannerException(e: SpannerException): EventGroupMetadataDescriptor? { + if (e.errorCode == ErrorCode.ALREADY_EXISTS) { + throw EventGroupMetadataDescriptorAlreadyExistsWithTypeException(cause = e) + } else { + throw e + } + } + private fun TransactionScope.createNewEventGroupMetadataDescriptor( dataProviderId: InternalId ): EventGroupMetadataDescriptor { @@ -76,6 +88,14 @@ class CreateEventGroupMetadataDescriptor( setJson("DescriptorDetailsJson" to eventGroupMetadataDescriptor.details) } + for (protobufTypeName in protobufTypeNames) { + transactionContext.bufferInsertMutation("EventGroupMetadataDescriptorTypes") { + set("DataProviderId" to dataProviderId) + set("EventGroupMetadataDescriptorId" to internalDescriptorId) + set("ProtobufTypeName" to protobufTypeName) + } + } + return eventGroupMetadataDescriptor .toBuilder() .setExternalEventGroupMetadataDescriptorId(externalDescriptorId.value) diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/writers/UpdateEventGroupMetadataDescriptor.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/writers/UpdateEventGroupMetadataDescriptor.kt index 7840f88cf4b..5ccf344ff02 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/writers/UpdateEventGroupMetadataDescriptor.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/writers/UpdateEventGroupMetadataDescriptor.kt @@ -14,11 +14,18 @@ package org.wfanet.measurement.kingdom.deploy.gcloud.spanner.writers +import com.google.cloud.spanner.ErrorCode +import com.google.cloud.spanner.Key +import com.google.cloud.spanner.KeySet +import com.google.cloud.spanner.Mutation +import com.google.cloud.spanner.SpannerException import org.wfanet.measurement.common.identity.ExternalId +import org.wfanet.measurement.gcloud.spanner.bufferInsertMutation import org.wfanet.measurement.gcloud.spanner.bufferUpdateMutation import org.wfanet.measurement.gcloud.spanner.set import org.wfanet.measurement.gcloud.spanner.setJson import org.wfanet.measurement.internal.kingdom.EventGroupMetadataDescriptor +import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.EventGroupMetadataDescriptorAlreadyExistsWithTypeException import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.EventGroupMetadataDescriptorNotFoundException import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.KingdomInternalException import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.readers.EventGroupMetadataDescriptorReader @@ -26,12 +33,13 @@ import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.readers.EventGroupMe /** * Update [EventGroupMetadataDescriptor] in the database. * - * Throws a subclass of [KingdomInternalException] on [execute]. - * - * @throws [EventGroupMetadataDescriptorNotFoundException] EventGroupMetadataDescriptor not found + * Throws one of the following subclasses of [KingdomInternalException] on [execute]: + * * [EventGroupMetadataDescriptorNotFoundException] + * * [EventGroupMetadataDescriptorAlreadyExistsWithTypeException] */ class UpdateEventGroupMetadataDescriptor( - private val eventGroupMetadataDescriptor: EventGroupMetadataDescriptor + private val eventGroupMetadataDescriptor: EventGroupMetadataDescriptor, + private val protobufTypeNames: List, ) : SpannerWriter() { override suspend fun TransactionScope.runTransaction(): EventGroupMetadataDescriptor { val internalMetadataDescriptorResult = @@ -46,8 +54,11 @@ class UpdateEventGroupMetadataDescriptor( ExternalId(eventGroupMetadataDescriptor.externalEventGroupMetadataDescriptorId), ) + val dataProviderId = internalMetadataDescriptorResult.internalDataProviderId + val eventGroupMetadataDescriptorId = internalMetadataDescriptorResult.internalDescriptorId + transactionContext.bufferUpdateMutation("EventGroupMetadataDescriptors") { - set("DataProviderId" to internalMetadataDescriptorResult.internalDataProviderId.value) + set("DataProviderId" to dataProviderId) set( "EventGroupMetadataDescriptorId" to internalMetadataDescriptorResult.internalDescriptorId.value @@ -61,6 +72,20 @@ class UpdateEventGroupMetadataDescriptor( setJson("DescriptorDetailsJson" to eventGroupMetadataDescriptor.details) } + transactionContext.buffer( + Mutation.delete( + "EventGroupMetadataDescriptorTypes", + KeySet.prefixRange(Key.of(dataProviderId.value, eventGroupMetadataDescriptorId.value)), + ) + ) + for (protobufTypeName in protobufTypeNames) { + transactionContext.bufferInsertMutation("EventGroupMetadataDescriptorTypes") { + set("DataProviderId" to dataProviderId) + set("EventGroupMetadataDescriptorId" to eventGroupMetadataDescriptorId) + set("ProtobufTypeName" to protobufTypeName) + } + } + return eventGroupMetadataDescriptor } @@ -68,4 +93,12 @@ class UpdateEventGroupMetadataDescriptor( EventGroupMetadataDescriptor { return eventGroupMetadataDescriptor } + + override suspend fun handleSpannerException(e: SpannerException): EventGroupMetadataDescriptor? { + if (e.errorCode == ErrorCode.ALREADY_EXISTS) { + throw EventGroupMetadataDescriptorAlreadyExistsWithTypeException(cause = e) + } else { + throw e + } + } } diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing/BUILD.bazel index 277e5253c7f..53681b28352 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing/BUILD.bazel @@ -13,6 +13,8 @@ kt_jvm_library( "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common:llv2_protocol_config", "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common:ro_llv2_protocol_config", "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/testing", + "//src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing:test_metadata_message_2_kt_jvm_proto", + "//src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing:test_metadata_message_kt_jvm_proto", "@wfa_common_jvm//imports/java/com/google/cloud/spanner", "@wfa_common_jvm//imports/java/com/google/common/truth", "@wfa_common_jvm//imports/java/com/google/common/truth/extensions/proto", diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing/EventGroupMetadataDescriptorsServiceTest.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing/EventGroupMetadataDescriptorsServiceTest.kt index 3fa8cfcdcd8..6bb44229d58 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing/EventGroupMetadataDescriptorsServiceTest.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing/EventGroupMetadataDescriptorsServiceTest.kt @@ -16,7 +16,6 @@ package org.wfanet.measurement.kingdom.service.internal.testing import com.google.common.truth.Truth.assertThat import com.google.common.truth.extensions.proto.ProtoTruth.assertThat -import com.google.protobuf.DescriptorProtos.FileDescriptorSet import io.grpc.Status import io.grpc.StatusRuntimeException import java.time.Clock @@ -30,6 +29,9 @@ import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 import org.wfanet.measurement.api.Version +import org.wfanet.measurement.api.v2alpha.event_group_metadata.testing.TestMetadataMessage +import org.wfanet.measurement.api.v2alpha.event_group_metadata.testing.TestMetadataMessage2 +import org.wfanet.measurement.common.ProtoReflection import org.wfanet.measurement.common.identity.IdGenerator import org.wfanet.measurement.common.identity.RandomIdGenerator import org.wfanet.measurement.internal.kingdom.DataProvidersGrpcKt.DataProvidersCoroutineImplBase @@ -48,7 +50,11 @@ import org.wfanet.measurement.kingdom.deploy.common.testing.DuchyIdSetter private const val RANDOM_SEED = 1 private val DETAILS = details { apiVersion = Version.V2_ALPHA.string - descriptorSet = FileDescriptorSet.getDefaultInstance() + descriptorSet = ProtoReflection.buildFileDescriptorSet(TestMetadataMessage.getDescriptor()) +} +private val DETAILS_2 = details { + apiVersion = Version.V2_ALPHA.string + descriptorSet = ProtoReflection.buildFileDescriptorSet(TestMetadataMessage2.getDescriptor()) } @RunWith(JUnit4::class) @@ -163,6 +169,33 @@ abstract class EventGroupMetadataDescriptorsServiceTest< assertThat(exception).hasMessageThat().contains("NOT_FOUND: DataProvider not found") } + @Test + fun `createEventGroupMetadataDescriptor fails for duplicate message type`() = runBlocking { + val externalDataProviderId = + population.createDataProvider(dataProvidersService).externalDataProviderId + val externalDataProviderId2 = + population.createDataProvider(dataProvidersService).externalDataProviderId + eventGroupMetadataDescriptorService.createEventGroupMetadataDescriptor( + eventGroupMetadataDescriptor { + this.externalDataProviderId = externalDataProviderId + details = DETAILS + } + ) + + val exception = + assertFailsWith { + eventGroupMetadataDescriptorService.createEventGroupMetadataDescriptor( + eventGroupMetadataDescriptor { + this.externalDataProviderId = externalDataProviderId2 + details = DETAILS + } + ) + } + + assertThat(exception.status.code).isEqualTo(Status.Code.ALREADY_EXISTS) + assertThat(exception).hasMessageThat().contains("EventGroupMetadataDescriptor") + } + @Test fun `createEventGroupMetadataDescriptor returns existing Descriptor by idempotency key`() = runBlocking { @@ -240,6 +273,43 @@ abstract class EventGroupMetadataDescriptorsServiceTest< ) } + @Test + fun `updateEventGroupMetadataDescriptor fails for duplicate message type`() = runBlocking { + val externalDataProviderId = + population.createDataProvider(dataProvidersService).externalDataProviderId + val externalDataProviderId2 = + population.createDataProvider(dataProvidersService).externalDataProviderId + eventGroupMetadataDescriptorService.createEventGroupMetadataDescriptor( + eventGroupMetadataDescriptor { + this.externalDataProviderId = externalDataProviderId + details = DETAILS + } + ) + val eventGroupMetadataDescriptor = + eventGroupMetadataDescriptorService.createEventGroupMetadataDescriptor( + eventGroupMetadataDescriptor { + this.externalDataProviderId = externalDataProviderId2 + details = DETAILS_2 + } + ) + + val exception = + assertFailsWith { + eventGroupMetadataDescriptorService.updateEventGroupMetadataDescriptor( + updateEventGroupMetadataDescriptorRequest { + this.eventGroupMetadataDescriptor = + eventGroupMetadataDescriptor.copy { + // This should error as the message type is already in use by the other EDP. + details = DETAILS + } + } + ) + } + + assertThat(exception.status.code).isEqualTo(Status.Code.ALREADY_EXISTS) + assertThat(exception).hasMessageThat().contains("EventGroupMetadataDescriptor") + } + @Test fun `streamEventGroupMetadataDescriptors returns all descriptors in order`(): Unit = runBlocking { val externalDataProviderId = @@ -257,7 +327,7 @@ abstract class EventGroupMetadataDescriptorsServiceTest< eventGroupMetadataDescriptorService.createEventGroupMetadataDescriptor( eventGroupMetadataDescriptor { this.externalDataProviderId = externalDataProviderId - details = DETAILS + details = DETAILS_2 } ) @@ -303,7 +373,7 @@ abstract class EventGroupMetadataDescriptorsServiceTest< eventGroupMetadataDescriptorService.createEventGroupMetadataDescriptor( eventGroupMetadataDescriptor { this.externalDataProviderId = externalDataProviderId - details = DETAILS + details = DETAILS_2 } ) @@ -342,7 +412,7 @@ abstract class EventGroupMetadataDescriptorsServiceTest< eventGroupMetadataDescriptorService.createEventGroupMetadataDescriptor( eventGroupMetadataDescriptor { this.externalDataProviderId = externalDataProviderId - details = DETAILS + details = DETAILS_2 } ) @@ -383,7 +453,7 @@ abstract class EventGroupMetadataDescriptorsServiceTest< eventGroupMetadataDescriptorService.createEventGroupMetadataDescriptor( eventGroupMetadataDescriptor { this.externalDataProviderId = externalDataProviderId - details = DETAILS + details = DETAILS_2 } ) diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BigQueryEdpSimulatorRunner.kt b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BigQueryEdpSimulatorRunner.kt index dc6cd7c1961..43ac7814b89 100644 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BigQueryEdpSimulatorRunner.kt +++ b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BigQueryEdpSimulatorRunner.kt @@ -18,6 +18,7 @@ import com.google.cloud.bigquery.BigQuery import com.google.cloud.bigquery.BigQueryOptions import kotlin.properties.Delegates import org.halo_cmm.uk.pilot.Event +import org.wfanet.measurement.api.v2alpha.event_group_metadata.testing.TestMetadataMessage import org.wfanet.measurement.common.commandLineMain import org.wfanet.measurement.loadtest.config.EventGroupMetadata import picocli.CommandLine @@ -76,6 +77,7 @@ class BigQueryEdpSimulatorRunner : EdpSimulatorRunner() { eventQuery, EdpSimulator.buildEventTemplates(Event.getDescriptor()), mapOf("" to EventGroupMetadata.testMetadata(publisherId)), + listOf(TestMetadataMessage.getDescriptor().file), ) } } diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/CsvEdpSimulatorRunner.kt b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/CsvEdpSimulatorRunner.kt index 590a57e0fa1..58a08ffa4e0 100644 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/CsvEdpSimulatorRunner.kt +++ b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/CsvEdpSimulatorRunner.kt @@ -16,6 +16,7 @@ package org.wfanet.measurement.loadtest.dataprovider import java.io.File import kotlin.properties.Delegates +import org.wfanet.measurement.api.v2alpha.event_group_metadata.testing.TestMetadataMessage import org.wfanet.measurement.api.v2alpha.event_templates.testing.TestEvent import org.wfanet.measurement.common.commandLineMain import org.wfanet.measurement.loadtest.config.EventGroupMetadata @@ -52,6 +53,7 @@ class CsvEdpSimulatorRunner : EdpSimulatorRunner() { eventQuery, EdpSimulator.buildEventTemplates(TestEvent.getDescriptor()), mapOf("" to EventGroupMetadata.testMetadata(publisherId)), + listOf(TestMetadataMessage.getDescriptor().file), ) } } diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulator.kt b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulator.kt index bdfd02db780..84fadee6901 100644 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulator.kt +++ b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulator.kt @@ -174,6 +174,13 @@ class EdpSimulator( private val throttler: Throttler, private val privacyBudgetManager: PrivacyBudgetManager, private val trustedCertificates: Map, + /** + * Known protobuf types for [EventGroupMetadataDescriptor]s. + * + * This is in addition to the standard + * [protobuf well-known types][ProtoReflection.WELL_KNOWN_TYPES]. + */ + private val knownEventGroupMetadataTypes: Iterable = emptyList(), private val sketchEncrypter: SketchEncrypter = SketchEncrypter.Default, private val random: Random = Random, private val logSketchDetails: Boolean = false, @@ -341,14 +348,18 @@ class EdpSimulator( private suspend fun ensureMetadataDescriptor( metadataDescriptor: Descriptors.Descriptor ): EventGroupMetadataDescriptor { - val descriptorSet = ProtoReflection.buildFileDescriptorSet(metadataDescriptor) + val descriptorSet = + ProtoReflection.buildFileDescriptorSet( + metadataDescriptor, + ProtoReflection.WELL_KNOWN_TYPES + knownEventGroupMetadataTypes, + ) val descriptorResource = try { eventGroupMetadataDescriptorsStub.createEventGroupMetadataDescriptor( createEventGroupMetadataDescriptorRequest { parent = edpData.name eventGroupMetadataDescriptor = eventGroupMetadataDescriptor { - this.descriptorSet = ProtoReflection.buildFileDescriptorSet(metadataDescriptor) + this.descriptorSet = descriptorSet } requestId = "type.googleapis.com/${metadataDescriptor.fullName}" } diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulatorRunner.kt b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulatorRunner.kt index a01fdbdc2f6..bea556a5017 100644 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulatorRunner.kt +++ b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulatorRunner.kt @@ -14,6 +14,7 @@ package org.wfanet.measurement.loadtest.dataprovider +import com.google.protobuf.Descriptors import com.google.protobuf.Message import io.grpc.ManagedChannel import java.time.Clock @@ -46,6 +47,7 @@ abstract class EdpSimulatorRunner : Runnable { eventQuery: EventQuery, eventTemplates: Iterable, metadataByReferenceIdSuffix: Map, + knownEventGroupMetadataTypes: Iterable, ) { val clientCerts = SigningCerts.fromPemFiles( @@ -112,6 +114,7 @@ abstract class EdpSimulatorRunner : Runnable { MinimumIntervalThrottler(Clock.systemUTC(), flags.throttlerMinimumInterval), createNoOpPrivacyBudgetManager(), clientCerts.trustedCertificates, + knownEventGroupMetadataTypes = knownEventGroupMetadataTypes, random = random, logSketchDetails = flags.logSketchDetails, ) diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/SyntheticGeneratorEdpSimulatorRunner.kt b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/SyntheticGeneratorEdpSimulatorRunner.kt index bff9b6b7ef4..710793420ad 100644 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/SyntheticGeneratorEdpSimulatorRunner.kt +++ b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/SyntheticGeneratorEdpSimulatorRunner.kt @@ -89,6 +89,7 @@ class SyntheticGeneratorEdpSimulatorRunner : EdpSimulatorRunner() { eventQuery, EdpSimulator.buildEventTemplates(eventMessageDescriptor), eventGroupSpecByReferenceIdSuffix, + listOf(SyntheticEventGroupSpec.getDescriptor().file), ) } diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/common/server/V1AlphaPublicApiServer.kt b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/common/server/V1AlphaPublicApiServer.kt index 2ab24692df3..affa050e7e0 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/common/server/V1AlphaPublicApiServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/common/server/V1AlphaPublicApiServer.kt @@ -15,6 +15,8 @@ package org.wfanet.measurement.reporting.deploy.common.server import com.google.protobuf.ByteString +import com.google.protobuf.DescriptorProtos +import com.google.protobuf.Descriptors import io.grpc.Channel import io.grpc.ServerServiceDefinition import java.io.File @@ -27,6 +29,7 @@ import org.wfanet.measurement.api.v2alpha.EventGroupsGrpcKt.EventGroupsCoroutine import org.wfanet.measurement.api.v2alpha.MeasurementConsumersGrpcKt.MeasurementConsumersCoroutineStub as KingdomMeasurementConsumersCoroutineStub import org.wfanet.measurement.api.v2alpha.MeasurementsGrpcKt.MeasurementsCoroutineStub as KingdomMeasurementsCoroutineStub import org.wfanet.measurement.api.withAuthenticationKey +import org.wfanet.measurement.common.ProtoReflection import org.wfanet.measurement.common.api.PrincipalLookup import org.wfanet.measurement.common.api.memoizing import org.wfanet.measurement.common.commandLineMain @@ -111,6 +114,7 @@ private fun run( .withAuthenticationKey(apiKey), EventGroup.getDescriptor(), reportingApiServerFlags.eventGroupMetadataDescriptorCacheDuration, + v1AlphaFlags.knownEventGroupMetadataTypes, Dispatchers.Default, ) @@ -190,4 +194,26 @@ private class V1AlphaFlags { ) lateinit var signingPrivateKeyStoreDir: File private set + + @CommandLine.Option( + names = ["--known-event-group-metadata-type"], + description = + [ + "File path to FileDescriptorSet containing known EventGroup metadata types.", + "This is in addition to standard protobuf well-known types.", + "Can be specified multiple times.", + ], + required = false, + defaultValue = "", + ) + private fun setKnownEventGroupMetadataTypes(fileDescriptorSetFiles: List) { + val fileDescriptorSets = + fileDescriptorSetFiles.map { file -> + file.inputStream().use { input -> DescriptorProtos.FileDescriptorSet.parseFrom(input) } + } + knownEventGroupMetadataTypes = ProtoReflection.buildFileDescriptors(fileDescriptorSets) + } + + lateinit var knownEventGroupMetadataTypes: List + private set } diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/server/V2AlphaPublicApiServer.kt b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/server/V2AlphaPublicApiServer.kt index 2db1375b855..c52a9f797bf 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/server/V2AlphaPublicApiServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/server/V2AlphaPublicApiServer.kt @@ -17,6 +17,8 @@ package org.wfanet.measurement.reporting.deploy.v2.common.server import com.google.protobuf.ByteString +import com.google.protobuf.DescriptorProtos +import com.google.protobuf.Descriptors import io.grpc.Channel import io.grpc.Server import io.grpc.ServerServiceDefinition @@ -41,6 +43,7 @@ import org.wfanet.measurement.api.v2alpha.MeasurementConsumerKey import org.wfanet.measurement.api.v2alpha.MeasurementConsumersGrpcKt.MeasurementConsumersCoroutineStub as KingdomMeasurementConsumersCoroutineStub import org.wfanet.measurement.api.v2alpha.MeasurementsGrpcKt.MeasurementsCoroutineStub as KingdomMeasurementsCoroutineStub import org.wfanet.measurement.api.withAuthenticationKey +import org.wfanet.measurement.common.ProtoReflection import org.wfanet.measurement.common.api.PrincipalLookup import org.wfanet.measurement.common.api.memoizing import org.wfanet.measurement.common.commandLineMain @@ -168,6 +171,7 @@ private fun run( .withAuthenticationKey(apiKey), EventGroup.getDescriptor(), reportingApiServerFlags.eventGroupMetadataDescriptorCacheDuration, + v2AlphaPublicServerFlags.knownEventGroupMetadataTypes, Dispatchers.Default, ) @@ -290,4 +294,26 @@ private class V2AlphaPublicServerFlags { ) lateinit var dataProviderCacheExpirationDuration: Duration private set + + @CommandLine.Option( + names = ["--known-event-group-metadata-type"], + description = + [ + "File path to FileDescriptorSet containing known EventGroup metadata types.", + "This is in addition to standard protobuf well-known types.", + "Can be specified multiple times.", + ], + required = false, + defaultValue = "", + ) + private fun setKnownEventGroupMetadataTypes(fileDescriptorSetFiles: List) { + val fileDescriptorSets = + fileDescriptorSetFiles.map { file -> + file.inputStream().use { input -> DescriptorProtos.FileDescriptorSet.parseFrom(input) } + } + knownEventGroupMetadataTypes = ProtoReflection.buildFileDescriptors(fileDescriptorSets) + } + + lateinit var knownEventGroupMetadataTypes: List + private set } diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/service/api/CelEnvProvider.kt b/src/main/kotlin/org/wfanet/measurement/reporting/service/api/CelEnvProvider.kt index e2c65f0f86c..f70e0dbfa61 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/service/api/CelEnvProvider.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/service/api/CelEnvProvider.kt @@ -16,7 +16,6 @@ package org.wfanet.measurement.reporting.service.api -import com.google.protobuf.DescriptorProtos import com.google.protobuf.Descriptors import com.google.protobuf.TypeRegistry import io.grpc.Status @@ -25,6 +24,7 @@ import java.time.Duration import java.util.logging.Level import java.util.logging.Logger import kotlin.coroutines.CoroutineContext +import kotlin.coroutines.EmptyCoroutineContext import kotlin.coroutines.coroutineContext import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope @@ -39,6 +39,7 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.time.delay +import org.jetbrains.annotations.NonBlockingExecutor import org.jetbrains.annotations.VisibleForTesting import org.projectnessie.cel.Env import org.projectnessie.cel.EnvOption @@ -65,9 +66,12 @@ class CelEnvCacheProvider( /** Protobuf descriptor of Reporting EventGroup message type. */ private val reportingEventGroupDescriptor: Descriptors.Descriptor, private val cacheRefreshInterval: Duration, - coroutineContext: CoroutineContext, + knownMetadataTypes: Iterable, + coroutineContext: @NonBlockingExecutor CoroutineContext = EmptyCoroutineContext, private val numRetriesInitialSync: Int = 3, ) : CelEnvProvider, AutoCloseable { + private val allKnownMetadataTypes: Set = + knownMetadataTypes.asSequence().plus(ProtoReflection.WELL_KNOWN_TYPES).toSet() private lateinit var typeRegistryAndEnv: CelEnvProvider.TypeRegistryAndEnv private val coroutineScope = CoroutineScope(coroutineContext + SupervisorJob()) private val initialSyncJob = CompletableDeferred() @@ -123,10 +127,11 @@ class CelEnvCacheProvider( val eventGroupMetadataDescriptors: List = getEventGroupMetadataDescriptors() - val fileDescriptorSets: List = - eventGroupMetadataDescriptors.map { it.descriptorSet } - val fileDescriptors: List = - ProtoReflection.buildDescriptors(fileDescriptorSets) + val fileDescriptors: List = + ProtoReflection.buildFileDescriptors( + eventGroupMetadataDescriptors.map { it.descriptorSet }, + allKnownMetadataTypes, + ) val env = buildCelEnvironment(fileDescriptors) val typeRegistry: TypeRegistry = buildTypeRegistry(fileDescriptors) @@ -134,11 +139,15 @@ class CelEnvCacheProvider( return CelEnvProvider.TypeRegistryAndEnv(typeRegistry, env) } - private fun buildCelEnvironment(descriptors: List): Env { + private fun buildCelEnvironment(fileDescriptors: Iterable): Env { // Build CEL ProtoTypeRegistry. - val celTypeRegistry = ProtoTypeRegistry.newRegistry() - descriptors.forEach { celTypeRegistry.registerDescriptor(it.file) } - celTypeRegistry.registerDescriptor(reportingEventGroupDescriptor.file) + val celTypeRegistry = + ProtoTypeRegistry.newRegistry().apply { + for (fileDescriptor in fileDescriptors) { + registerDescriptor(fileDescriptor) + } + registerDescriptor(reportingEventGroupDescriptor.file) + } // Build CEL Env. val env = @@ -196,14 +205,8 @@ class CelEnvCacheProvider( } } - private fun buildTypeRegistry(descriptors: List): TypeRegistry { - return TypeRegistry.newBuilder() - .apply { - for (descriptor in descriptors) { - add(descriptor) - } - } - .build() + private fun buildTypeRegistry(fileDescriptors: List): TypeRegistry { + return TypeRegistry.newBuilder().add(fileDescriptors.flatMap { it.messageTypes }).build() } /** Suspends until any in-flight sync operations are complete. */ diff --git a/src/main/kotlin/org/wfanet/panelmatch/integration/AbstractInProcessPanelMatchIntegrationTest.kt b/src/main/kotlin/org/wfanet/panelmatch/integration/AbstractInProcessPanelMatchIntegrationTest.kt index 74ce42abd9b..102f13b1a78 100644 --- a/src/main/kotlin/org/wfanet/panelmatch/integration/AbstractInProcessPanelMatchIntegrationTest.kt +++ b/src/main/kotlin/org/wfanet/panelmatch/integration/AbstractInProcessPanelMatchIntegrationTest.kt @@ -21,7 +21,6 @@ import com.google.protobuf.ByteString import com.google.protobuf.TypeRegistry import io.grpc.StatusException import java.nio.file.Path -import java.time.Clock import java.time.LocalDate import kotlinx.coroutines.CoroutineName import kotlinx.coroutines.CoroutineScope @@ -54,8 +53,8 @@ import org.wfanet.measurement.common.identity.withPrincipalName import org.wfanet.measurement.common.parseTextProto import org.wfanet.measurement.common.testing.chainRulesSequentially import org.wfanet.measurement.common.toProtoDate -import org.wfanet.measurement.integration.deploy.gcloud.buildKingdomSpannerEmulatorDatabaseRule -import org.wfanet.measurement.integration.deploy.gcloud.buildSpannerInProcessKingdom +import org.wfanet.measurement.integration.common.InProcessKingdom +import org.wfanet.measurement.integration.deploy.gcloud.KingdomDataServicesProviderRule import org.wfanet.measurement.loadtest.resourcesetup.EntityContent import org.wfanet.measurement.storage.StorageClient import org.wfanet.measurement.storage.filesystem.FileSystemStorageClient @@ -113,10 +112,9 @@ abstract class AbstractInProcessPanelMatchIntegrationTest { abstract val workflow: ExchangeWorkflow - private val databaseRule = buildKingdomSpannerEmulatorDatabaseRule() - private val inProcessKingdom by lazy { - buildSpannerInProcessKingdom(databaseRule, Clock.systemUTC(), verboseGrpcLogging = false) - } + private val kingdomDataServicesProvider = KingdomDataServicesProviderRule() + private val inProcessKingdom = + InProcessKingdom({ kingdomDataServicesProvider.value }, REDIRECT_URI) private val resourceSetup by lazy { inProcessKingdom.panelMatchResourceSetup } private lateinit var exchangesClient: ExchangesCoroutineStub @@ -127,7 +125,9 @@ abstract class AbstractInProcessPanelMatchIntegrationTest { private lateinit var recurringExchangeId: String @get:Rule - val ruleChain: TestRule by lazy { chainRulesSequentially(databaseRule, inProcessKingdom) } + val ruleChain: TestRule by lazy { + chainRulesSequentially(kingdomDataServicesProvider, inProcessKingdom) + } @get:Rule val dataProviderFolder = TemporaryFolder() @get:Rule val modelProviderFolder = TemporaryFolder() @@ -417,6 +417,8 @@ abstract class AbstractInProcessPanelMatchIntegrationTest { private val typeRegistry = TypeRegistry.newBuilder().add(Shared.Parameters.getDescriptor()).build() + private const val REDIRECT_URI = "https://localhost:2048" + // TODO(@yunyeng): Think about the tests that start running around midnight. val EXCHANGE_DATE: LocalDate = LocalDate.now() diff --git a/src/main/kotlin/org/wfanet/panelmatch/integration/BUILD.bazel b/src/main/kotlin/org/wfanet/panelmatch/integration/BUILD.bazel index f3c1285a5f6..9ed860f7a60 100644 --- a/src/main/kotlin/org/wfanet/panelmatch/integration/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/panelmatch/integration/BUILD.bazel @@ -16,6 +16,8 @@ kt_jvm_library( "//imports/java/org/apache/beam/runners/direct", ], deps = [ + "//src/main/kotlin/org/wfanet/measurement/integration/common:in_process_kingdom", + "//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:kingdom_data_services_provider_rule", "//src/main/kotlin/org/wfanet/panelmatch/client/deploy", "//src/main/kotlin/org/wfanet/panelmatch/client/launcher", "//src/main/kotlin/org/wfanet/panelmatch/client/storage/testing", @@ -45,6 +47,5 @@ kt_jvm_library( "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/storage/testing", "@wfa_measurement_system//src/main/kotlin/org/wfanet/measurement/common/api:resource_key", "@wfa_measurement_system//src/main/kotlin/org/wfanet/measurement/common/identity:principal_identity", - "@wfa_measurement_system//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:in_process_kingdom", ], ) diff --git a/src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing/BUILD.bazel b/src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing/BUILD.bazel index 4c584d6cb37..aeb5019ebdb 100644 --- a/src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing/BUILD.bazel +++ b/src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing/BUILD.bazel @@ -19,6 +19,17 @@ kt_jvm_proto_library( deps = [":test_metadata_message_proto"], ) +proto_library( + name = "test_metadata_message_2_proto", + srcs = ["test_metadata_message_2.proto"], + strip_import_prefix = IMPORT_PREFIX, +) + +kt_jvm_proto_library( + name = "test_metadata_message_2_kt_jvm_proto", + deps = [":test_metadata_message_2_proto"], +) + proto_library( name = "test_parent_metadata_message_proto", srcs = ["test_parent_metadata_message.proto"], diff --git a/src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing/test_metadata_message_2.proto b/src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing/test_metadata_message_2.proto new file mode 100644 index 00000000000..20f1c5329a6 --- /dev/null +++ b/src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing/test_metadata_message_2.proto @@ -0,0 +1,25 @@ + +// Copyright 2021 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package wfa.measurement.api.v2alpha.event_group_metadata.testing; + +option java_package = "org.wfanet.measurement.api.v2alpha.event_group_metadata.testing"; +option java_multiple_files = true; + +message TestMetadataMessage2 { + int32 publisher_id = 1; +} diff --git a/src/main/proto/wfa/measurement/internal/kingdom/error_code.proto b/src/main/proto/wfa/measurement/internal/kingdom/error_code.proto index 75f94f8065c..456e77015b3 100644 --- a/src/main/proto/wfa/measurement/internal/kingdom/error_code.proto +++ b/src/main/proto/wfa/measurement/internal/kingdom/error_code.proto @@ -68,6 +68,8 @@ enum ErrorCode { EVENT_GROUP_INVALID_ARGS = 21; /** EventGroupMetadataDescriptor could not be found. */ EVENT_GROUP_METADATA_DESCRIPTOR_NOT_FOUND = 22; + // An EventGroupMetadataDescriptor already exists with the same protobuf type. + EVENT_GROUP_METADATA_DESCRIPTOR_ALREADY_EXISTS_WITH_TYPE = 43; /** RecurringExchange could not be found. */ RECURRING_EXCHANGE_NOT_FOUND = 23; /** ExchangeStepAttempt could not be found. */ diff --git a/src/main/resources/kingdom/spanner/add-event-group-metadata-descriptor-types.sql b/src/main/resources/kingdom/spanner/add-event-group-metadata-descriptor-types.sql new file mode 100644 index 00000000000..91c86d07976 --- /dev/null +++ b/src/main/resources/kingdom/spanner/add-event-group-metadata-descriptor-types.sql @@ -0,0 +1,30 @@ +-- liquibase formatted sql + +-- Copyright 2024 The Cross-Media Measurement Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- changeset sanjayvas:15 dbms:cloudspanner +-- comment: Add unique index for EventGroupMetadataDescriptor protobuf types. + +CREATE TABLE EventGroupMetadataDescriptorTypes ( + DataProviderId INT64 NOT NULL, + EventGroupMetadataDescriptorId INT64 NOT NULL, + + -- Fully-qualified protobuf type name. + ProtobufTypeName STRING(MAX) NOT NULL, +) PRIMARY KEY (DataProviderId, EventGroupMetadataDescriptorId, ProtobufTypeName), +INTERLEAVE IN PARENT EventGroupMetadataDescriptors ON DELETE CASCADE; + +CREATE UNIQUE INDEX EventGroupMetadataDescriptorTypesByType + ON EventGroupMetadataDescriptorTypes(ProtobufTypeName); diff --git a/src/main/resources/kingdom/spanner/changelog.yaml b/src/main/resources/kingdom/spanner/changelog.yaml index 9818e27f6f8..e59e19b113d 100644 --- a/src/main/resources/kingdom/spanner/changelog.yaml +++ b/src/main/resources/kingdom/spanner/changelog.yaml @@ -60,3 +60,6 @@ databaseChangeLog: - include: file: add-result-api-version.sql relativeToChangeLogFile: true +- include: + file: add-event-group-metadata-descriptor-types.sql + relativeToChangeLogFile: true diff --git a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/BUILD.bazel b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/BUILD.bazel index 69f990ff0a6..cf8dad70911 100644 --- a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/BUILD.bazel +++ b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/BUILD.bazel @@ -12,7 +12,7 @@ spanner_emulator_test( test_class = "org.wfanet.measurement.integration.deploy.gcloud.GCloudInProcessLifeOfAnEventGroupIntegrationTest", deps = [ "//src/main/kotlin/org/wfanet/measurement/integration/common:in_process_event_group_components", - "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing", + "//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:kingdom_data_services_provider_rule", "@wfa_common_jvm//imports/kotlin/kotlinx/coroutines/debug", ], ) @@ -23,8 +23,8 @@ kt_jvm_library( data = ["@cloud_spanner_emulator//:emulator"], deps = [ "//src/main/kotlin/org/wfanet/measurement/integration/common:in_process_life_of_a_measurement_integration_test", + "//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:kingdom_data_services_provider_rule", "//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:spanner_duchy_dependency_provider_rule", - "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing", ], ) @@ -45,7 +45,7 @@ kt_jvm_library( "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/postgres/testing", "//src/main/kotlin/org/wfanet/measurement/integration/common:in_process_life_of_a_measurement_integration_test", "//src/main/kotlin/org/wfanet/measurement/integration/deploy/common/postgres:postgres_duchy_dependency_provider_rule", - "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing", + "//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:kingdom_data_services_provider_rule", ], ) @@ -72,7 +72,7 @@ kt_jvm_library( "//src/main/kotlin/org/wfanet/measurement/integration/common:in_process_duchy", "//src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2:in_process_life_of_a_report_integration_test", "//src/main/kotlin/org/wfanet/measurement/integration/deploy/common/postgres:postgres_duchy_dependency_provider_rule", - "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing", + "//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:kingdom_data_services_provider_rule", "//src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/server/postgres:services", "//src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/postgres/testing", "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common/db/r2dbc/postgres/testing:database_provider", @@ -97,8 +97,8 @@ kt_jvm_library( data = ["@cloud_spanner_emulator//:emulator"], deps = [ "//src/main/kotlin/org/wfanet/measurement/integration/common:in_process_reach_measurement_accuracy_test", + "//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:kingdom_data_services_provider_rule", "//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:spanner_duchy_dependency_provider_rule", - "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing", ], ) diff --git a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudInProcessLifeOfAReportV2IntegrationTest.kt b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudInProcessLifeOfAReportV2IntegrationTest.kt index eaa3230e4f8..e75f7997d21 100644 --- a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudInProcessLifeOfAReportV2IntegrationTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudInProcessLifeOfAReportV2IntegrationTest.kt @@ -24,7 +24,6 @@ import org.wfanet.measurement.duchy.deploy.common.postgres.testing.Schemata.DUCH import org.wfanet.measurement.integration.common.ALL_DUCHY_NAMES import org.wfanet.measurement.integration.common.reporting.v2.InProcessLifeOfAReportIntegrationTest import org.wfanet.measurement.integration.deploy.common.postgres.PostgresDuchyDependencyProviderRule -import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.testing.KingdomDataServicesProviderRule import org.wfanet.measurement.reporting.deploy.v2.common.server.postgres.PostgresServices import org.wfanet.measurement.reporting.deploy.v2.postgres.testing.Schemata.REPORTING_CHANGELOG_PATH diff --git a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudInProcessLifeOfAnEventGroupIntegrationTest.kt b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudInProcessLifeOfAnEventGroupIntegrationTest.kt index 70ef3820725..3194ea2dc59 100644 --- a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudInProcessLifeOfAnEventGroupIntegrationTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudInProcessLifeOfAnEventGroupIntegrationTest.kt @@ -17,7 +17,6 @@ package org.wfanet.measurement.integration.deploy.gcloud import org.wfanet.measurement.integration.common.InProcessLifeOfAnEventGroupIntegrationTest -import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.testing.KingdomDataServicesProviderRule /** * Implementation of [InProcessLifeOfAnEventGroupIntegrationTest] for GCloud backends (Spanner, diff --git a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudPostgresInProcessLifeOfAMeasurementIntegrationTest.kt b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudPostgresInProcessLifeOfAMeasurementIntegrationTest.kt index 50f24af4955..b1aef1ea519 100644 --- a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudPostgresInProcessLifeOfAMeasurementIntegrationTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudPostgresInProcessLifeOfAMeasurementIntegrationTest.kt @@ -24,7 +24,6 @@ import org.wfanet.measurement.duchy.deploy.common.postgres.testing.Schemata import org.wfanet.measurement.integration.common.ALL_DUCHY_NAMES import org.wfanet.measurement.integration.common.InProcessLifeOfAMeasurementIntegrationTest import org.wfanet.measurement.integration.deploy.common.postgres.PostgresDuchyDependencyProviderRule -import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.testing.KingdomDataServicesProviderRule /** * Implementation of [InProcessLifeOfAMeasurementIntegrationTest] for GCloud backends with Postgres diff --git a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudSpannerInProcessLifeOfAMeasurementIntegrationTest.kt b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudSpannerInProcessLifeOfAMeasurementIntegrationTest.kt index ae1c514920b..cdb5ac04607 100644 --- a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudSpannerInProcessLifeOfAMeasurementIntegrationTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudSpannerInProcessLifeOfAMeasurementIntegrationTest.kt @@ -18,7 +18,6 @@ import org.junit.Rule import org.junit.rules.Timeout import org.wfanet.measurement.integration.common.ALL_DUCHY_NAMES import org.wfanet.measurement.integration.common.InProcessLifeOfAMeasurementIntegrationTest -import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.testing.KingdomDataServicesProviderRule /** * Implementation of [InProcessLifeOfAMeasurementIntegrationTest] for GCloud backends with Spanner diff --git a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudSpannerInProcessReachMeasurementAccuracyTest.kt b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudSpannerInProcessReachMeasurementAccuracyTest.kt index 1d77e064749..a09beb09b00 100644 --- a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudSpannerInProcessReachMeasurementAccuracyTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudSpannerInProcessReachMeasurementAccuracyTest.kt @@ -18,7 +18,6 @@ import org.junit.Rule import org.junit.rules.Timeout import org.wfanet.measurement.integration.common.ALL_DUCHY_NAMES import org.wfanet.measurement.integration.common.InProcessReachMeasurementAccuracyTest -import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.testing.KingdomDataServicesProviderRule /** * Implementation of [InProcessReachMeasurementAccuracyTest] for GCloud backends with Spanner diff --git a/src/test/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/BUILD.bazel b/src/test/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/BUILD.bazel index b5728ea8230..b3c8085bbbe 100644 --- a/src/test/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/BUILD.bazel +++ b/src/test/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/BUILD.bazel @@ -39,6 +39,7 @@ spanner_emulator_test( test_class = "org.wfanet.measurement.kingdom.deploy.gcloud.spanner.SpannerMeasurementsServiceTest", deps = [ "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner:measurements_service", + "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner:services", "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing", "//src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing", ], diff --git a/src/test/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulatorTest.kt b/src/test/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulatorTest.kt index ff83ebe8e49..c97e80223a6 100644 --- a/src/test/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulatorTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulatorTest.kt @@ -347,6 +347,7 @@ class EdpSimulatorTest { @Test fun `ensureEventGroup creates EventGroup and EventGroupMetadataDescriptor`() { + val knownEventGroupMetadataTypes = listOf(SyntheticEventGroupSpec.getDescriptor().file) val edpSimulator = EdpSimulator( EDP_DATA, @@ -362,22 +363,15 @@ class EdpSimulatorTest { dummyThrottler, privacyBudgetManager, TRUSTED_CERTIFICATES, + knownEventGroupMetadataTypes = knownEventGroupMetadataTypes, ) runBlocking { edpSimulator.ensureEventGroup(TEST_EVENT_TEMPLATES, SYNTHETIC_DATA_SPEC) } - // Verify metadata descriptor set contains synthetic data spec. - val createDescriptorRequest: CreateEventGroupMetadataDescriptorRequest = - verifyAndCapture( - eventGroupMetadataDescriptorsServiceMock, - EventGroupMetadataDescriptorsCoroutineImplBase::createEventGroupMetadataDescriptor, - ) - val descriptors = - ProtoReflection.buildDescriptors( - listOf(createDescriptorRequest.eventGroupMetadataDescriptor.descriptorSet) - ) - assertThat(descriptors.map { it.fullName }) - .contains(SyntheticEventGroupSpec.getDescriptor().fullName) + // Verify metadata descriptor is created. + verifyBlocking(eventGroupMetadataDescriptorsServiceMock) { + createEventGroupMetadataDescriptor(any()) + } // Verify EventGroup metadata. val createRequest: CreateEventGroupRequest = diff --git a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/CelEnvProviderTest.kt b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/CelEnvProviderTest.kt index baa6aa177b6..0197f01788b 100644 --- a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/CelEnvProviderTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/CelEnvProviderTest.kt @@ -98,6 +98,7 @@ class CelEnvProviderTest { ), REPORTING_EVENT_GROUP_DESCRIPTOR, Duration.ofMinutes(5), + emptyList(), coroutineContext, ) .use { it.getTypeRegistryAndEnv() } @@ -135,8 +136,9 @@ class CelEnvProviderTest { ), REPORTING_EVENT_GROUP_DESCRIPTOR, Duration.ofMinutes(5), + emptyList(), coroutineContext, - 1, + numRetriesInitialSync = 1, ) .use { advanceTimeBy(CelEnvCacheProvider.RETRY_DELAY.toMillis()) @@ -175,8 +177,9 @@ class CelEnvProviderTest { ), REPORTING_EVENT_GROUP_DESCRIPTOR, Duration.ofMinutes(5), + emptyList(), coroutineContext, - numRetries, + numRetriesInitialSync = numRetries, ) .use { it.getTypeRegistryAndEnv() } } @@ -206,8 +209,9 @@ class CelEnvProviderTest { ), REPORTING_EVENT_GROUP_DESCRIPTOR, Duration.ofMinutes(5), + emptyList(), coroutineContext, - numRetries, + numRetriesInitialSync = numRetries, ) .use { it.getTypeRegistryAndEnv() } } @@ -251,6 +255,7 @@ class CelEnvProviderTest { ), REPORTING_EVENT_GROUP_DESCRIPTOR, cacheRefreshInterval, + emptyList(), coroutineContext, ) .use { diff --git a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v1alpha/EventGroupsServiceTest.kt b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v1alpha/EventGroupsServiceTest.kt index 90e4c89b3e2..17238a9d814 100644 --- a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v1alpha/EventGroupsServiceTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v1alpha/EventGroupsServiceTest.kt @@ -23,7 +23,6 @@ import java.nio.file.Path import java.nio.file.Paths import java.time.Duration import kotlin.test.assertFailsWith -import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.runBlocking import org.junit.Before import org.junit.Rule @@ -190,7 +189,7 @@ class EventGroupsServiceTest { EventGroupMetadataDescriptorsCoroutineStub(grpcTestServerRule.channel), EventGroup.getDescriptor(), Duration.ofSeconds(5), - Dispatchers.Default, + emptyList(), ) service = diff --git a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/EventGroupsServiceTest.kt b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/EventGroupsServiceTest.kt index 1c7a43c6dd8..96efd205a01 100644 --- a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/EventGroupsServiceTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/EventGroupsServiceTest.kt @@ -25,7 +25,6 @@ import java.nio.file.Path import java.nio.file.Paths import java.time.Duration import kotlin.test.assertFailsWith -import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.runBlocking import org.junit.Before import org.junit.Rule @@ -111,7 +110,7 @@ class EventGroupsServiceTest { EventGroupMetadataDescriptorsCoroutineStub(grpcTestServerRule.channel), EventGroup.getDescriptor(), Duration.ofSeconds(5), - Dispatchers.Default, + emptyList(), ) service = @@ -512,7 +511,7 @@ class EventGroupsServiceTest { EventGroupMetadataDescriptorsCoroutineStub(grpcTestServerRule.channel), EventGroup.getDescriptor(), Duration.ofSeconds(5), - Dispatchers.Default, + emptyList(), ) service =