From bfdf13582651eb331d63fd78ba4e2bc98224895c Mon Sep 17 00:00:00 2001 From: Sanjay Vasandani Date: Mon, 11 Mar 2024 12:31:22 -0700 Subject: [PATCH] Introduce the concept of known EventGroup metadata types. (#1512) Known EventGroup metadata types are types expected to be known by all callers. These need not be included in the EventGroupMetadataDescriptor FileDescriptorSet, and are instead loaded separately. This also enforces uniqueness for EventGroupMetadataDescriptor message types. Note that this only affects the creation of new resources. Existing resources may exist that already violate this constraint. Includes a workaround for bazelbuild/rules_proto#203. Closes #1511 --- MODULE.bazel | 14 +- MODULE.bazel.lock | 132 ++++++++---------- build/rules_proto/BUILD.bazel | 0 build/rules_proto/module_dot_bazel.patch | 14 ++ docs/gke/kingdom-deployment.md | 14 +- docs/gke/reporting-server-deployment.md | 21 +-- docs/gke/reporting-v2-server-deployment.md | 40 +++--- src/main/k8s/dev/BUILD.bazel | 21 ++- .../k8s/dev/config_files_kustomization.yaml | 1 + .../reporting_config_files_kustomization.yaml | 1 + ...porting_v2_config_files_kustomization.yaml | 1 + src/main/k8s/kingdom.cue | 3 + src/main/k8s/local/BUILD.bazel | 2 + .../k8s/local/config_files_kustomization.yaml | 1 + src/main/k8s/local/testing/BUILD.bazel | 1 + .../testing/config_files_kustomization.yaml | 1 + src/main/k8s/panelmatch/local/BUILD.bazel | 1 + .../local/config_files_kustomization.yaml | 3 +- src/main/k8s/reporting.cue | 2 + src/main/k8s/reporting_v2.cue | 2 + src/main/k8s/testing/secretfiles/BUILD.bazel | 11 +- .../common/InProcessCmmsComponents.kt | 2 +- .../common/InProcessEdpSimulator.kt | 1 + .../integration/common/InProcessKingdom.kt | 6 +- ...rocessLifeOfAnEventGroupIntegrationTest.kt | 2 +- .../reporting/InProcessReportingServer.kt | 3 +- .../InProcessLifeOfAReportIntegrationTest.kt | 92 ++++++------ .../reporting/v2/InProcessReportingServer.kt | 43 +++--- .../integration/deploy/gcloud/BUILD.bazel | 10 +- .../deploy/gcloud/InProcessKingdom.kt | 42 ------ .../KingdomDataServicesProviderRule.kt | 17 ++- .../deploy/common/server/KingdomDataServer.kt | 26 ++++ .../deploy/common/service/DataServices.kt | 9 +- .../gcloud/server/SpannerKingdomDataServer.kt | 2 +- .../gcloud/spanner/SpannerDataServices.kt | 8 +- ...nerEventGroupMetadataDescriptorsService.kt | 35 ++++- .../common/KingdomInternalException.kt | 13 ++ .../deploy/gcloud/spanner/testing/BUILD.bazel | 1 - .../CreateEventGroupMetadataDescriptor.kt | 28 +++- .../UpdateEventGroupMetadataDescriptor.kt | 43 +++++- .../service/internal/testing/BUILD.bazel | 2 + ...ventGroupMetadataDescriptorsServiceTest.kt | 82 ++++++++++- .../BigQueryEdpSimulatorRunner.kt | 2 + .../dataprovider/CsvEdpSimulatorRunner.kt | 2 + .../loadtest/dataprovider/EdpSimulator.kt | 15 +- .../dataprovider/EdpSimulatorRunner.kt | 3 + .../SyntheticGeneratorEdpSimulatorRunner.kt | 1 + .../common/server/V1AlphaPublicApiServer.kt | 26 ++++ .../common/server/V2AlphaPublicApiServer.kt | 26 ++++ .../reporting/service/api/CelEnvProvider.kt | 39 +++--- ...tractInProcessPanelMatchIntegrationTest.kt | 18 +-- .../wfanet/panelmatch/integration/BUILD.bazel | 3 +- .../event_group_metadata/testing/BUILD.bazel | 11 ++ .../testing/test_metadata_message_2.proto | 25 ++++ .../internal/kingdom/error_code.proto | 2 + ...-event-group-metadata-descriptor-types.sql | 30 ++++ .../resources/kingdom/spanner/changelog.yaml | 3 + .../integration/deploy/gcloud/BUILD.bazel | 10 +- ...InProcessLifeOfAReportV2IntegrationTest.kt | 1 - ...rocessLifeOfAnEventGroupIntegrationTest.kt | 1 - ...rocessLifeOfAMeasurementIntegrationTest.kt | 1 - ...rocessLifeOfAMeasurementIntegrationTest.kt | 1 - ...erInProcessReachMeasurementAccuracyTest.kt | 1 - .../kingdom/deploy/gcloud/spanner/BUILD.bazel | 1 + .../loadtest/dataprovider/EdpSimulatorTest.kt | 18 +-- .../service/api/CelEnvProviderTest.kt | 11 +- .../api/v1alpha/EventGroupsServiceTest.kt | 3 +- .../api/v2alpha/EventGroupsServiceTest.kt | 5 +- 68 files changed, 699 insertions(+), 313 deletions(-) create mode 100644 build/rules_proto/BUILD.bazel create mode 100644 build/rules_proto/module_dot_bazel.patch delete mode 100644 src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/InProcessKingdom.kt rename src/main/kotlin/org/wfanet/measurement/{kingdom/deploy/gcloud/spanner/testing => integration/deploy/gcloud}/KingdomDataServicesProviderRule.kt (70%) create mode 100644 src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing/test_metadata_message_2.proto create mode 100644 src/main/resources/kingdom/spanner/add-event-group-metadata-descriptor-types.sql diff --git a/MODULE.bazel b/MODULE.bazel index 5849f61a7f7..b938ad057dd 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -31,7 +31,7 @@ bazel_dep( ) bazel_dep( name = "rules_proto", - version = "6.0.0-rc1", + version = "6.0.0-rc2", ) bazel_dep( name = "rules_cc", @@ -313,6 +313,18 @@ http_archive( url = "https://github.com/world-federation-of-advertisers/uk-pilot-event-templates/archive/refs/tags/v0.2.0.tar.gz", ) +# TODO(bazelbuild/rules_proto#203): Remove when fixed. +archive_override( + module_name = "rules_proto", + integrity = "sha256-cf2+0AoHCVIa0hIFjGDROZe5IqXQHb/Zl/DVfWiee2c=", + patch_strip = 0, + patches = ["//build/rules_proto:module_dot_bazel.patch"], + strip_prefix = "rules_proto-6.0.0-rc2", + urls = [ + "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc2/rules_proto-6.0.0-rc2.tar.gz", + ], +) + # Force use of newer version of boringssl. single_version_override( module_name = "boringssl", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index cc910d23238..2d6913ff6db 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,6 +1,6 @@ { "lockFileVersion": 3, - "moduleFileHash": "adda45eb45d25cb6e7f9dd34b20119d2e8e43e7ac3cef37cff2f52a9036fe1e4", + "moduleFileHash": "8708b0e5573f4f48c1b66c6194d5342e976a77caa74e2c763582c858911e0e1f", "flags": { "cmdRegistries": [ "https://raw.githubusercontent.com/world-federation-of-advertisers/bazel-registry/main", @@ -354,7 +354,7 @@ "deps": { "platforms": "platforms@0.0.8", "bazel_skylib": "bazel_skylib@1.5.0", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "rules_cc": "rules_cc@0.0.9", "rules_pkg": "rules_pkg@0.9.1", "io_bazel_rules_go": "rules_go@0.43.0", @@ -450,10 +450,10 @@ } } }, - "rules_proto@6.0.0-rc1": { + "rules_proto@_": { "name": "rules_proto", - "version": "6.0.0-rc1", - "key": "rules_proto@6.0.0-rc1", + "version": "0.0.0", + "key": "rules_proto@_", "repoName": "rules_proto", "executionPlatformsToRegister": [], "toolchainsToRegister": [], @@ -461,24 +461,10 @@ "deps": { "rules_license": "rules_license@0.0.7", "bazel_skylib": "bazel_skylib@1.5.0", + "bazel_features": "bazel_features@1.2.0", + "rules_cc": "rules_cc@0.0.9", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "name": "rules_proto~6.0.0-rc1", - "urls": [ - "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc1/rules_proto-6.0.0-rc1.tar.gz" - ], - "integrity": "sha256-kEqAl/rkKmkMjgjYBSEOQMzLBp9fmg9nJ89PqnvtLJw=", - "strip_prefix": "rules_proto-6.0.0-rc1", - "remote_patches": { - "https://bcr.bazel.build/modules/rules_proto/6.0.0-rc1/patches/module_dot_bazel_version.patch": "sha256-OepRECTunWiz3WREvMK/pOvpt/HRZ6g87fKfokcnQGE=" - }, - "remote_patch_strip": 1 - } } }, "rules_cc@0.0.9": { @@ -671,10 +657,10 @@ } ], "deps": { - "bazel_features": "bazel_features@1.1.1", + "bazel_features": "bazel_features@1.2.0", "bazel_skylib": "bazel_skylib@1.5.0", "platforms": "platforms@0.0.8", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_protobuf": "protobuf@23.1", "gazelle": "gazelle@0.34.0", "bazel_tools": "bazel_tools@_", @@ -797,7 +783,7 @@ "bazel_skylib": "bazel_skylib@1.5.0", "com_google_protobuf": "protobuf@23.1", "io_bazel_rules_go": "rules_go@0.43.0", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" }, @@ -897,7 +883,7 @@ "platforms": "platforms@0.0.8", "rules_cc": "rules_cc@0.0.9", "bazel_skylib": "bazel_skylib@1.5.0", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "rules_license": "rules_license@0.0.7", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" @@ -1325,10 +1311,10 @@ } ], "deps": { - "bazel_features": "bazel_features@1.1.1", + "bazel_features": "bazel_features@1.2.0", "bazel_skylib": "bazel_skylib@1.5.0", "platforms": "platforms@0.0.8", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_protobuf": "protobuf@23.1", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" @@ -1421,7 +1407,7 @@ "bazel_skylib": "bazel_skylib@1.5.0", "rules_python": "rules_python@0.29.0", "rules_cc": "rules_cc@0.0.9", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "rules_java": "rules_java@7.3.2", "rules_pkg": "rules_pkg@0.9.1", "platforms": "platforms@0.0.8", @@ -1573,7 +1559,7 @@ "deps": { "aspect_bazel_lib": "aspect_bazel_lib@1.39.0", "aspect_rules_lint": "aspect_rules_lint@0.7.0", - "bazel_features": "bazel_features@1.1.1", + "bazel_features": "bazel_features@1.2.0", "bazel_skylib": "bazel_skylib@1.5.0", "rules_nodejs": "rules_nodejs@5.8.2", "platforms": "platforms@0.0.8", @@ -1955,7 +1941,7 @@ ], "deps": { "bazel_skylib": "bazel_skylib@1.5.0", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "rules_java": "rules_java@7.3.2", "rules_kotlin": "rules_kotlin@1.9.0", "rules_jvm_external": "rules_jvm_external@6.0", @@ -2477,7 +2463,7 @@ "deps": { "platforms": "platforms@0.0.8", "bazel_skylib": "bazel_skylib@1.5.0", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "rules_pkg": "rules_pkg@0.9.1", "rules_java": "rules_java@7.3.2", "rules_multirun": "rules_multirun@0.6.1", @@ -2554,7 +2540,7 @@ ], "deps": { "rules_cc": "rules_cc@0.0.9", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_absl": "abseil-cpp@20230802.1", "com_google_googletest": "googletest@1.14.0.bcr.1", "com_google_protobuf": "protobuf@23.1", @@ -2607,7 +2593,7 @@ } ], "deps": { - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_protobuf": "protobuf@23.1", "com_google_googleapis": "googleapis@0.0.0-bzlmod.1", "bazel_tools": "bazel_tools@_", @@ -2672,7 +2658,7 @@ "extensionUsages": [], "deps": { "rules_cc": "rules_cc@0.0.9", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_absl": "abseil-cpp@20230802.1", "com_google_googletest": "googletest@1.14.0.bcr.1", "com_github_google_glog": "glog@0.6.0", @@ -2800,7 +2786,7 @@ } ], "deps": { - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_protobuf": "protobuf@23.1", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" @@ -2834,7 +2820,7 @@ "com_github_grpc_grpc": "grpc@1.48.1.bcr.2", "com_google_protobuf": "protobuf@23.1", "rules_cc": "rules_cc@0.0.9", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_absl": "abseil-cpp@20230802.1", "com_github_google_googletest": "googletest@1.14.0.bcr.1", "boringssl": "boringssl@0.0.0-20230215-5c22014", @@ -2907,7 +2893,7 @@ } ], "deps": { - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_protobuf": "protobuf@23.1", "rules_cc": "rules_cc@0.0.9", "com_google_absl": "abseil-cpp@20230802.1", @@ -3018,7 +3004,7 @@ } ], "deps": { - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "io_bazel_rules_go": "rules_go@0.43.0", "bazel_gazelle": "gazelle@0.34.0", "bazel_skylib": "bazel_skylib@1.5.0", @@ -3179,7 +3165,7 @@ ], "deps": { "platforms": "platforms@0.0.8", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_protobuf": "protobuf@23.1", "com_google_absl": "abseil-cpp@20230802.1", "bazel_tools": "bazel_tools@_", @@ -3327,7 +3313,7 @@ "rules_cc": "rules_cc@0.0.9", "rules_java": "rules_java@7.3.2", "rules_license": "rules_license@0.0.7", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "rules_python": "rules_python@0.29.0", "platforms": "platforms@0.0.8", "com_google_protobuf": "protobuf@23.1", @@ -3376,10 +3362,10 @@ } } }, - "bazel_features@1.1.1": { + "bazel_features@1.2.0": { "name": "bazel_features", - "version": "1.1.1", - "key": "bazel_features@1.1.1", + "version": "1.2.0", + "key": "bazel_features@1.2.0", "repoName": "bazel_features", "executionPlatformsToRegister": [], "toolchainsToRegister": [], @@ -3387,9 +3373,9 @@ { "extensionBzlFile": "@bazel_features//private:extensions.bzl", "extensionName": "version_extension", - "usingModule": "bazel_features@1.1.1", + "usingModule": "bazel_features@1.2.0", "location": { - "file": "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel", + "file": "https://bcr.bazel.build/modules/bazel_features/1.2.0/MODULE.bazel", "line": 6, "column": 24 }, @@ -3411,14 +3397,14 @@ "bzlFile": "@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_archive", "attributes": { - "name": "bazel_features~1.1.1", + "name": "bazel_features~1.2.0", "urls": [ - "https://github.com/bazel-contrib/bazel_features/releases/download/v1.1.1/bazel_features-v1.1.1.tar.gz" + "https://github.com/bazel-contrib/bazel_features/releases/download/v1.2.0/bazel_features-v1.2.0.tar.gz" ], - "integrity": "sha256-YsJuQn5cvHUQJERpJ2IuOYqdzfMsZDJSOIFXCdEcEag=", - "strip_prefix": "bazel_features-1.1.1", + "integrity": "sha256-uHicg8iT1+8wQdPyeVd0k2sn/2FwGnBd9S/UHW3b9pI=", + "strip_prefix": "bazel_features-1.2.0", "remote_patches": { - "https://bcr.bazel.build/modules/bazel_features/1.1.1/patches/module_dot_bazel_version.patch": "sha256-+56MAEsc7bYN/Pzhn252ZQUxiRzZg9bynXj1qpsmCYs=" + "https://bcr.bazel.build/modules/bazel_features/1.2.0/patches/module_dot_bazel_version.patch": "sha256-B0rNFSA7VB65YY95ORkzTgsn21RkUwlAb1Xy2y2PyXA=" }, "remote_patch_strip": 1 } @@ -3540,7 +3526,7 @@ "rules_jvm_external": "rules_jvm_external@6.0", "rules_pkg": "rules_pkg@0.9.1", "io_bazel_stardoc": "stardoc@0.5.6", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" }, @@ -3651,7 +3637,7 @@ ], "deps": { "bazel_skylib": "bazel_skylib@1.5.0", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "com_google_protobuf": "protobuf@23.1", "com_google_absl": "abseil-cpp@20230802.1", "rules_pkg": "rules_pkg@0.9.1", @@ -3815,7 +3801,7 @@ "aspect_rules_js": "aspect_rules_js@1.35.0", "bazel_skylib": "bazel_skylib@1.5.0", "platforms": "platforms@0.0.8", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "rules_buf": "rules_buf@0.1.1", "com_google_protobuf": "protobuf@23.1", "bazel_tools": "bazel_tools@_", @@ -4148,7 +4134,7 @@ "com_google_absl": "abseil-cpp@20230802.1", "com_google_protobuf": "protobuf@23.1", "com_googlesource_code_re2": "re2@2023-09-01", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "upb": "upb@0.0.0-20230516-61a97ef", "zlib": "zlib@1.3", "rules_java": "rules_java@7.3.2", @@ -4255,7 +4241,7 @@ "deps": { "platforms": "platforms@0.0.8", "com_google_protobuf": "protobuf@23.1", - "rules_proto": "rules_proto@6.0.0-rc1", + "rules_proto": "rules_proto@_", "bazel_tools": "bazel_tools@_", "local_config_platform": "local_config_platform@_" }, @@ -135753,7 +135739,7 @@ [ "aspect_rules_js~1.35.0", "bazel_features", - "bazel_features~1.1.1" + "bazel_features~1.2.0" ], [ "aspect_rules_js~1.35.0", @@ -135860,24 +135846,24 @@ ] } }, - "@@bazel_features~1.1.1//private:extensions.bzl%version_extension": { + "@@bazel_features~1.2.0//private:extensions.bzl%version_extension": { "general": { "bzlTransitiveDigest": "xm7Skm1Las5saxzFWt2hbS+e68BWi+MXyt6+lKIhjPA=", "accumulatedFileDigests": {}, "envVariables": {}, "generatedRepoSpecs": { "bazel_features_version": { - "bzlFile": "@@bazel_features~1.1.1//private:version_repo.bzl", + "bzlFile": "@@bazel_features~1.2.0//private:version_repo.bzl", "ruleClassName": "version_repo", "attributes": { - "name": "bazel_features~1.1.1~version_extension~bazel_features_version" + "name": "bazel_features~1.2.0~version_extension~bazel_features_version" } }, "bazel_features_globals": { - "bzlFile": "@@bazel_features~1.1.1//private:globals_repo.bzl", + "bzlFile": "@@bazel_features~1.2.0//private:globals_repo.bzl", "ruleClassName": "globals_repo", "attributes": { - "name": "bazel_features~1.1.1~version_extension~bazel_features_globals", + "name": "bazel_features~1.2.0~version_extension~bazel_features_globals", "globals": { "RunEnvironmentInfo": "5.3.0", "DefaultInfo": "0.0.1", @@ -137624,7 +137610,7 @@ }, "@@rules_go~0.43.0//go:extensions.bzl%go_sdk": { "os:linux,arch:amd64": { - "bzlTransitiveDigest": "vnmAJ4XgQqHHLFcBK8FIw6dj/TSctEBQAwaVeDRGyiI=", + "bzlTransitiveDigest": "rXV9vbWpSmIyZo4jRaNZj4wcI9IJZWc3um7z5mmTu18=", "accumulatedFileDigests": {}, "envVariables": {}, "generatedRepoSpecs": { @@ -137682,19 +137668,19 @@ }, "recordedRepoMappingEntries": [ [ - "bazel_features~1.1.1", + "bazel_features~1.2.0", "bazel_features_globals", - "bazel_features~1.1.1~version_extension~bazel_features_globals" + "bazel_features~1.2.0~version_extension~bazel_features_globals" ], [ - "bazel_features~1.1.1", + "bazel_features~1.2.0", "bazel_features_version", - "bazel_features~1.1.1~version_extension~bazel_features_version" + "bazel_features~1.2.0~version_extension~bazel_features_version" ], [ "rules_go~0.43.0", "bazel_features", - "bazel_features~1.1.1" + "bazel_features~1.2.0" ], [ "rules_go~0.43.0", @@ -137706,7 +137692,7 @@ }, "@@rules_go~0.43.0//go/private:extensions.bzl%non_module_dependencies": { "general": { - "bzlTransitiveDigest": "ZKrgcEEVppo3fWRms84VMXw08TJubOdDD/oJsStb68g=", + "bzlTransitiveDigest": "XNuPeWvAtP8chNDlXrCCWQVfBVQS2OQmdvYGeDJKsmI=", "accumulatedFileDigests": {}, "envVariables": {}, "generatedRepoSpecs": { @@ -137926,19 +137912,19 @@ }, "recordedRepoMappingEntries": [ [ - "bazel_features~1.1.1", + "bazel_features~1.2.0", "bazel_features_globals", - "bazel_features~1.1.1~version_extension~bazel_features_globals" + "bazel_features~1.2.0~version_extension~bazel_features_globals" ], [ - "bazel_features~1.1.1", + "bazel_features~1.2.0", "bazel_features_version", - "bazel_features~1.1.1~version_extension~bazel_features_version" + "bazel_features~1.2.0~version_extension~bazel_features_version" ], [ "rules_go~0.43.0", "bazel_features", - "bazel_features~1.1.1" + "bazel_features~1.2.0" ], [ "rules_go~0.43.0", diff --git a/build/rules_proto/BUILD.bazel b/build/rules_proto/BUILD.bazel new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build/rules_proto/module_dot_bazel.patch b/build/rules_proto/module_dot_bazel.patch new file mode 100644 index 00000000000..f9c278a9827 --- /dev/null +++ b/build/rules_proto/module_dot_bazel.patch @@ -0,0 +1,14 @@ +--- MODULE.bazel ++++ MODULE.bazel +@@ -2,10 +2,10 @@ + bazel_dep(name = "rules_license", version = "0.0.7") + bazel_dep(name = "bazel_skylib", version = "1.3.0") + bazel_dep(name = "bazel_features", version = "1.2.0") ++bazel_dep(name = "rules_cc", version = "0.0.1") + + # Dependencies needed in tests + bazel_dep(name = "stardoc", version = "0.5.6", dev_dependency = True, repo_name = "io_bazel_stardoc") +-bazel_dep(name = "rules_cc", version = "0.0.1", dev_dependency = True) + bazel_dep(name = "googletest", version = "1.11.0", dev_dependency = True, repo_name = "com_google_googletest") + bazel_dep(name = "protobuf", version = "23.1", dev_dependency = True, repo_name = "com_google_protobuf") + bazel_dep(name = "platforms", version = "0.0.8", dev_dependency = True) diff --git a/docs/gke/kingdom-deployment.md b/docs/gke/kingdom-deployment.md index 8b1576b242a..195a25a9ec2 100644 --- a/docs/gke/kingdom-deployment.md +++ b/docs/gke/kingdom-deployment.md @@ -214,14 +214,16 @@ within the Kustomization directory. ## Customize the K8s configMap Configuration that may frequently change is stored in a K8s configMap. The `dev` -configuration uses one named `config-files` containing the file -`authority_key_identifier_to_principal_map.textproto`. +configuration uses one named `config-files` containing the following files: -Place this file in the `src/main/k8s/dev/config_files/` path within the -Kustomization directory. +* `authority_key_identifier_to_principal_map.textproto` + * See [Creating Resources](../operations/creating-resources.md) +* `known_event_group_metadata_type_set.pb` + * Protobuf `FileDescriptorSet` containing known `EventGroup` metadata + types. -See [Creating Resources](../operations/creating-resources.md) for information on -this file format. +Place these files in the `src/main/k8s/dev/config_files/` path within the +Kustomization directory. ## Apply the K8s Kustomization diff --git a/docs/gke/reporting-server-deployment.md b/docs/gke/reporting-server-deployment.md index 0badde25784..68d8d0e7b0c 100644 --- a/docs/gke/reporting-server-deployment.md +++ b/docs/gke/reporting-server-deployment.md @@ -294,16 +294,17 @@ secretGenerator: ## Customize the K8s ConfigMap Configuration that may frequently change is stored in a K8s configMap. The `dev` -configuration uses one named `config-files`, which contains configuration files -in -[protobuf text format](https://developers.google.com/protocol-buffers/docs/text-format-spec). - -* `authority_key_identifier_to_principal_map.textproto` - - [`AuthorityKeyToPrincipalMap`](../../src/main/proto/wfa/measurement/config/authority_key_to_principal_map.proto) -* `encryption_key_pair_config.textproto` - - [`EncryptionKeyPairConfig`](../../src/main/proto/wfa/measurement/config/reporting/encryption_key_pair_config.proto) -* `measurement_spec_config.textproto` - - [`MeasurementSpecConfig`](../../src/main/proto/wfa/measurement/config/reporting/measurement_spec_config.proto) +configuration uses one named `config-files`. + +* `authority_key_identifier_to_principal_map.textproto` + * [`AuthorityKeyToPrincipalMap`](../../src/main/proto/wfa/measurement/config/authority_key_to_principal_map.proto) +* `encryption_key_pair_config.textproto` + * [`EncryptionKeyPairConfig`](../../src/main/proto/wfa/measurement/config/reporting/encryption_key_pair_config.proto) +* `measurement_spec_config.textproto` + * [`MeasurementSpecConfig`](../../src/main/proto/wfa/measurement/config/reporting/measurement_spec_config.proto) +* `known_event_group_metadata_type_set.pb` + * Protobuf `FileDescriptorSet` containing known `EventGroup` metadata + types. Place these files into the `src/main/k8s/dev/reporting_config_files/` path within the Kustomization directory. diff --git a/docs/gke/reporting-v2-server-deployment.md b/docs/gke/reporting-v2-server-deployment.md index 226d1bd24b0..6242b853a94 100644 --- a/docs/gke/reporting-v2-server-deployment.md +++ b/docs/gke/reporting-v2-server-deployment.md @@ -24,7 +24,7 @@ free to use whichever you prefer. - 1 Kubernetes configmap - `config-files` - 2 Kubernetes services - - `postgres-internal-reporting-server` (Cluster IP) + - `postgres-internal-reporting-server` (Cluster IP) - `reporting-v2alpha-public-api-server` (External load balancer) - 2 Kubernetes deployments - `postgres-internal-reporting-server-deployment` @@ -153,16 +153,16 @@ In order to use the IAM service account that we created earlier from our cluster, we need to create a K8s ServiceAccount and give it access to that IAM service account. -For example, to create a K8s ServiceAccount named `internal-reporting-v2-server`, -run +For example, to create a K8s ServiceAccount named +`internal-reporting-v2-server`, run ```shell kubectl create serviceaccount internal-reporting-v2-server ``` Supposing the IAM service account you created in a previous step is named -`reporting-v2-internal` within the `halo-cmm-dev` project. You'll need to allow the -K8s service account to impersonate it +`reporting-v2-internal` within the `halo-cmm-dev` project. You'll need to allow +the K8s service account to impersonate it ```shell gcloud iam service-accounts add-iam-policy-binding \ @@ -259,8 +259,8 @@ Generate the archive: bazel build //src/main/k8s/testing/secretfiles:archive ``` -Extract the generated archive to the `src/main/k8s/dev/reporting_v2_secrets/` path -within the Kustomization directory. +Extract the generated archive to the `src/main/k8s/dev/reporting_v2_secrets/` +path within the Kustomization directory. ### Measurement Consumer config @@ -273,8 +273,8 @@ Contents: ### Generator -Place the above files into the `src/main/k8s/dev/reporting_v2_secrets/` path within -the Kustomization directory. +Place the above files into the `src/main/k8s/dev/reporting_v2_secrets/` path +within the Kustomization directory. Create a `kustomization.yaml` file in that path with the following content, substituting the names of your own keys: @@ -297,17 +297,17 @@ secretGenerator: ## Customize the K8s ConfigMap Configuration that may frequently change is stored in a K8s configMap. The `dev` -configuration uses one named `config-files`, which contains configuration files -in -[protobuf text format](https://developers.google.com/protocol-buffers/docs/text-format-spec). - -* `authority_key_identifier_to_principal_map.textproto` - - [`AuthorityKeyToPrincipalMap`](../../src/main/proto/wfa/measurement/config/authority_key_to_principal_map.proto) -* `encryption_key_pair_config.textproto` - - [`EncryptionKeyPairConfig`](../../src/main/proto/wfa/measurement/config/reporting/encryption_key_pair_config.proto) -* `metric_spec_config.textproto` - - [`MetricSpecConfig`](../../src/main/proto/wfa/measurement/config/reporting/metric_spec_config.proto) - +configuration uses one named `config-files`. + +* `authority_key_identifier_to_principal_map.textproto` + * [`AuthorityKeyToPrincipalMap`](../../src/main/proto/wfa/measurement/config/authority_key_to_principal_map.proto) +* `encryption_key_pair_config.textproto` + * [`EncryptionKeyPairConfig`](../../src/main/proto/wfa/measurement/config/reporting/encryption_key_pair_config.proto) +* `metric_spec_config.textproto` + * [`MetricSpecConfig`](../../src/main/proto/wfa/measurement/config/reporting/metric_spec_config.proto) +* `known_event_group_metadata_type_set.pb` + * Protobuf `FileDescriptorSet` containing known `EventGroup` metadata + types. Place these files into the `src/main/k8s/dev/reporting_v2_config_files/` path within the Kustomization directory. diff --git a/src/main/k8s/dev/BUILD.bazel b/src/main/k8s/dev/BUILD.bazel index bac8ecf9606..e29320e0f23 100644 --- a/src/main/k8s/dev/BUILD.bazel +++ b/src/main/k8s/dev/BUILD.bazel @@ -82,12 +82,17 @@ kustomization_dir( kustomization_dir( name = "config_files", - srcs = ["config_files_kustomization.yaml"], + testonly = True, + srcs = [ + "config_files_kustomization.yaml", + "//src/main/k8s/testing/secretfiles:known_event_group_metadata_type_set", + ], renames = {"config_files_kustomization.yaml": "kustomization.yaml"}, ) kustomization_dir( name = "kingdom", + testonly = True, srcs = [ "resource_requirements.yaml", ":kingdom_gke", @@ -426,7 +431,11 @@ cue_dump( kustomization_dir( name = "reporting_config_files", - srcs = ["reporting_config_files_kustomization.yaml"], + testonly = True, + srcs = [ + "reporting_config_files_kustomization.yaml", + "//src/main/k8s/testing/secretfiles:known_event_group_metadata_type_set", + ], renames = {"reporting_config_files_kustomization.yaml": "kustomization.yaml"}, ) @@ -436,6 +445,7 @@ kustomization_dir( kustomization_dir( name = "reporting", + testonly = True, srcs = [ "resource_requirements.yaml", ":reporting_gke", @@ -472,7 +482,11 @@ cue_dump( kustomization_dir( name = "reporting_v2_config_files", - srcs = ["reporting_v2_config_files_kustomization.yaml"], + testonly = True, + srcs = [ + "reporting_v2_config_files_kustomization.yaml", + "//src/main/k8s/testing/secretfiles:known_event_group_metadata_type_set", + ], renames = {"reporting_v2_config_files_kustomization.yaml": "kustomization.yaml"}, ) @@ -482,6 +496,7 @@ kustomization_dir( kustomization_dir( name = "reporting_v2", + testonly = True, srcs = [ "resource_requirements.yaml", ":reporting_v2_gke", diff --git a/src/main/k8s/dev/config_files_kustomization.yaml b/src/main/k8s/dev/config_files_kustomization.yaml index 2ffb149570a..fb191e02f49 100644 --- a/src/main/k8s/dev/config_files_kustomization.yaml +++ b/src/main/k8s/dev/config_files_kustomization.yaml @@ -16,3 +16,4 @@ configMapGenerator: - name: config-files files: - authority_key_identifier_to_principal_map.textproto + - known_event_group_metadata_type_set.pb diff --git a/src/main/k8s/dev/reporting_config_files_kustomization.yaml b/src/main/k8s/dev/reporting_config_files_kustomization.yaml index 269aca98fd7..a15c366fe23 100644 --- a/src/main/k8s/dev/reporting_config_files_kustomization.yaml +++ b/src/main/k8s/dev/reporting_config_files_kustomization.yaml @@ -18,3 +18,4 @@ configMapGenerator: - authority_key_identifier_to_principal_map.textproto - encryption_key_pair_config.textproto - measurement_spec_config.textproto + - known_event_group_metadata_type_set.pb diff --git a/src/main/k8s/dev/reporting_v2_config_files_kustomization.yaml b/src/main/k8s/dev/reporting_v2_config_files_kustomization.yaml index d58b0bf5559..07593c9aa07 100644 --- a/src/main/k8s/dev/reporting_v2_config_files_kustomization.yaml +++ b/src/main/k8s/dev/reporting_v2_config_files_kustomization.yaml @@ -18,3 +18,4 @@ configMapGenerator: - authority_key_identifier_to_principal_map.textproto - encryption_key_pair_config.textproto - metric_spec_config.textproto + - known_event_group_metadata_type_set.pb diff --git a/src/main/k8s/kingdom.cue b/src/main/k8s/kingdom.cue index 16b2a790335..17f0a84f53f 100644 --- a/src/main/k8s/kingdom.cue +++ b/src/main/k8s/kingdom.cue @@ -65,6 +65,7 @@ import ("strings") _kingdom_cert_collection_file_flag: "--cert-collection-file=/var/run/secrets/files/all_root_certs.pem" _kingdom_root_cert_file_flag: "--cert-collection-file=/var/run/secrets/files/kingdom_root.pem" _akid_to_principal_map_file_flag: "--authority-key-identifier-to-principal-map-file=/etc/\(#AppName)/config-files/authority_key_identifier_to_principal_map.textproto" + _knownEventGroupMetadataTypeFlag: "--known-event-group-metadata-type=/etc/\(#AppName)/config-files/known_event_group_metadata_type_set.pb" _debug_verbose_grpc_client_logging_flag: "--debug-verbose-grpc-client-logging=\(_verboseGrpcClientLogging)" _debug_verbose_grpc_server_logging_flag: "--debug-verbose-grpc-server-logging=\(_verboseGrpcServerLogging)" @@ -131,6 +132,7 @@ import ("strings") _kingdom_tls_key_file_flag, // Internal Kingdom API server should only trust Kingdom certs. _kingdom_root_cert_file_flag, + _knownEventGroupMetadataTypeFlag, _debug_verbose_grpc_server_logging_flag, ] + Container._commonServerFlags + _spannerConfig.flags } @@ -140,6 +142,7 @@ import ("strings") args: _spannerConfig.flags } spec: template: spec: { + _mounts: "config-files": #ConfigMapMount _initContainers: { "update-kingdom-schema": _updateSchemaContainer } diff --git a/src/main/k8s/local/BUILD.bazel b/src/main/k8s/local/BUILD.bazel index fe27c8e4b5b..0a16b1a8a2d 100644 --- a/src/main/k8s/local/BUILD.bazel +++ b/src/main/k8s/local/BUILD.bazel @@ -291,6 +291,7 @@ kustomization_dir( "empty_akid_mapping_config.textproto", "empty_encryption_key_pair_config.textproto", "//src/main/k8s/testing/data:synthetic_generation_specs", + "//src/main/k8s/testing/secretfiles:known_event_group_metadata_type_set", "//src/main/k8s/testing/secretfiles:measurement_spec_config.textproto", "//src/main/k8s/testing/secretfiles:metric_spec_config.textproto", ], @@ -307,6 +308,7 @@ kustomization_dir( "config_files_kustomization.yaml", ":encryption_key_pair_config.textproto", "//src/main/k8s/testing/data:synthetic_generation_specs", + "//src/main/k8s/testing/secretfiles:known_event_group_metadata_type_set", "//src/main/k8s/testing/secretfiles:measurement_spec_config.textproto", "//src/main/k8s/testing/secretfiles:metric_spec_config.textproto", ], diff --git a/src/main/k8s/local/config_files_kustomization.yaml b/src/main/k8s/local/config_files_kustomization.yaml index 7d90a168c8b..fe114f0f15f 100644 --- a/src/main/k8s/local/config_files_kustomization.yaml +++ b/src/main/k8s/local/config_files_kustomization.yaml @@ -22,3 +22,4 @@ configMapGenerator: - synthetic_population_spec.textproto - synthetic_event_group_spec_1.textproto - synthetic_event_group_spec_2.textproto + - known_event_group_metadata_type_set.pb diff --git a/src/main/k8s/local/testing/BUILD.bazel b/src/main/k8s/local/testing/BUILD.bazel index ab97696cee6..d46ac15b4fa 100644 --- a/src/main/k8s/local/testing/BUILD.bazel +++ b/src/main/k8s/local/testing/BUILD.bazel @@ -62,6 +62,7 @@ kustomization_dir( srcs = [ "config_files_kustomization.yaml", "//src/main/k8s/testing/data:synthetic_generation_specs", + "//src/main/k8s/testing/secretfiles:known_event_group_metadata_type_set", ], renames = {"config_files_kustomization.yaml": "kustomization.yaml"}, ) diff --git a/src/main/k8s/local/testing/config_files_kustomization.yaml b/src/main/k8s/local/testing/config_files_kustomization.yaml index 361437132f9..be06e62c888 100644 --- a/src/main/k8s/local/testing/config_files_kustomization.yaml +++ b/src/main/k8s/local/testing/config_files_kustomization.yaml @@ -19,3 +19,4 @@ configMapGenerator: - synthetic_population_spec.textproto - synthetic_event_group_spec_1.textproto - synthetic_event_group_spec_2.textproto + - known_event_group_metadata_type_set.pb diff --git a/src/main/k8s/panelmatch/local/BUILD.bazel b/src/main/k8s/panelmatch/local/BUILD.bazel index 25c1fe135ea..cf16a2fd291 100644 --- a/src/main/k8s/panelmatch/local/BUILD.bazel +++ b/src/main/k8s/panelmatch/local/BUILD.bazel @@ -92,6 +92,7 @@ kustomization_dir( name = "config_files", srcs = [ "config_files_kustomization.yaml", + "//src/main/k8s/testing/secretfiles:known_event_group_metadata_type_set", ], renames = { "config_files_kustomization.yaml": "kustomization.yaml", diff --git a/src/main/k8s/panelmatch/local/config_files_kustomization.yaml b/src/main/k8s/panelmatch/local/config_files_kustomization.yaml index cf8653ff11a..811b72be0a7 100644 --- a/src/main/k8s/panelmatch/local/config_files_kustomization.yaml +++ b/src/main/k8s/panelmatch/local/config_files_kustomization.yaml @@ -15,4 +15,5 @@ configMapGenerator: - name: config-files files: - - authority_key_identifier_to_principal_map.textproto \ No newline at end of file + - authority_key_identifier_to_principal_map.textproto + - known_event_group_metadata_type_set.pb diff --git a/src/main/k8s/reporting.cue b/src/main/k8s/reporting.cue index 209409c494a..e9b5eefb136 100644 --- a/src/main/k8s/reporting.cue +++ b/src/main/k8s/reporting.cue @@ -61,6 +61,7 @@ package k8s _encryptionKeyPairDirFlag: "--key-pair-dir=/var/run/secrets/files" _encryptionKeyPairConfigFileFlag: "--key-pair-config-file=/etc/\(#AppName)/config-files/encryption_key_pair_config.textproto" _measurementSpecConfigFileFlag: "--measurement-spec-config-file=/etc/\(#AppName)/config-files/measurement_spec_config.textproto" + _knownEventGroupMetadataTypeFlag: "--known-event-group-metadata-type=/etc/\(#AppName)/config-files/known_event_group_metadata_type_set.pb" _debugVerboseGrpcClientLoggingFlag: "--debug-verbose-grpc-client-logging=\(_verboseGrpcClientLogging)" _debugVerboseGrpcServerLoggingFlag: "--debug-verbose-grpc-server-logging=\(_verboseGrpcServerLogging)" @@ -116,6 +117,7 @@ package k8s _encryptionKeyPairDirFlag, _encryptionKeyPairConfigFileFlag, _measurementSpecConfigFileFlag, + _knownEventGroupMetadataTypeFlag, "--port=8443", "--health-port=8080", "--event-group-metadata-descriptor-cache-duration=1h", diff --git a/src/main/k8s/reporting_v2.cue b/src/main/k8s/reporting_v2.cue index 68f8b207d60..a63768bdec1 100644 --- a/src/main/k8s/reporting_v2.cue +++ b/src/main/k8s/reporting_v2.cue @@ -67,6 +67,7 @@ package k8s _encryptionKeyPairDirFlag: "--key-pair-dir=/var/run/secrets/files" _encryptionKeyPairConfigFileFlag: "--key-pair-config-file=/etc/\(#AppName)/config-files/encryption_key_pair_config.textproto" _metricSpecConfigFileFlag: "--metric-spec-config-file=/etc/\(#AppName)/config-files/metric_spec_config.textproto" + _knownEventGroupMetadataTypeFlag: "--known-event-group-metadata-type=/etc/\(#AppName)/config-files/known_event_group_metadata_type_set.pb" _debugVerboseGrpcClientLoggingFlag: "--debug-verbose-grpc-client-logging=\(_verboseGrpcClientLogging)" _debugVerboseGrpcServerLoggingFlag: "--debug-verbose-grpc-server-logging=\(_verboseGrpcServerLogging)" @@ -122,6 +123,7 @@ package k8s _encryptionKeyPairDirFlag, _encryptionKeyPairConfigFileFlag, _metricSpecConfigFileFlag, + _knownEventGroupMetadataTypeFlag, "--port=8443", "--health-port=8080", "--event-group-metadata-descriptor-cache-duration=1h", diff --git a/src/main/k8s/testing/secretfiles/BUILD.bazel b/src/main/k8s/testing/secretfiles/BUILD.bazel index 2c7527642c2..81460542db3 100644 --- a/src/main/k8s/testing/secretfiles/BUILD.bazel +++ b/src/main/k8s/testing/secretfiles/BUILD.bazel @@ -1,6 +1,7 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file") -load("//build/k8s:defs.bzl", "k8s_apply", "kustomization_dir") load("@rules_pkg//pkg:pkg.bzl", "pkg_tar") +load("@rules_proto//proto:defs.bzl", "proto_descriptor_set") +load("//build/k8s:defs.bzl", "k8s_apply", "kustomization_dir") package( default_testonly = True, @@ -123,6 +124,14 @@ genrule( tools = ["//src/main/kotlin/org/wfanet/measurement/api/v2alpha/tools:EncryptionPublicKeys"], ) +proto_descriptor_set( + name = "known_event_group_metadata_type_set", + deps = [ + "//src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing:simulator_synthetic_data_spec_proto", + "//src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing:test_metadata_message_proto", + ], +) + SECRET_FILES = [ "all_root_certs.pem", "edp_trusted_certs.pem", diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessCmmsComponents.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessCmmsComponents.kt index 6956eaabae8..b9bd4c32dd3 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessCmmsComponents.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessCmmsComponents.kt @@ -62,8 +62,8 @@ class InProcessCmmsComponents( val kingdom: InProcessKingdom = InProcessKingdom( dataServicesProvider = { kingdomDataServices }, - verboseGrpcLogging = false, REDIRECT_URI, + verboseGrpcLogging = false, ) private val duchies: List by lazy { diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessEdpSimulator.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessEdpSimulator.kt index 1f6209ec69a..d5923d60ff3 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessEdpSimulator.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessEdpSimulator.kt @@ -110,6 +110,7 @@ class InProcessEdpSimulator( 100.0f, ), trustedCertificates = trustedCertificates, + knownEventGroupMetadataTypes = listOf(SyntheticEventGroupSpec.getDescriptor().file), random = random, ) diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessKingdom.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessKingdom.kt index d516bb0003b..385427fb988 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessKingdom.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessKingdom.kt @@ -14,6 +14,7 @@ package org.wfanet.measurement.integration.common +import com.google.protobuf.Descriptors import io.grpc.Channel import java.util.concurrent.TimeUnit import java.util.logging.Logger @@ -68,12 +69,15 @@ import org.wfanet.measurement.loadtest.panelmatchresourcesetup.PanelMatchResourc /** TestRule that starts and stops all Kingdom gRPC services. */ class InProcessKingdom( dataServicesProvider: () -> DataServices, - val verboseGrpcLogging: Boolean = true, /** The open id client redirect uri when creating the authentication uri. */ private val redirectUri: String, + val verboseGrpcLogging: Boolean = true, ) : TestRule { private val kingdomDataServices by lazy { dataServicesProvider() } + val knownEventGroupMetadataTypes: Iterable + get() = kingdomDataServices.knownEventGroupMetadataTypes + private val internalApiChannel by lazy { internalDataServer.channel.withDefaultDeadline( DEFAULT_INTERNAL_DEADLINE_MILLIS, diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessLifeOfAnEventGroupIntegrationTest.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessLifeOfAnEventGroupIntegrationTest.kt index d8d3322bb9c..30a4704ee59 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessLifeOfAnEventGroupIntegrationTest.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/InProcessLifeOfAnEventGroupIntegrationTest.kt @@ -59,8 +59,8 @@ abstract class InProcessLifeOfAnEventGroupIntegrationTest { private val kingdom: InProcessKingdom = InProcessKingdom( dataServicesProvider = { kingdomDataServices }, - verboseGrpcLogging = false, REDIRECT_URI, + verboseGrpcLogging = false, ) @get:Rule diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/InProcessReportingServer.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/InProcessReportingServer.kt index 43df27aa538..2f931a8e5cb 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/InProcessReportingServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/InProcessReportingServer.kt @@ -21,7 +21,6 @@ import java.security.SecureRandom import java.security.cert.X509Certificate import java.time.Duration import java.util.logging.Logger -import kotlinx.coroutines.Dispatchers import org.junit.rules.TestRule import org.junit.runner.Description import org.junit.runners.model.Statement @@ -126,7 +125,7 @@ class InProcessReportingServer( ), EventGroup.getDescriptor(), Duration.ofSeconds(5), - Dispatchers.Default, + knownMetadataTypes = emptyList(), ) listOf( diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessLifeOfAReportIntegrationTest.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessLifeOfAReportIntegrationTest.kt index 1a749b2b3c3..1d38b1e5a88 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessLifeOfAReportIntegrationTest.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessLifeOfAReportIntegrationTest.kt @@ -37,6 +37,7 @@ import org.junit.BeforeClass import org.junit.Rule import org.junit.Test import org.junit.rules.TestRule +import org.junit.runner.Description import org.junit.runners.model.Statement import org.wfanet.measurement.api.v2alpha.DataProviderCertificateKey import org.wfanet.measurement.api.v2alpha.DataProvidersGrpcKt.DataProvidersCoroutineStub @@ -60,10 +61,8 @@ import org.wfanet.measurement.common.getRuntimePath import org.wfanet.measurement.common.testing.ProviderRule import org.wfanet.measurement.common.testing.chainRulesSequentially import org.wfanet.measurement.common.toInterval -import org.wfanet.measurement.config.reporting.EncryptionKeyPairConfig import org.wfanet.measurement.config.reporting.EncryptionKeyPairConfigKt.keyPair import org.wfanet.measurement.config.reporting.EncryptionKeyPairConfigKt.principalKeyPairs -import org.wfanet.measurement.config.reporting.MeasurementConsumerConfig import org.wfanet.measurement.config.reporting.encryptionKeyPairConfig import org.wfanet.measurement.config.reporting.measurementConsumerConfig import org.wfanet.measurement.consent.client.dataprovider.encryptMetadata @@ -143,53 +142,68 @@ abstract class InProcessLifeOfAReportIntegrationTest( abstract val internalReportingServerServices: InternalReportingServer.Services - private val reportingServer: InProcessReportingServer by lazy { - val encryptionKeyPairConfigGenerator: () -> EncryptionKeyPairConfig = { - val measurementConsumerData = inProcessCmmsComponents.getMeasurementConsumerData() + private val reportingServerRule = + object : TestRule { + lateinit var reportingServer: InProcessReportingServer + private set + + private fun buildReportingServer(): InProcessReportingServer { + val measurementConsumerData = inProcessCmmsComponents.getMeasurementConsumerData() + val measurementConsumer = runBlocking { + publicKingdomMeasurementConsumersClient + .withAuthenticationKey(measurementConsumerData.apiAuthenticationKey) + .getMeasurementConsumer( + getMeasurementConsumerRequest { name = measurementConsumerData.name } + ) + } - encryptionKeyPairConfig { - principalKeyPairs += principalKeyPairs { - principal = measurementConsumerData.name - keyPairs += keyPair { - publicKeyFile = "mc_enc_public.tink" - privateKeyFile = "mc_enc_private.tink" + val encryptionKeyPairConfig = encryptionKeyPairConfig { + principalKeyPairs += principalKeyPairs { + principal = measurementConsumerData.name + keyPairs += keyPair { + publicKeyFile = "mc_enc_public.tink" + privateKeyFile = "mc_enc_private.tink" + } } } - } - } - - val measurementConsumerConfigGenerator: suspend () -> MeasurementConsumerConfig = { - val measurementConsumerData = inProcessCmmsComponents.getMeasurementConsumerData() + val measurementConsumerConfig = measurementConsumerConfig { + apiKey = measurementConsumerData.apiAuthenticationKey + signingCertificateName = measurementConsumer.certificate + signingPrivateKeyPath = MC_SIGNING_PRIVATE_KEY_PATH + } - val measurementConsumer = - publicKingdomMeasurementConsumersClient - .withAuthenticationKey(measurementConsumerData.apiAuthenticationKey) - .getMeasurementConsumer( - getMeasurementConsumerRequest { name = measurementConsumerData.name } - ) + return InProcessReportingServer( + internalReportingServerServices, + inProcessCmmsComponents.kingdom.publicApiChannel, + encryptionKeyPairConfig, + SECRETS_DIR, + measurementConsumerConfig, + TRUSTED_CERTIFICATES, + inProcessCmmsComponents.kingdom.knownEventGroupMetadataTypes, + verboseGrpcLogging = false, + ) + } - measurementConsumerConfig { - apiKey = measurementConsumerData.apiAuthenticationKey - signingCertificateName = measurementConsumer.certificate - signingPrivateKeyPath = MC_SIGNING_PRIVATE_KEY_PATH + override fun apply(base: Statement, description: Description): Statement { + return object : Statement() { + override fun evaluate() { + reportingServer = buildReportingServer() + reportingServer.apply(base, description) + } + } } } - InProcessReportingServer( - internalReportingServerServices, - { inProcessCmmsComponents.kingdom.publicApiChannel }, - encryptionKeyPairConfigGenerator, - SECRETS_DIR, - measurementConsumerConfigGenerator, - TRUSTED_CERTIFICATES, - verboseGrpcLogging = false, - ) - } + private val reportingServer: InProcessReportingServer + get() = reportingServerRule.reportingServer @get:Rule - val ruleChain: TestRule by lazy { - chainRulesSequentially(inProcessCmmsComponents, inProcessCmmsComponentsStartup, reportingServer) - } + val ruleChain: TestRule = + chainRulesSequentially( + inProcessCmmsComponents, + inProcessCmmsComponentsStartup, + reportingServerRule, + ) private val publicKingdomMeasurementConsumersClient by lazy { MeasurementConsumersCoroutineStub(inProcessCmmsComponents.kingdom.publicApiChannel) diff --git a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessReportingServer.kt b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessReportingServer.kt index fac39d73fef..42e4c4d3141 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessReportingServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2/InProcessReportingServer.kt @@ -17,6 +17,7 @@ package org.wfanet.measurement.integration.common.reporting.v2 import com.google.protobuf.ByteString +import com.google.protobuf.Descriptors import com.google.protobuf.util.Durations import io.grpc.Channel import io.grpc.Status @@ -77,31 +78,26 @@ import org.wfanet.measurement.reporting.v2alpha.MetricsGrpcKt.MetricsCoroutineSt /** TestRule that starts and stops all Reporting Server gRPC services. */ class InProcessReportingServer( private val internalReportingServerServices: InternalReportingServer.Services, - private val publicKingdomChannelGenerator: () -> Channel, - private val encryptionKeyPairConfigGenerator: () -> EncryptionKeyPairConfig, + private val kingdomPublicApiChannel: Channel, + private val encryptionKeyPairConfig: EncryptionKeyPairConfig, private val signingPrivateKeyDir: File, - private val measurementConsumerConfigGenerator: suspend () -> MeasurementConsumerConfig, + private val measurementConsumerConfig: MeasurementConsumerConfig, private val trustedCertificates: Map, + private val knownEventGroupMetadataTypes: Iterable, private val verboseGrpcLogging: Boolean = true, ) : TestRule { - private val publicKingdomMeasurementConsumersClient by lazy { - PublicKingdomMeasurementConsumersCoroutineStub(publicKingdomChannelGenerator()) - } - private val publicKingdomMeasurementsClient by lazy { - PublicKingdomMeasurementsCoroutineStub(publicKingdomChannelGenerator()) - } - private val publicKingdomCertificatesClient by lazy { - PublicKingdomCertificatesCoroutineStub(publicKingdomChannelGenerator()) - } - private val publicKingdomDataProvidersClient by lazy { - PublicKingdomDataProvidersCoroutineStub(publicKingdomChannelGenerator()) - } - private val publicKingdomEventGroupMetadataDescriptorsClient by lazy { - PublicKingdomEventGroupMetadataDescriptorsCoroutineStub(publicKingdomChannelGenerator()) - } - private val publicKingdomEventGroupsClient by lazy { - PublicKingdomEventGroupsCoroutineStub(publicKingdomChannelGenerator()) - } + private val publicKingdomMeasurementConsumersClient = + PublicKingdomMeasurementConsumersCoroutineStub(kingdomPublicApiChannel) + private val publicKingdomMeasurementsClient = + PublicKingdomMeasurementsCoroutineStub(kingdomPublicApiChannel) + private val publicKingdomCertificatesClient = + PublicKingdomCertificatesCoroutineStub(kingdomPublicApiChannel) + private val publicKingdomDataProvidersClient = + PublicKingdomDataProvidersCoroutineStub(kingdomPublicApiChannel) + private val publicKingdomEventGroupMetadataDescriptorsClient = + PublicKingdomEventGroupMetadataDescriptorsCoroutineStub(kingdomPublicApiChannel) + private val publicKingdomEventGroupsClient = + PublicKingdomEventGroupsCoroutineStub(kingdomPublicApiChannel) private val internalApiChannel by lazy { internalReportingServer.channel } private val internalMeasurementConsumersClient by lazy { @@ -136,8 +132,6 @@ class InProcessReportingServer( runBlocking { logger.info("Building Reporting Server's public API services") - val encryptionKeyPairConfig = encryptionKeyPairConfigGenerator() - val encryptionKeyPairStore = InMemoryEncryptionKeyPairStore( encryptionKeyPairConfig.principalKeyPairsList.associateBy( @@ -153,7 +147,6 @@ class InProcessReportingServer( ) ) - val measurementConsumerConfig = measurementConsumerConfigGenerator() val measurementConsumerName = MeasurementConsumerKey( MeasurementConsumerCertificateKey.fromName( @@ -187,7 +180,7 @@ class InProcessReportingServer( ), EventGroup.getDescriptor(), Duration.ofSeconds(5), - Dispatchers.Default, + knownEventGroupMetadataTypes, ) metricSpecConfig = METRIC_SPEC_CONFIG diff --git a/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/BUILD.bazel index 88985804cd4..470892f1dd1 100644 --- a/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/BUILD.bazel @@ -6,14 +6,14 @@ package( ) kt_jvm_library( - name = "in_process_kingdom", - srcs = [ - "InProcessKingdom.kt", - ], + name = "kingdom_data_services_provider_rule", + srcs = ["KingdomDataServicesProviderRule.kt"], deps = [ - "//src/main/kotlin/org/wfanet/measurement/integration/common:in_process_kingdom", + "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/service:data_services", "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner:services", "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing", + "//src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing:simulator_synthetic_data_spec_kt_jvm_proto", + "//src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing:test_metadata_message_kt_jvm_proto", "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common/testing", "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/spanner/testing", ], diff --git a/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/InProcessKingdom.kt b/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/InProcessKingdom.kt deleted file mode 100644 index bbf07dba4cc..00000000000 --- a/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/InProcessKingdom.kt +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2021 The Cross-Media Measurement Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.wfanet.measurement.integration.deploy.gcloud - -import java.time.Clock -import org.wfanet.measurement.common.identity.RandomIdGenerator -import org.wfanet.measurement.gcloud.spanner.testing.SpannerEmulatorDatabaseRule -import org.wfanet.measurement.integration.common.InProcessKingdom -import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.SpannerDataServices -import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.testing.Schemata - -private const val REDIRECT_URI = "https://localhost:2048" - -fun buildKingdomSpannerEmulatorDatabaseRule(): SpannerEmulatorDatabaseRule { - return SpannerEmulatorDatabaseRule(Schemata.KINGDOM_CHANGELOG_PATH) -} - -fun buildSpannerInProcessKingdom( - databaseRule: SpannerEmulatorDatabaseRule, - clock: Clock = Clock.systemUTC(), - verboseGrpcLogging: Boolean = false, -): InProcessKingdom { - return InProcessKingdom( - dataServicesProvider = { - SpannerDataServices(clock, RandomIdGenerator(clock), databaseRule.databaseClient) - }, - verboseGrpcLogging = verboseGrpcLogging, - REDIRECT_URI, - ) -} diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing/KingdomDataServicesProviderRule.kt b/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/KingdomDataServicesProviderRule.kt similarity index 70% rename from src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing/KingdomDataServicesProviderRule.kt rename to src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/KingdomDataServicesProviderRule.kt index 169673e56e2..e76e9b782e8 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing/KingdomDataServicesProviderRule.kt +++ b/src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud/KingdomDataServicesProviderRule.kt @@ -12,16 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.wfanet.measurement.kingdom.deploy.gcloud.spanner.testing +package org.wfanet.measurement.integration.deploy.gcloud import java.time.Clock import org.junit.runner.Description import org.junit.runners.model.Statement +import org.wfanet.measurement.api.v2alpha.event_group_metadata.testing.SyntheticEventGroupSpec +import org.wfanet.measurement.api.v2alpha.event_group_metadata.testing.TestMetadataMessage import org.wfanet.measurement.common.identity.RandomIdGenerator import org.wfanet.measurement.common.testing.ProviderRule import org.wfanet.measurement.gcloud.spanner.testing.SpannerEmulatorDatabaseRule import org.wfanet.measurement.kingdom.deploy.common.service.DataServices import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.SpannerDataServices +import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.testing.Schemata class KingdomDataServicesProviderRule : ProviderRule { private val spannerDatabase = SpannerEmulatorDatabaseRule(Schemata.KINGDOM_CHANGELOG_PATH) @@ -37,10 +40,20 @@ class KingdomDataServicesProviderRule : ProviderRule { override fun evaluate() { val clock = Clock.systemUTC() dataServices = - SpannerDataServices(clock, RandomIdGenerator(clock), spannerDatabase.databaseClient) + SpannerDataServices( + clock, + RandomIdGenerator(clock), + spannerDatabase.databaseClient, + KNOWN_EVENT_GROUP_METADATA_TYPES, + ) base.evaluate() } } return spannerDatabase.apply(statement, description) } + + companion object { + private val KNOWN_EVENT_GROUP_METADATA_TYPES = + listOf(TestMetadataMessage.getDescriptor().file, SyntheticEventGroupSpec.getDescriptor().file) + } } diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/KingdomDataServer.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/KingdomDataServer.kt index 1dc07e09073..f3fc867fb7f 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/KingdomDataServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/server/KingdomDataServer.kt @@ -14,7 +14,11 @@ package org.wfanet.measurement.kingdom.deploy.common.server +import com.google.protobuf.DescriptorProtos +import com.google.protobuf.Descriptors +import java.io.File import kotlinx.coroutines.runInterruptible +import org.wfanet.measurement.common.ProtoReflection import org.wfanet.measurement.common.grpc.CommonServer import org.wfanet.measurement.common.identity.DuchyInfo import org.wfanet.measurement.common.identity.DuchyInfoFlags @@ -39,6 +43,28 @@ abstract class KingdomDataServer : Runnable { @CommandLine.Mixin private lateinit var roLlv2ProtocolConfigFlags: RoLlv2ProtocolConfigFlags + @CommandLine.Option( + names = ["--known-event-group-metadata-type"], + description = + [ + "File path to FileDescriptorSet containing known EventGroup metadata types.", + "This is in addition to standard protobuf well-known types.", + "Can be specified multiple times.", + ], + required = false, + defaultValue = "", + ) + private fun setKnownEventGroupMetadataTypes(fileDescriptorSetFiles: List) { + val fileDescriptorSets = + fileDescriptorSetFiles.map { file -> + file.inputStream().use { input -> DescriptorProtos.FileDescriptorSet.parseFrom(input) } + } + knownEventGroupMetadataTypes = ProtoReflection.buildFileDescriptors(fileDescriptorSets) + } + + protected lateinit var knownEventGroupMetadataTypes: List + private set + protected suspend fun run(dataServices: DataServices) { DuchyInfo.initializeFromFlags(duchyInfoFlags) DuchyIds.initializeFromFlags(duchyIdsFlags) diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/service/DataServices.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/service/DataServices.kt index 1cb7afeace5..03ccf1f87cd 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/service/DataServices.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/service/DataServices.kt @@ -14,6 +14,7 @@ package org.wfanet.measurement.kingdom.deploy.common.service +import com.google.protobuf.Descriptors import io.grpc.BindableService import kotlin.reflect.full.declaredMemberProperties import org.wfanet.measurement.internal.kingdom.AccountsGrpcKt.AccountsCoroutineImplBase @@ -42,9 +43,15 @@ import org.wfanet.measurement.internal.kingdom.RecurringExchangesGrpcKt.Recurrin import org.wfanet.measurement.internal.kingdom.RequisitionsGrpcKt.RequisitionsCoroutineImplBase interface DataServices { - /** Builds a list of all the Kingdom's internal data-layer services. */ fun buildDataServices(): KingdomDataServices + + /** + * Known types for EventGroup metadata. + * + * This is in addition to standard protobuf well-known types. + */ + val knownEventGroupMetadataTypes: Iterable } data class KingdomDataServices( diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/server/SpannerKingdomDataServer.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/server/SpannerKingdomDataServer.kt index f6ed3413cd5..9bd00eff4d4 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/server/SpannerKingdomDataServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/server/SpannerKingdomDataServer.kt @@ -40,7 +40,7 @@ class SpannerKingdomDataServer : KingdomDataServer() { val idGenerator = RandomIdGenerator(clock) val client = spanner.databaseClient - run(SpannerDataServices(clock, idGenerator, client)) + run(SpannerDataServices(clock, idGenerator, client, knownEventGroupMetadataTypes)) } } } diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/SpannerDataServices.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/SpannerDataServices.kt index f1fe5d3f1bc..2e7c0e8040d 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/SpannerDataServices.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/SpannerDataServices.kt @@ -14,6 +14,7 @@ package org.wfanet.measurement.kingdom.deploy.gcloud.spanner +import com.google.protobuf.Descriptors import java.time.Clock import org.wfanet.measurement.common.identity.IdGenerator import org.wfanet.measurement.gcloud.spanner.AsyncDatabaseClient @@ -24,6 +25,7 @@ class SpannerDataServices( private val clock: Clock, private val idGenerator: IdGenerator, private val client: AsyncDatabaseClient, + override val knownEventGroupMetadataTypes: Iterable = emptyList(), ) : DataServices { override fun buildDataServices(): KingdomDataServices { return KingdomDataServices( @@ -32,7 +34,11 @@ class SpannerDataServices( SpannerCertificatesService(idGenerator, client), SpannerDataProvidersService(idGenerator, client), SpannerModelProvidersService(idGenerator, client), - SpannerEventGroupMetadataDescriptorsService(idGenerator, client), + SpannerEventGroupMetadataDescriptorsService( + idGenerator, + client, + knownEventGroupMetadataTypes, + ), SpannerEventGroupsService(idGenerator, client), SpannerMeasurementConsumersService(idGenerator, client), SpannerMeasurementsService(idGenerator, client), diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/SpannerEventGroupMetadataDescriptorsService.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/SpannerEventGroupMetadataDescriptorsService.kt index 09f94f6a559..c3aba6a6703 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/SpannerEventGroupMetadataDescriptorsService.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/SpannerEventGroupMetadataDescriptorsService.kt @@ -14,9 +14,12 @@ package org.wfanet.measurement.kingdom.deploy.gcloud.spanner +import com.google.protobuf.DescriptorProtos +import com.google.protobuf.Descriptors import io.grpc.Status import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.map +import org.wfanet.measurement.common.ProtoReflection import org.wfanet.measurement.common.grpc.failGrpc import org.wfanet.measurement.common.grpc.grpcRequire import org.wfanet.measurement.common.identity.IdGenerator @@ -27,6 +30,7 @@ import org.wfanet.measurement.internal.kingdom.GetEventGroupMetadataDescriptorRe import org.wfanet.measurement.internal.kingdom.StreamEventGroupMetadataDescriptorsRequest import org.wfanet.measurement.internal.kingdom.UpdateEventGroupMetadataDescriptorRequest import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.DataProviderNotFoundException +import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.EventGroupMetadataDescriptorAlreadyExistsWithTypeException import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.EventGroupMetadataDescriptorNotFoundException import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.KingdomInternalException import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.queries.StreamEventGroupMetadataDescriptors @@ -37,14 +41,24 @@ import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.writers.UpdateEventG class SpannerEventGroupMetadataDescriptorsService( private val idGenerator: IdGenerator, private val client: AsyncDatabaseClient, + knownMetadataTypes: Iterable, ) : EventGroupMetadataDescriptorsCoroutineImplBase() { + private val allKnownMetadataTypes: Set = + knownMetadataTypes.asSequence().plus(ProtoReflection.WELL_KNOWN_TYPES).toSet() + override suspend fun createEventGroupMetadataDescriptor( request: EventGroupMetadataDescriptor ): EventGroupMetadataDescriptor { try { - return CreateEventGroupMetadataDescriptor(request).execute(client, idGenerator) + return CreateEventGroupMetadataDescriptor( + request, + getProtobufTypeNames(request.details.descriptorSet), + ) + .execute(client, idGenerator) } catch (e: DataProviderNotFoundException) { throw e.asStatusRuntimeException(Status.Code.NOT_FOUND, "DataProvider not found.") + } catch (e: EventGroupMetadataDescriptorAlreadyExistsWithTypeException) { + throw e.asStatusRuntimeException(Status.Code.ALREADY_EXISTS) } catch (e: KingdomInternalException) { throw e.asStatusRuntimeException(Status.Code.INTERNAL, "Unexpected internal error") } @@ -73,13 +87,18 @@ class SpannerEventGroupMetadataDescriptorsService( "ExternalEventGroupMetadataDescriptorId unspecified" } try { - return UpdateEventGroupMetadataDescriptor(request.eventGroupMetadataDescriptor) + return UpdateEventGroupMetadataDescriptor( + request.eventGroupMetadataDescriptor, + getProtobufTypeNames(request.eventGroupMetadataDescriptor.details.descriptorSet), + ) .execute(client, idGenerator) } catch (e: EventGroupMetadataDescriptorNotFoundException) { throw e.asStatusRuntimeException( Status.Code.NOT_FOUND, "EventGroupMetadataDescriptor not found.", ) + } catch (e: EventGroupMetadataDescriptorAlreadyExistsWithTypeException) { + throw e.asStatusRuntimeException(Status.Code.ALREADY_EXISTS) } catch (e: KingdomInternalException) { throw e.asStatusRuntimeException(Status.Code.INTERNAL, "Unexpected internal error") } @@ -94,4 +113,16 @@ class SpannerEventGroupMetadataDescriptorsService( .execute(client.singleUse()) .map { it.eventGroupMetadataDescriptor } } + + /** + * Returns the protobuf type names of types defined in [fileDescriptorSet], excluding those + * defined in [allKnownMetadataTypes]. + */ + private fun getProtobufTypeNames( + fileDescriptorSet: DescriptorProtos.FileDescriptorSet + ): List { + return ProtoReflection.buildDescriptors(listOf(fileDescriptorSet), allKnownMetadataTypes).map { + it.fullName + } + } } diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/common/KingdomInternalException.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/common/KingdomInternalException.kt index 9d05c83ae38..33ef66434a7 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/common/KingdomInternalException.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/common/KingdomInternalException.kt @@ -580,6 +580,19 @@ class EventGroupMetadataDescriptorNotFoundException( ) } +class EventGroupMetadataDescriptorAlreadyExistsWithTypeException( + message: String = "EventGroupMetadataDescriptor with same protobuf type already exists", + cause: Throwable? = null, +) : + KingdomInternalException( + ErrorCode.EVENT_GROUP_METADATA_DESCRIPTOR_ALREADY_EXISTS_WITH_TYPE, + message, + cause, + ) { + override val context: Map + get() = emptyMap() +} + class RecurringExchangeNotFoundException( val externalRecurringExchangeId: ExternalId, provideDescription: () -> String = { "RecurringExchange not found" }, diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing/BUILD.bazel index 7a387821b93..302c1160240 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing/BUILD.bazel @@ -14,7 +14,6 @@ kt_jvm_library( srcs = glob(["*.kt"]), resources = ["//src/main/resources/kingdom/spanner"], deps = [ - "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner:services", "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common", "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/gcloud/spanner/testing", ], diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/writers/CreateEventGroupMetadataDescriptor.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/writers/CreateEventGroupMetadataDescriptor.kt index 424885ddf00..fb6ca2335c0 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/writers/CreateEventGroupMetadataDescriptor.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/writers/CreateEventGroupMetadataDescriptor.kt @@ -14,6 +14,8 @@ package org.wfanet.measurement.kingdom.deploy.gcloud.spanner.writers +import com.google.cloud.spanner.ErrorCode +import com.google.cloud.spanner.SpannerException import org.wfanet.measurement.common.identity.ExternalId import org.wfanet.measurement.common.identity.InternalId import org.wfanet.measurement.gcloud.spanner.bufferInsertMutation @@ -21,6 +23,7 @@ import org.wfanet.measurement.gcloud.spanner.set import org.wfanet.measurement.gcloud.spanner.setJson import org.wfanet.measurement.internal.kingdom.EventGroupMetadataDescriptor import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.DataProviderNotFoundException +import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.EventGroupMetadataDescriptorAlreadyExistsWithTypeException import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.KingdomInternalException import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.readers.DataProviderReader import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.readers.EventGroupMetadataDescriptorReader @@ -28,12 +31,13 @@ import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.readers.EventGroupMe /** * Creates a EventGroupMetadataDescriptor in the database. * - * Throws a subclass of [KingdomInternalException] on [execute]. - * - * @throws [DataProviderNotFoundException] DataProvider not found + * Throws one of the following subclasses of [KingdomInternalException] on [execute]: + * * [DataProviderNotFoundException] + * * [EventGroupMetadataDescriptorAlreadyExistsWithTypeException] */ class CreateEventGroupMetadataDescriptor( - private val eventGroupMetadataDescriptor: EventGroupMetadataDescriptor + private val eventGroupMetadataDescriptor: EventGroupMetadataDescriptor, + private val protobufTypeNames: List, ) : SpannerWriter() { init { require(eventGroupMetadataDescriptor.externalEventGroupMetadataDescriptorId == 0L) @@ -58,6 +62,14 @@ class CreateEventGroupMetadataDescriptor( return createNewEventGroupMetadataDescriptor(dataProviderId) } + override suspend fun handleSpannerException(e: SpannerException): EventGroupMetadataDescriptor? { + if (e.errorCode == ErrorCode.ALREADY_EXISTS) { + throw EventGroupMetadataDescriptorAlreadyExistsWithTypeException(cause = e) + } else { + throw e + } + } + private fun TransactionScope.createNewEventGroupMetadataDescriptor( dataProviderId: InternalId ): EventGroupMetadataDescriptor { @@ -76,6 +88,14 @@ class CreateEventGroupMetadataDescriptor( setJson("DescriptorDetailsJson" to eventGroupMetadataDescriptor.details) } + for (protobufTypeName in protobufTypeNames) { + transactionContext.bufferInsertMutation("EventGroupMetadataDescriptorTypes") { + set("DataProviderId" to dataProviderId) + set("EventGroupMetadataDescriptorId" to internalDescriptorId) + set("ProtobufTypeName" to protobufTypeName) + } + } + return eventGroupMetadataDescriptor .toBuilder() .setExternalEventGroupMetadataDescriptorId(externalDescriptorId.value) diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/writers/UpdateEventGroupMetadataDescriptor.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/writers/UpdateEventGroupMetadataDescriptor.kt index 7840f88cf4b..5ccf344ff02 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/writers/UpdateEventGroupMetadataDescriptor.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/writers/UpdateEventGroupMetadataDescriptor.kt @@ -14,11 +14,18 @@ package org.wfanet.measurement.kingdom.deploy.gcloud.spanner.writers +import com.google.cloud.spanner.ErrorCode +import com.google.cloud.spanner.Key +import com.google.cloud.spanner.KeySet +import com.google.cloud.spanner.Mutation +import com.google.cloud.spanner.SpannerException import org.wfanet.measurement.common.identity.ExternalId +import org.wfanet.measurement.gcloud.spanner.bufferInsertMutation import org.wfanet.measurement.gcloud.spanner.bufferUpdateMutation import org.wfanet.measurement.gcloud.spanner.set import org.wfanet.measurement.gcloud.spanner.setJson import org.wfanet.measurement.internal.kingdom.EventGroupMetadataDescriptor +import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.EventGroupMetadataDescriptorAlreadyExistsWithTypeException import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.EventGroupMetadataDescriptorNotFoundException import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.common.KingdomInternalException import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.readers.EventGroupMetadataDescriptorReader @@ -26,12 +33,13 @@ import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.readers.EventGroupMe /** * Update [EventGroupMetadataDescriptor] in the database. * - * Throws a subclass of [KingdomInternalException] on [execute]. - * - * @throws [EventGroupMetadataDescriptorNotFoundException] EventGroupMetadataDescriptor not found + * Throws one of the following subclasses of [KingdomInternalException] on [execute]: + * * [EventGroupMetadataDescriptorNotFoundException] + * * [EventGroupMetadataDescriptorAlreadyExistsWithTypeException] */ class UpdateEventGroupMetadataDescriptor( - private val eventGroupMetadataDescriptor: EventGroupMetadataDescriptor + private val eventGroupMetadataDescriptor: EventGroupMetadataDescriptor, + private val protobufTypeNames: List, ) : SpannerWriter() { override suspend fun TransactionScope.runTransaction(): EventGroupMetadataDescriptor { val internalMetadataDescriptorResult = @@ -46,8 +54,11 @@ class UpdateEventGroupMetadataDescriptor( ExternalId(eventGroupMetadataDescriptor.externalEventGroupMetadataDescriptorId), ) + val dataProviderId = internalMetadataDescriptorResult.internalDataProviderId + val eventGroupMetadataDescriptorId = internalMetadataDescriptorResult.internalDescriptorId + transactionContext.bufferUpdateMutation("EventGroupMetadataDescriptors") { - set("DataProviderId" to internalMetadataDescriptorResult.internalDataProviderId.value) + set("DataProviderId" to dataProviderId) set( "EventGroupMetadataDescriptorId" to internalMetadataDescriptorResult.internalDescriptorId.value @@ -61,6 +72,20 @@ class UpdateEventGroupMetadataDescriptor( setJson("DescriptorDetailsJson" to eventGroupMetadataDescriptor.details) } + transactionContext.buffer( + Mutation.delete( + "EventGroupMetadataDescriptorTypes", + KeySet.prefixRange(Key.of(dataProviderId.value, eventGroupMetadataDescriptorId.value)), + ) + ) + for (protobufTypeName in protobufTypeNames) { + transactionContext.bufferInsertMutation("EventGroupMetadataDescriptorTypes") { + set("DataProviderId" to dataProviderId) + set("EventGroupMetadataDescriptorId" to eventGroupMetadataDescriptorId) + set("ProtobufTypeName" to protobufTypeName) + } + } + return eventGroupMetadataDescriptor } @@ -68,4 +93,12 @@ class UpdateEventGroupMetadataDescriptor( EventGroupMetadataDescriptor { return eventGroupMetadataDescriptor } + + override suspend fun handleSpannerException(e: SpannerException): EventGroupMetadataDescriptor? { + if (e.errorCode == ErrorCode.ALREADY_EXISTS) { + throw EventGroupMetadataDescriptorAlreadyExistsWithTypeException(cause = e) + } else { + throw e + } + } } diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing/BUILD.bazel index 277e5253c7f..53681b28352 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing/BUILD.bazel @@ -13,6 +13,8 @@ kt_jvm_library( "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common:llv2_protocol_config", "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common:ro_llv2_protocol_config", "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/common/testing", + "//src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing:test_metadata_message_2_kt_jvm_proto", + "//src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing:test_metadata_message_kt_jvm_proto", "@wfa_common_jvm//imports/java/com/google/cloud/spanner", "@wfa_common_jvm//imports/java/com/google/common/truth", "@wfa_common_jvm//imports/java/com/google/common/truth/extensions/proto", diff --git a/src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing/EventGroupMetadataDescriptorsServiceTest.kt b/src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing/EventGroupMetadataDescriptorsServiceTest.kt index 3fa8cfcdcd8..6bb44229d58 100644 --- a/src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing/EventGroupMetadataDescriptorsServiceTest.kt +++ b/src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing/EventGroupMetadataDescriptorsServiceTest.kt @@ -16,7 +16,6 @@ package org.wfanet.measurement.kingdom.service.internal.testing import com.google.common.truth.Truth.assertThat import com.google.common.truth.extensions.proto.ProtoTruth.assertThat -import com.google.protobuf.DescriptorProtos.FileDescriptorSet import io.grpc.Status import io.grpc.StatusRuntimeException import java.time.Clock @@ -30,6 +29,9 @@ import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 import org.wfanet.measurement.api.Version +import org.wfanet.measurement.api.v2alpha.event_group_metadata.testing.TestMetadataMessage +import org.wfanet.measurement.api.v2alpha.event_group_metadata.testing.TestMetadataMessage2 +import org.wfanet.measurement.common.ProtoReflection import org.wfanet.measurement.common.identity.IdGenerator import org.wfanet.measurement.common.identity.RandomIdGenerator import org.wfanet.measurement.internal.kingdom.DataProvidersGrpcKt.DataProvidersCoroutineImplBase @@ -48,7 +50,11 @@ import org.wfanet.measurement.kingdom.deploy.common.testing.DuchyIdSetter private const val RANDOM_SEED = 1 private val DETAILS = details { apiVersion = Version.V2_ALPHA.string - descriptorSet = FileDescriptorSet.getDefaultInstance() + descriptorSet = ProtoReflection.buildFileDescriptorSet(TestMetadataMessage.getDescriptor()) +} +private val DETAILS_2 = details { + apiVersion = Version.V2_ALPHA.string + descriptorSet = ProtoReflection.buildFileDescriptorSet(TestMetadataMessage2.getDescriptor()) } @RunWith(JUnit4::class) @@ -163,6 +169,33 @@ abstract class EventGroupMetadataDescriptorsServiceTest< assertThat(exception).hasMessageThat().contains("NOT_FOUND: DataProvider not found") } + @Test + fun `createEventGroupMetadataDescriptor fails for duplicate message type`() = runBlocking { + val externalDataProviderId = + population.createDataProvider(dataProvidersService).externalDataProviderId + val externalDataProviderId2 = + population.createDataProvider(dataProvidersService).externalDataProviderId + eventGroupMetadataDescriptorService.createEventGroupMetadataDescriptor( + eventGroupMetadataDescriptor { + this.externalDataProviderId = externalDataProviderId + details = DETAILS + } + ) + + val exception = + assertFailsWith { + eventGroupMetadataDescriptorService.createEventGroupMetadataDescriptor( + eventGroupMetadataDescriptor { + this.externalDataProviderId = externalDataProviderId2 + details = DETAILS + } + ) + } + + assertThat(exception.status.code).isEqualTo(Status.Code.ALREADY_EXISTS) + assertThat(exception).hasMessageThat().contains("EventGroupMetadataDescriptor") + } + @Test fun `createEventGroupMetadataDescriptor returns existing Descriptor by idempotency key`() = runBlocking { @@ -240,6 +273,43 @@ abstract class EventGroupMetadataDescriptorsServiceTest< ) } + @Test + fun `updateEventGroupMetadataDescriptor fails for duplicate message type`() = runBlocking { + val externalDataProviderId = + population.createDataProvider(dataProvidersService).externalDataProviderId + val externalDataProviderId2 = + population.createDataProvider(dataProvidersService).externalDataProviderId + eventGroupMetadataDescriptorService.createEventGroupMetadataDescriptor( + eventGroupMetadataDescriptor { + this.externalDataProviderId = externalDataProviderId + details = DETAILS + } + ) + val eventGroupMetadataDescriptor = + eventGroupMetadataDescriptorService.createEventGroupMetadataDescriptor( + eventGroupMetadataDescriptor { + this.externalDataProviderId = externalDataProviderId2 + details = DETAILS_2 + } + ) + + val exception = + assertFailsWith { + eventGroupMetadataDescriptorService.updateEventGroupMetadataDescriptor( + updateEventGroupMetadataDescriptorRequest { + this.eventGroupMetadataDescriptor = + eventGroupMetadataDescriptor.copy { + // This should error as the message type is already in use by the other EDP. + details = DETAILS + } + } + ) + } + + assertThat(exception.status.code).isEqualTo(Status.Code.ALREADY_EXISTS) + assertThat(exception).hasMessageThat().contains("EventGroupMetadataDescriptor") + } + @Test fun `streamEventGroupMetadataDescriptors returns all descriptors in order`(): Unit = runBlocking { val externalDataProviderId = @@ -257,7 +327,7 @@ abstract class EventGroupMetadataDescriptorsServiceTest< eventGroupMetadataDescriptorService.createEventGroupMetadataDescriptor( eventGroupMetadataDescriptor { this.externalDataProviderId = externalDataProviderId - details = DETAILS + details = DETAILS_2 } ) @@ -303,7 +373,7 @@ abstract class EventGroupMetadataDescriptorsServiceTest< eventGroupMetadataDescriptorService.createEventGroupMetadataDescriptor( eventGroupMetadataDescriptor { this.externalDataProviderId = externalDataProviderId - details = DETAILS + details = DETAILS_2 } ) @@ -342,7 +412,7 @@ abstract class EventGroupMetadataDescriptorsServiceTest< eventGroupMetadataDescriptorService.createEventGroupMetadataDescriptor( eventGroupMetadataDescriptor { this.externalDataProviderId = externalDataProviderId - details = DETAILS + details = DETAILS_2 } ) @@ -383,7 +453,7 @@ abstract class EventGroupMetadataDescriptorsServiceTest< eventGroupMetadataDescriptorService.createEventGroupMetadataDescriptor( eventGroupMetadataDescriptor { this.externalDataProviderId = externalDataProviderId - details = DETAILS + details = DETAILS_2 } ) diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BigQueryEdpSimulatorRunner.kt b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BigQueryEdpSimulatorRunner.kt index dc6cd7c1961..43ac7814b89 100644 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BigQueryEdpSimulatorRunner.kt +++ b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/BigQueryEdpSimulatorRunner.kt @@ -18,6 +18,7 @@ import com.google.cloud.bigquery.BigQuery import com.google.cloud.bigquery.BigQueryOptions import kotlin.properties.Delegates import org.halo_cmm.uk.pilot.Event +import org.wfanet.measurement.api.v2alpha.event_group_metadata.testing.TestMetadataMessage import org.wfanet.measurement.common.commandLineMain import org.wfanet.measurement.loadtest.config.EventGroupMetadata import picocli.CommandLine @@ -76,6 +77,7 @@ class BigQueryEdpSimulatorRunner : EdpSimulatorRunner() { eventQuery, EdpSimulator.buildEventTemplates(Event.getDescriptor()), mapOf("" to EventGroupMetadata.testMetadata(publisherId)), + listOf(TestMetadataMessage.getDescriptor().file), ) } } diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/CsvEdpSimulatorRunner.kt b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/CsvEdpSimulatorRunner.kt index 590a57e0fa1..58a08ffa4e0 100644 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/CsvEdpSimulatorRunner.kt +++ b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/CsvEdpSimulatorRunner.kt @@ -16,6 +16,7 @@ package org.wfanet.measurement.loadtest.dataprovider import java.io.File import kotlin.properties.Delegates +import org.wfanet.measurement.api.v2alpha.event_group_metadata.testing.TestMetadataMessage import org.wfanet.measurement.api.v2alpha.event_templates.testing.TestEvent import org.wfanet.measurement.common.commandLineMain import org.wfanet.measurement.loadtest.config.EventGroupMetadata @@ -52,6 +53,7 @@ class CsvEdpSimulatorRunner : EdpSimulatorRunner() { eventQuery, EdpSimulator.buildEventTemplates(TestEvent.getDescriptor()), mapOf("" to EventGroupMetadata.testMetadata(publisherId)), + listOf(TestMetadataMessage.getDescriptor().file), ) } } diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulator.kt b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulator.kt index bdfd02db780..84fadee6901 100644 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulator.kt +++ b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulator.kt @@ -174,6 +174,13 @@ class EdpSimulator( private val throttler: Throttler, private val privacyBudgetManager: PrivacyBudgetManager, private val trustedCertificates: Map, + /** + * Known protobuf types for [EventGroupMetadataDescriptor]s. + * + * This is in addition to the standard + * [protobuf well-known types][ProtoReflection.WELL_KNOWN_TYPES]. + */ + private val knownEventGroupMetadataTypes: Iterable = emptyList(), private val sketchEncrypter: SketchEncrypter = SketchEncrypter.Default, private val random: Random = Random, private val logSketchDetails: Boolean = false, @@ -341,14 +348,18 @@ class EdpSimulator( private suspend fun ensureMetadataDescriptor( metadataDescriptor: Descriptors.Descriptor ): EventGroupMetadataDescriptor { - val descriptorSet = ProtoReflection.buildFileDescriptorSet(metadataDescriptor) + val descriptorSet = + ProtoReflection.buildFileDescriptorSet( + metadataDescriptor, + ProtoReflection.WELL_KNOWN_TYPES + knownEventGroupMetadataTypes, + ) val descriptorResource = try { eventGroupMetadataDescriptorsStub.createEventGroupMetadataDescriptor( createEventGroupMetadataDescriptorRequest { parent = edpData.name eventGroupMetadataDescriptor = eventGroupMetadataDescriptor { - this.descriptorSet = ProtoReflection.buildFileDescriptorSet(metadataDescriptor) + this.descriptorSet = descriptorSet } requestId = "type.googleapis.com/${metadataDescriptor.fullName}" } diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulatorRunner.kt b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulatorRunner.kt index a01fdbdc2f6..bea556a5017 100644 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulatorRunner.kt +++ b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulatorRunner.kt @@ -14,6 +14,7 @@ package org.wfanet.measurement.loadtest.dataprovider +import com.google.protobuf.Descriptors import com.google.protobuf.Message import io.grpc.ManagedChannel import java.time.Clock @@ -46,6 +47,7 @@ abstract class EdpSimulatorRunner : Runnable { eventQuery: EventQuery, eventTemplates: Iterable, metadataByReferenceIdSuffix: Map, + knownEventGroupMetadataTypes: Iterable, ) { val clientCerts = SigningCerts.fromPemFiles( @@ -112,6 +114,7 @@ abstract class EdpSimulatorRunner : Runnable { MinimumIntervalThrottler(Clock.systemUTC(), flags.throttlerMinimumInterval), createNoOpPrivacyBudgetManager(), clientCerts.trustedCertificates, + knownEventGroupMetadataTypes = knownEventGroupMetadataTypes, random = random, logSketchDetails = flags.logSketchDetails, ) diff --git a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/SyntheticGeneratorEdpSimulatorRunner.kt b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/SyntheticGeneratorEdpSimulatorRunner.kt index bff9b6b7ef4..710793420ad 100644 --- a/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/SyntheticGeneratorEdpSimulatorRunner.kt +++ b/src/main/kotlin/org/wfanet/measurement/loadtest/dataprovider/SyntheticGeneratorEdpSimulatorRunner.kt @@ -89,6 +89,7 @@ class SyntheticGeneratorEdpSimulatorRunner : EdpSimulatorRunner() { eventQuery, EdpSimulator.buildEventTemplates(eventMessageDescriptor), eventGroupSpecByReferenceIdSuffix, + listOf(SyntheticEventGroupSpec.getDescriptor().file), ) } diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/common/server/V1AlphaPublicApiServer.kt b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/common/server/V1AlphaPublicApiServer.kt index 2ab24692df3..affa050e7e0 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/common/server/V1AlphaPublicApiServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/common/server/V1AlphaPublicApiServer.kt @@ -15,6 +15,8 @@ package org.wfanet.measurement.reporting.deploy.common.server import com.google.protobuf.ByteString +import com.google.protobuf.DescriptorProtos +import com.google.protobuf.Descriptors import io.grpc.Channel import io.grpc.ServerServiceDefinition import java.io.File @@ -27,6 +29,7 @@ import org.wfanet.measurement.api.v2alpha.EventGroupsGrpcKt.EventGroupsCoroutine import org.wfanet.measurement.api.v2alpha.MeasurementConsumersGrpcKt.MeasurementConsumersCoroutineStub as KingdomMeasurementConsumersCoroutineStub import org.wfanet.measurement.api.v2alpha.MeasurementsGrpcKt.MeasurementsCoroutineStub as KingdomMeasurementsCoroutineStub import org.wfanet.measurement.api.withAuthenticationKey +import org.wfanet.measurement.common.ProtoReflection import org.wfanet.measurement.common.api.PrincipalLookup import org.wfanet.measurement.common.api.memoizing import org.wfanet.measurement.common.commandLineMain @@ -111,6 +114,7 @@ private fun run( .withAuthenticationKey(apiKey), EventGroup.getDescriptor(), reportingApiServerFlags.eventGroupMetadataDescriptorCacheDuration, + v1AlphaFlags.knownEventGroupMetadataTypes, Dispatchers.Default, ) @@ -190,4 +194,26 @@ private class V1AlphaFlags { ) lateinit var signingPrivateKeyStoreDir: File private set + + @CommandLine.Option( + names = ["--known-event-group-metadata-type"], + description = + [ + "File path to FileDescriptorSet containing known EventGroup metadata types.", + "This is in addition to standard protobuf well-known types.", + "Can be specified multiple times.", + ], + required = false, + defaultValue = "", + ) + private fun setKnownEventGroupMetadataTypes(fileDescriptorSetFiles: List) { + val fileDescriptorSets = + fileDescriptorSetFiles.map { file -> + file.inputStream().use { input -> DescriptorProtos.FileDescriptorSet.parseFrom(input) } + } + knownEventGroupMetadataTypes = ProtoReflection.buildFileDescriptors(fileDescriptorSets) + } + + lateinit var knownEventGroupMetadataTypes: List + private set } diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/server/V2AlphaPublicApiServer.kt b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/server/V2AlphaPublicApiServer.kt index 2db1375b855..c52a9f797bf 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/server/V2AlphaPublicApiServer.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/server/V2AlphaPublicApiServer.kt @@ -17,6 +17,8 @@ package org.wfanet.measurement.reporting.deploy.v2.common.server import com.google.protobuf.ByteString +import com.google.protobuf.DescriptorProtos +import com.google.protobuf.Descriptors import io.grpc.Channel import io.grpc.Server import io.grpc.ServerServiceDefinition @@ -41,6 +43,7 @@ import org.wfanet.measurement.api.v2alpha.MeasurementConsumerKey import org.wfanet.measurement.api.v2alpha.MeasurementConsumersGrpcKt.MeasurementConsumersCoroutineStub as KingdomMeasurementConsumersCoroutineStub import org.wfanet.measurement.api.v2alpha.MeasurementsGrpcKt.MeasurementsCoroutineStub as KingdomMeasurementsCoroutineStub import org.wfanet.measurement.api.withAuthenticationKey +import org.wfanet.measurement.common.ProtoReflection import org.wfanet.measurement.common.api.PrincipalLookup import org.wfanet.measurement.common.api.memoizing import org.wfanet.measurement.common.commandLineMain @@ -168,6 +171,7 @@ private fun run( .withAuthenticationKey(apiKey), EventGroup.getDescriptor(), reportingApiServerFlags.eventGroupMetadataDescriptorCacheDuration, + v2AlphaPublicServerFlags.knownEventGroupMetadataTypes, Dispatchers.Default, ) @@ -290,4 +294,26 @@ private class V2AlphaPublicServerFlags { ) lateinit var dataProviderCacheExpirationDuration: Duration private set + + @CommandLine.Option( + names = ["--known-event-group-metadata-type"], + description = + [ + "File path to FileDescriptorSet containing known EventGroup metadata types.", + "This is in addition to standard protobuf well-known types.", + "Can be specified multiple times.", + ], + required = false, + defaultValue = "", + ) + private fun setKnownEventGroupMetadataTypes(fileDescriptorSetFiles: List) { + val fileDescriptorSets = + fileDescriptorSetFiles.map { file -> + file.inputStream().use { input -> DescriptorProtos.FileDescriptorSet.parseFrom(input) } + } + knownEventGroupMetadataTypes = ProtoReflection.buildFileDescriptors(fileDescriptorSets) + } + + lateinit var knownEventGroupMetadataTypes: List + private set } diff --git a/src/main/kotlin/org/wfanet/measurement/reporting/service/api/CelEnvProvider.kt b/src/main/kotlin/org/wfanet/measurement/reporting/service/api/CelEnvProvider.kt index e2c65f0f86c..f70e0dbfa61 100644 --- a/src/main/kotlin/org/wfanet/measurement/reporting/service/api/CelEnvProvider.kt +++ b/src/main/kotlin/org/wfanet/measurement/reporting/service/api/CelEnvProvider.kt @@ -16,7 +16,6 @@ package org.wfanet.measurement.reporting.service.api -import com.google.protobuf.DescriptorProtos import com.google.protobuf.Descriptors import com.google.protobuf.TypeRegistry import io.grpc.Status @@ -25,6 +24,7 @@ import java.time.Duration import java.util.logging.Level import java.util.logging.Logger import kotlin.coroutines.CoroutineContext +import kotlin.coroutines.EmptyCoroutineContext import kotlin.coroutines.coroutineContext import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope @@ -39,6 +39,7 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.time.delay +import org.jetbrains.annotations.NonBlockingExecutor import org.jetbrains.annotations.VisibleForTesting import org.projectnessie.cel.Env import org.projectnessie.cel.EnvOption @@ -65,9 +66,12 @@ class CelEnvCacheProvider( /** Protobuf descriptor of Reporting EventGroup message type. */ private val reportingEventGroupDescriptor: Descriptors.Descriptor, private val cacheRefreshInterval: Duration, - coroutineContext: CoroutineContext, + knownMetadataTypes: Iterable, + coroutineContext: @NonBlockingExecutor CoroutineContext = EmptyCoroutineContext, private val numRetriesInitialSync: Int = 3, ) : CelEnvProvider, AutoCloseable { + private val allKnownMetadataTypes: Set = + knownMetadataTypes.asSequence().plus(ProtoReflection.WELL_KNOWN_TYPES).toSet() private lateinit var typeRegistryAndEnv: CelEnvProvider.TypeRegistryAndEnv private val coroutineScope = CoroutineScope(coroutineContext + SupervisorJob()) private val initialSyncJob = CompletableDeferred() @@ -123,10 +127,11 @@ class CelEnvCacheProvider( val eventGroupMetadataDescriptors: List = getEventGroupMetadataDescriptors() - val fileDescriptorSets: List = - eventGroupMetadataDescriptors.map { it.descriptorSet } - val fileDescriptors: List = - ProtoReflection.buildDescriptors(fileDescriptorSets) + val fileDescriptors: List = + ProtoReflection.buildFileDescriptors( + eventGroupMetadataDescriptors.map { it.descriptorSet }, + allKnownMetadataTypes, + ) val env = buildCelEnvironment(fileDescriptors) val typeRegistry: TypeRegistry = buildTypeRegistry(fileDescriptors) @@ -134,11 +139,15 @@ class CelEnvCacheProvider( return CelEnvProvider.TypeRegistryAndEnv(typeRegistry, env) } - private fun buildCelEnvironment(descriptors: List): Env { + private fun buildCelEnvironment(fileDescriptors: Iterable): Env { // Build CEL ProtoTypeRegistry. - val celTypeRegistry = ProtoTypeRegistry.newRegistry() - descriptors.forEach { celTypeRegistry.registerDescriptor(it.file) } - celTypeRegistry.registerDescriptor(reportingEventGroupDescriptor.file) + val celTypeRegistry = + ProtoTypeRegistry.newRegistry().apply { + for (fileDescriptor in fileDescriptors) { + registerDescriptor(fileDescriptor) + } + registerDescriptor(reportingEventGroupDescriptor.file) + } // Build CEL Env. val env = @@ -196,14 +205,8 @@ class CelEnvCacheProvider( } } - private fun buildTypeRegistry(descriptors: List): TypeRegistry { - return TypeRegistry.newBuilder() - .apply { - for (descriptor in descriptors) { - add(descriptor) - } - } - .build() + private fun buildTypeRegistry(fileDescriptors: List): TypeRegistry { + return TypeRegistry.newBuilder().add(fileDescriptors.flatMap { it.messageTypes }).build() } /** Suspends until any in-flight sync operations are complete. */ diff --git a/src/main/kotlin/org/wfanet/panelmatch/integration/AbstractInProcessPanelMatchIntegrationTest.kt b/src/main/kotlin/org/wfanet/panelmatch/integration/AbstractInProcessPanelMatchIntegrationTest.kt index 74ce42abd9b..102f13b1a78 100644 --- a/src/main/kotlin/org/wfanet/panelmatch/integration/AbstractInProcessPanelMatchIntegrationTest.kt +++ b/src/main/kotlin/org/wfanet/panelmatch/integration/AbstractInProcessPanelMatchIntegrationTest.kt @@ -21,7 +21,6 @@ import com.google.protobuf.ByteString import com.google.protobuf.TypeRegistry import io.grpc.StatusException import java.nio.file.Path -import java.time.Clock import java.time.LocalDate import kotlinx.coroutines.CoroutineName import kotlinx.coroutines.CoroutineScope @@ -54,8 +53,8 @@ import org.wfanet.measurement.common.identity.withPrincipalName import org.wfanet.measurement.common.parseTextProto import org.wfanet.measurement.common.testing.chainRulesSequentially import org.wfanet.measurement.common.toProtoDate -import org.wfanet.measurement.integration.deploy.gcloud.buildKingdomSpannerEmulatorDatabaseRule -import org.wfanet.measurement.integration.deploy.gcloud.buildSpannerInProcessKingdom +import org.wfanet.measurement.integration.common.InProcessKingdom +import org.wfanet.measurement.integration.deploy.gcloud.KingdomDataServicesProviderRule import org.wfanet.measurement.loadtest.resourcesetup.EntityContent import org.wfanet.measurement.storage.StorageClient import org.wfanet.measurement.storage.filesystem.FileSystemStorageClient @@ -113,10 +112,9 @@ abstract class AbstractInProcessPanelMatchIntegrationTest { abstract val workflow: ExchangeWorkflow - private val databaseRule = buildKingdomSpannerEmulatorDatabaseRule() - private val inProcessKingdom by lazy { - buildSpannerInProcessKingdom(databaseRule, Clock.systemUTC(), verboseGrpcLogging = false) - } + private val kingdomDataServicesProvider = KingdomDataServicesProviderRule() + private val inProcessKingdom = + InProcessKingdom({ kingdomDataServicesProvider.value }, REDIRECT_URI) private val resourceSetup by lazy { inProcessKingdom.panelMatchResourceSetup } private lateinit var exchangesClient: ExchangesCoroutineStub @@ -127,7 +125,9 @@ abstract class AbstractInProcessPanelMatchIntegrationTest { private lateinit var recurringExchangeId: String @get:Rule - val ruleChain: TestRule by lazy { chainRulesSequentially(databaseRule, inProcessKingdom) } + val ruleChain: TestRule by lazy { + chainRulesSequentially(kingdomDataServicesProvider, inProcessKingdom) + } @get:Rule val dataProviderFolder = TemporaryFolder() @get:Rule val modelProviderFolder = TemporaryFolder() @@ -417,6 +417,8 @@ abstract class AbstractInProcessPanelMatchIntegrationTest { private val typeRegistry = TypeRegistry.newBuilder().add(Shared.Parameters.getDescriptor()).build() + private const val REDIRECT_URI = "https://localhost:2048" + // TODO(@yunyeng): Think about the tests that start running around midnight. val EXCHANGE_DATE: LocalDate = LocalDate.now() diff --git a/src/main/kotlin/org/wfanet/panelmatch/integration/BUILD.bazel b/src/main/kotlin/org/wfanet/panelmatch/integration/BUILD.bazel index f3c1285a5f6..9ed860f7a60 100644 --- a/src/main/kotlin/org/wfanet/panelmatch/integration/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/panelmatch/integration/BUILD.bazel @@ -16,6 +16,8 @@ kt_jvm_library( "//imports/java/org/apache/beam/runners/direct", ], deps = [ + "//src/main/kotlin/org/wfanet/measurement/integration/common:in_process_kingdom", + "//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:kingdom_data_services_provider_rule", "//src/main/kotlin/org/wfanet/panelmatch/client/deploy", "//src/main/kotlin/org/wfanet/panelmatch/client/launcher", "//src/main/kotlin/org/wfanet/panelmatch/client/storage/testing", @@ -45,6 +47,5 @@ kt_jvm_library( "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/storage/testing", "@wfa_measurement_system//src/main/kotlin/org/wfanet/measurement/common/api:resource_key", "@wfa_measurement_system//src/main/kotlin/org/wfanet/measurement/common/identity:principal_identity", - "@wfa_measurement_system//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:in_process_kingdom", ], ) diff --git a/src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing/BUILD.bazel b/src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing/BUILD.bazel index 4c584d6cb37..aeb5019ebdb 100644 --- a/src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing/BUILD.bazel +++ b/src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing/BUILD.bazel @@ -19,6 +19,17 @@ kt_jvm_proto_library( deps = [":test_metadata_message_proto"], ) +proto_library( + name = "test_metadata_message_2_proto", + srcs = ["test_metadata_message_2.proto"], + strip_import_prefix = IMPORT_PREFIX, +) + +kt_jvm_proto_library( + name = "test_metadata_message_2_kt_jvm_proto", + deps = [":test_metadata_message_2_proto"], +) + proto_library( name = "test_parent_metadata_message_proto", srcs = ["test_parent_metadata_message.proto"], diff --git a/src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing/test_metadata_message_2.proto b/src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing/test_metadata_message_2.proto new file mode 100644 index 00000000000..20f1c5329a6 --- /dev/null +++ b/src/main/proto/wfa/measurement/api/v2alpha/event_group_metadata/testing/test_metadata_message_2.proto @@ -0,0 +1,25 @@ + +// Copyright 2021 The Cross-Media Measurement Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package wfa.measurement.api.v2alpha.event_group_metadata.testing; + +option java_package = "org.wfanet.measurement.api.v2alpha.event_group_metadata.testing"; +option java_multiple_files = true; + +message TestMetadataMessage2 { + int32 publisher_id = 1; +} diff --git a/src/main/proto/wfa/measurement/internal/kingdom/error_code.proto b/src/main/proto/wfa/measurement/internal/kingdom/error_code.proto index 75f94f8065c..456e77015b3 100644 --- a/src/main/proto/wfa/measurement/internal/kingdom/error_code.proto +++ b/src/main/proto/wfa/measurement/internal/kingdom/error_code.proto @@ -68,6 +68,8 @@ enum ErrorCode { EVENT_GROUP_INVALID_ARGS = 21; /** EventGroupMetadataDescriptor could not be found. */ EVENT_GROUP_METADATA_DESCRIPTOR_NOT_FOUND = 22; + // An EventGroupMetadataDescriptor already exists with the same protobuf type. + EVENT_GROUP_METADATA_DESCRIPTOR_ALREADY_EXISTS_WITH_TYPE = 43; /** RecurringExchange could not be found. */ RECURRING_EXCHANGE_NOT_FOUND = 23; /** ExchangeStepAttempt could not be found. */ diff --git a/src/main/resources/kingdom/spanner/add-event-group-metadata-descriptor-types.sql b/src/main/resources/kingdom/spanner/add-event-group-metadata-descriptor-types.sql new file mode 100644 index 00000000000..91c86d07976 --- /dev/null +++ b/src/main/resources/kingdom/spanner/add-event-group-metadata-descriptor-types.sql @@ -0,0 +1,30 @@ +-- liquibase formatted sql + +-- Copyright 2024 The Cross-Media Measurement Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- changeset sanjayvas:15 dbms:cloudspanner +-- comment: Add unique index for EventGroupMetadataDescriptor protobuf types. + +CREATE TABLE EventGroupMetadataDescriptorTypes ( + DataProviderId INT64 NOT NULL, + EventGroupMetadataDescriptorId INT64 NOT NULL, + + -- Fully-qualified protobuf type name. + ProtobufTypeName STRING(MAX) NOT NULL, +) PRIMARY KEY (DataProviderId, EventGroupMetadataDescriptorId, ProtobufTypeName), +INTERLEAVE IN PARENT EventGroupMetadataDescriptors ON DELETE CASCADE; + +CREATE UNIQUE INDEX EventGroupMetadataDescriptorTypesByType + ON EventGroupMetadataDescriptorTypes(ProtobufTypeName); diff --git a/src/main/resources/kingdom/spanner/changelog.yaml b/src/main/resources/kingdom/spanner/changelog.yaml index 9818e27f6f8..e59e19b113d 100644 --- a/src/main/resources/kingdom/spanner/changelog.yaml +++ b/src/main/resources/kingdom/spanner/changelog.yaml @@ -60,3 +60,6 @@ databaseChangeLog: - include: file: add-result-api-version.sql relativeToChangeLogFile: true +- include: + file: add-event-group-metadata-descriptor-types.sql + relativeToChangeLogFile: true diff --git a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/BUILD.bazel b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/BUILD.bazel index 69f990ff0a6..cf8dad70911 100644 --- a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/BUILD.bazel +++ b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/BUILD.bazel @@ -12,7 +12,7 @@ spanner_emulator_test( test_class = "org.wfanet.measurement.integration.deploy.gcloud.GCloudInProcessLifeOfAnEventGroupIntegrationTest", deps = [ "//src/main/kotlin/org/wfanet/measurement/integration/common:in_process_event_group_components", - "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing", + "//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:kingdom_data_services_provider_rule", "@wfa_common_jvm//imports/kotlin/kotlinx/coroutines/debug", ], ) @@ -23,8 +23,8 @@ kt_jvm_library( data = ["@cloud_spanner_emulator//:emulator"], deps = [ "//src/main/kotlin/org/wfanet/measurement/integration/common:in_process_life_of_a_measurement_integration_test", + "//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:kingdom_data_services_provider_rule", "//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:spanner_duchy_dependency_provider_rule", - "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing", ], ) @@ -45,7 +45,7 @@ kt_jvm_library( "//src/main/kotlin/org/wfanet/measurement/duchy/deploy/common/postgres/testing", "//src/main/kotlin/org/wfanet/measurement/integration/common:in_process_life_of_a_measurement_integration_test", "//src/main/kotlin/org/wfanet/measurement/integration/deploy/common/postgres:postgres_duchy_dependency_provider_rule", - "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing", + "//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:kingdom_data_services_provider_rule", ], ) @@ -72,7 +72,7 @@ kt_jvm_library( "//src/main/kotlin/org/wfanet/measurement/integration/common:in_process_duchy", "//src/main/kotlin/org/wfanet/measurement/integration/common/reporting/v2:in_process_life_of_a_report_integration_test", "//src/main/kotlin/org/wfanet/measurement/integration/deploy/common/postgres:postgres_duchy_dependency_provider_rule", - "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing", + "//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:kingdom_data_services_provider_rule", "//src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/common/server/postgres:services", "//src/main/kotlin/org/wfanet/measurement/reporting/deploy/v2/postgres/testing", "@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common/db/r2dbc/postgres/testing:database_provider", @@ -97,8 +97,8 @@ kt_jvm_library( data = ["@cloud_spanner_emulator//:emulator"], deps = [ "//src/main/kotlin/org/wfanet/measurement/integration/common:in_process_reach_measurement_accuracy_test", + "//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:kingdom_data_services_provider_rule", "//src/main/kotlin/org/wfanet/measurement/integration/deploy/gcloud:spanner_duchy_dependency_provider_rule", - "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing", ], ) diff --git a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudInProcessLifeOfAReportV2IntegrationTest.kt b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudInProcessLifeOfAReportV2IntegrationTest.kt index eaa3230e4f8..e75f7997d21 100644 --- a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudInProcessLifeOfAReportV2IntegrationTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudInProcessLifeOfAReportV2IntegrationTest.kt @@ -24,7 +24,6 @@ import org.wfanet.measurement.duchy.deploy.common.postgres.testing.Schemata.DUCH import org.wfanet.measurement.integration.common.ALL_DUCHY_NAMES import org.wfanet.measurement.integration.common.reporting.v2.InProcessLifeOfAReportIntegrationTest import org.wfanet.measurement.integration.deploy.common.postgres.PostgresDuchyDependencyProviderRule -import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.testing.KingdomDataServicesProviderRule import org.wfanet.measurement.reporting.deploy.v2.common.server.postgres.PostgresServices import org.wfanet.measurement.reporting.deploy.v2.postgres.testing.Schemata.REPORTING_CHANGELOG_PATH diff --git a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudInProcessLifeOfAnEventGroupIntegrationTest.kt b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudInProcessLifeOfAnEventGroupIntegrationTest.kt index 70ef3820725..3194ea2dc59 100644 --- a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudInProcessLifeOfAnEventGroupIntegrationTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudInProcessLifeOfAnEventGroupIntegrationTest.kt @@ -17,7 +17,6 @@ package org.wfanet.measurement.integration.deploy.gcloud import org.wfanet.measurement.integration.common.InProcessLifeOfAnEventGroupIntegrationTest -import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.testing.KingdomDataServicesProviderRule /** * Implementation of [InProcessLifeOfAnEventGroupIntegrationTest] for GCloud backends (Spanner, diff --git a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudPostgresInProcessLifeOfAMeasurementIntegrationTest.kt b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudPostgresInProcessLifeOfAMeasurementIntegrationTest.kt index 50f24af4955..b1aef1ea519 100644 --- a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudPostgresInProcessLifeOfAMeasurementIntegrationTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudPostgresInProcessLifeOfAMeasurementIntegrationTest.kt @@ -24,7 +24,6 @@ import org.wfanet.measurement.duchy.deploy.common.postgres.testing.Schemata import org.wfanet.measurement.integration.common.ALL_DUCHY_NAMES import org.wfanet.measurement.integration.common.InProcessLifeOfAMeasurementIntegrationTest import org.wfanet.measurement.integration.deploy.common.postgres.PostgresDuchyDependencyProviderRule -import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.testing.KingdomDataServicesProviderRule /** * Implementation of [InProcessLifeOfAMeasurementIntegrationTest] for GCloud backends with Postgres diff --git a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudSpannerInProcessLifeOfAMeasurementIntegrationTest.kt b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudSpannerInProcessLifeOfAMeasurementIntegrationTest.kt index ae1c514920b..cdb5ac04607 100644 --- a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudSpannerInProcessLifeOfAMeasurementIntegrationTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudSpannerInProcessLifeOfAMeasurementIntegrationTest.kt @@ -18,7 +18,6 @@ import org.junit.Rule import org.junit.rules.Timeout import org.wfanet.measurement.integration.common.ALL_DUCHY_NAMES import org.wfanet.measurement.integration.common.InProcessLifeOfAMeasurementIntegrationTest -import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.testing.KingdomDataServicesProviderRule /** * Implementation of [InProcessLifeOfAMeasurementIntegrationTest] for GCloud backends with Spanner diff --git a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudSpannerInProcessReachMeasurementAccuracyTest.kt b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudSpannerInProcessReachMeasurementAccuracyTest.kt index 1d77e064749..a09beb09b00 100644 --- a/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudSpannerInProcessReachMeasurementAccuracyTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/integration/deploy/gcloud/GCloudSpannerInProcessReachMeasurementAccuracyTest.kt @@ -18,7 +18,6 @@ import org.junit.Rule import org.junit.rules.Timeout import org.wfanet.measurement.integration.common.ALL_DUCHY_NAMES import org.wfanet.measurement.integration.common.InProcessReachMeasurementAccuracyTest -import org.wfanet.measurement.kingdom.deploy.gcloud.spanner.testing.KingdomDataServicesProviderRule /** * Implementation of [InProcessReachMeasurementAccuracyTest] for GCloud backends with Spanner diff --git a/src/test/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/BUILD.bazel b/src/test/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/BUILD.bazel index b5728ea8230..b3c8085bbbe 100644 --- a/src/test/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/BUILD.bazel +++ b/src/test/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/BUILD.bazel @@ -39,6 +39,7 @@ spanner_emulator_test( test_class = "org.wfanet.measurement.kingdom.deploy.gcloud.spanner.SpannerMeasurementsServiceTest", deps = [ "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner:measurements_service", + "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner:services", "//src/main/kotlin/org/wfanet/measurement/kingdom/deploy/gcloud/spanner/testing", "//src/main/kotlin/org/wfanet/measurement/kingdom/service/internal/testing", ], diff --git a/src/test/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulatorTest.kt b/src/test/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulatorTest.kt index ff83ebe8e49..c97e80223a6 100644 --- a/src/test/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulatorTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/loadtest/dataprovider/EdpSimulatorTest.kt @@ -347,6 +347,7 @@ class EdpSimulatorTest { @Test fun `ensureEventGroup creates EventGroup and EventGroupMetadataDescriptor`() { + val knownEventGroupMetadataTypes = listOf(SyntheticEventGroupSpec.getDescriptor().file) val edpSimulator = EdpSimulator( EDP_DATA, @@ -362,22 +363,15 @@ class EdpSimulatorTest { dummyThrottler, privacyBudgetManager, TRUSTED_CERTIFICATES, + knownEventGroupMetadataTypes = knownEventGroupMetadataTypes, ) runBlocking { edpSimulator.ensureEventGroup(TEST_EVENT_TEMPLATES, SYNTHETIC_DATA_SPEC) } - // Verify metadata descriptor set contains synthetic data spec. - val createDescriptorRequest: CreateEventGroupMetadataDescriptorRequest = - verifyAndCapture( - eventGroupMetadataDescriptorsServiceMock, - EventGroupMetadataDescriptorsCoroutineImplBase::createEventGroupMetadataDescriptor, - ) - val descriptors = - ProtoReflection.buildDescriptors( - listOf(createDescriptorRequest.eventGroupMetadataDescriptor.descriptorSet) - ) - assertThat(descriptors.map { it.fullName }) - .contains(SyntheticEventGroupSpec.getDescriptor().fullName) + // Verify metadata descriptor is created. + verifyBlocking(eventGroupMetadataDescriptorsServiceMock) { + createEventGroupMetadataDescriptor(any()) + } // Verify EventGroup metadata. val createRequest: CreateEventGroupRequest = diff --git a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/CelEnvProviderTest.kt b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/CelEnvProviderTest.kt index baa6aa177b6..0197f01788b 100644 --- a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/CelEnvProviderTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/CelEnvProviderTest.kt @@ -98,6 +98,7 @@ class CelEnvProviderTest { ), REPORTING_EVENT_GROUP_DESCRIPTOR, Duration.ofMinutes(5), + emptyList(), coroutineContext, ) .use { it.getTypeRegistryAndEnv() } @@ -135,8 +136,9 @@ class CelEnvProviderTest { ), REPORTING_EVENT_GROUP_DESCRIPTOR, Duration.ofMinutes(5), + emptyList(), coroutineContext, - 1, + numRetriesInitialSync = 1, ) .use { advanceTimeBy(CelEnvCacheProvider.RETRY_DELAY.toMillis()) @@ -175,8 +177,9 @@ class CelEnvProviderTest { ), REPORTING_EVENT_GROUP_DESCRIPTOR, Duration.ofMinutes(5), + emptyList(), coroutineContext, - numRetries, + numRetriesInitialSync = numRetries, ) .use { it.getTypeRegistryAndEnv() } } @@ -206,8 +209,9 @@ class CelEnvProviderTest { ), REPORTING_EVENT_GROUP_DESCRIPTOR, Duration.ofMinutes(5), + emptyList(), coroutineContext, - numRetries, + numRetriesInitialSync = numRetries, ) .use { it.getTypeRegistryAndEnv() } } @@ -251,6 +255,7 @@ class CelEnvProviderTest { ), REPORTING_EVENT_GROUP_DESCRIPTOR, cacheRefreshInterval, + emptyList(), coroutineContext, ) .use { diff --git a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v1alpha/EventGroupsServiceTest.kt b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v1alpha/EventGroupsServiceTest.kt index 90e4c89b3e2..17238a9d814 100644 --- a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v1alpha/EventGroupsServiceTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v1alpha/EventGroupsServiceTest.kt @@ -23,7 +23,6 @@ import java.nio.file.Path import java.nio.file.Paths import java.time.Duration import kotlin.test.assertFailsWith -import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.runBlocking import org.junit.Before import org.junit.Rule @@ -190,7 +189,7 @@ class EventGroupsServiceTest { EventGroupMetadataDescriptorsCoroutineStub(grpcTestServerRule.channel), EventGroup.getDescriptor(), Duration.ofSeconds(5), - Dispatchers.Default, + emptyList(), ) service = diff --git a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/EventGroupsServiceTest.kt b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/EventGroupsServiceTest.kt index 1c7a43c6dd8..96efd205a01 100644 --- a/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/EventGroupsServiceTest.kt +++ b/src/test/kotlin/org/wfanet/measurement/reporting/service/api/v2alpha/EventGroupsServiceTest.kt @@ -25,7 +25,6 @@ import java.nio.file.Path import java.nio.file.Paths import java.time.Duration import kotlin.test.assertFailsWith -import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.runBlocking import org.junit.Before import org.junit.Rule @@ -111,7 +110,7 @@ class EventGroupsServiceTest { EventGroupMetadataDescriptorsCoroutineStub(grpcTestServerRule.channel), EventGroup.getDescriptor(), Duration.ofSeconds(5), - Dispatchers.Default, + emptyList(), ) service = @@ -512,7 +511,7 @@ class EventGroupsServiceTest { EventGroupMetadataDescriptorsCoroutineStub(grpcTestServerRule.channel), EventGroup.getDescriptor(), Duration.ofSeconds(5), - Dispatchers.Default, + emptyList(), ) service =