Add GrpcFieldExtraction Filter (envoyproxy#28393)

Signed-off-by: Xuyang Tao <taoxuy@google.com>
Amila-Rukshan · Jul 20, 2023 · 0eeeef3 · 0eeeef3
1 parent 9e833c5
commit 0eeeef3
Show file tree

Hide file tree

Showing 23 changed files with 2,523 additions and 83 deletions.
diff --git a/api/envoy/extensions/filters/http/grpc_field_extraction/v3/BUILD b/api/envoy/extensions/filters/http/grpc_field_extraction/v3/BUILD
@@ -8,6 +8,5 @@ api_proto_package(
     deps = [
         "//envoy/config/core/v3:pkg",
         "@com_github_cncf_udpa//udpa/annotations:pkg",
-        "@com_github_cncf_udpa//xds/annotations/v3:pkg",
     ],
 )
diff --git a/api/envoy/extensions/filters/http/grpc_field_extraction/v3/config.proto b/api/envoy/extensions/filters/http/grpc_field_extraction/v3/config.proto
@@ -4,8 +4,6 @@ package envoy.extensions.filters.http.grpc_field_extraction.v3;
 
 import "envoy/config/core/v3/base.proto";
 
-import "xds/annotations/v3/status.proto";
-
 import "udpa/annotations/status.proto";
 import "validate/validate.proto";
 
@@ -14,92 +12,130 @@ option java_outer_classname = "ConfigProto";
 option java_multiple_files = true;
 option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/grpc_field_extraction/v3;grpc_field_extractionv3";
 option (udpa.annotations.file_status).package_version_status = ACTIVE;
-option (xds.annotations.v3.file_status).work_in_progress = true;
 
-// [#not-implemented-hide:]
 // [#protodoc-title: gRPC Field Extraction]
-// gRPC Field Extraction :ref:`configuration overview
-// <config_http_filters_grpc_field_extraction>`.
+//
 // [#extension: envoy.filters.http.grpc_field_extraction]
-
-// GrpcFieldExtraction filter supports extracting the fields from the first gRPC
+//
+// Overview
+// --------
+//
+// This filter supports extracting the fields from the first gRPC
 // request message no matter if it is unary or streaming and writing the result
 // to the destination, for which currently only the static Envoy dynamic metadata `envoy.filters.http.grpc_field_extraction` is supported.
 //
-// # Assumptions
+// Assumptions
+// -----------
+//
+// This filter assumes
+//
 // 1. this filter is only applicable for gRPC with Protobuf as payload.
 // 2. for bi-directional and client-side gRPC streaming, the initial message from the client should not depend on receiving the server initial metadata.
 //
-// # Process Flow
+// Process Flow
+// ------------
+//
 // When a request reaches the filter, it will check
-// 1. if the request is gRPC request with a protobuf body, the filter tries to:
-//    a. block the incoming data before decoding the first complete gRPC message
-//    b. look up the target field from the buffered gRPC message
-//    c. if the extraction result isn't empty, write it into the dynamic metadata and resume the request propagation.
+//
+// 1. if the request is the gRPC request configured for extraction, the filter tries to:
+//
+//   a. block the incoming data before decoding the first complete gRPC message
+//   b. look up the target field from the buffered gRPC message
+//   c. write the extraction result into the dynamic metadata and resume the request propagation.
+//
 // 2. otherwise, pass through the request.
 //
 // If the request is a malformed one found during 1.a or 1.b, the filter will reject the request.
 //
-// # Config Requirements
-// 1. the target field should be of a singular primitive type or a repeated primitive type
-// and its value will be extracted in string format.
-// 2. the intermediate type could also be repeated.
+// Config Requirements
+// -------------------
+//
+// Here are config requirements
+//
+// 1. the target field should be among the following primitive types: `string`, `uint32`, `uint64`, `int32`, `int64`, `sint32`, `sint64`, `fixed32`, `fixed64`, `sfixed32`, `sfixed64`, `float`, `double`.
+//
+// 2. the target field could be repeated.
+//
+// 3. the intermediate type could also be repeated.
+//
+// Output Format
+// -------------
+//
+// 1. the extracted field names/values will be wrapped in be ``field<StringValue>`` -> ``values<ListValue of StringValue>``, which will be added in the dynamic ``metadata<google.protobuf.Struct>``.
+//
+// 2. if the field value is empty, a empty ``<ListValue>`` will be set.
 //
-// # Output Format
-// The result format will be `field<StringValue>` -> `values<ListValue of StringValue>` in the dynamic metadata<google.protobuf.Struct>.
+// Performance
+// -----------
 //
-// # Performance
 // This filter should be performant as it
+//
 // 1. converts between the gRPC message from EnvoyBuffer without data copy.
 // 2. parse the gRPC message binary directly without deserialization.
+//
 // though buffering the first message introduces some latency.
 //
-// # Example,
-// we have the following request definition for the gRPC method `pkg.svc.Method`.
-//
-// message MethodRequest {
-//   string foo = 1;
-//   Nested nested = 2;
-//   ...
-// }
-//
-// message Nested {
-//  repeated string bar = 1;
-// }
-//
-// This is the filter config(expressed in JSON format).
-// {
-//   "descriptor_set":{...},
-//   "extractions_by_method": {
-//      "pkg.svc.Method":{
-//         "request_field_extractions":{
-//            "foo":{
-//            },
-//            "nested.bar":{
-//            }
-//         }
-//      }
-//   },
-//   ...
-// }
-//
-//
-// During runtime, the filter receives the following `MethodRequest` message(expressed in JSON format).
-// {
-//   foo: "val_foo",
-//   nested: { "bar": ["val_bar1", "val_bar2"]}
-// }
-//
-// The filter will write the following dynamic metadata(expressed in JSON format).
-//
-// `envoy.filters.http.grpc_field_extraction`: {
-//   "foo":[
-//      "val_foo"
-//   ],
-//   "nested.bar":[
-//      "val_bar1", "val_bar2"
-//   ]
-// }
+// Example
+// -------
+//
+// We have the following request definition for the gRPC method `pkg.svc.Method`.
+//
+// .. code-block:: proto
+//
+// 	message MethodRequest {
+// 	  string foo = 1;
+// 	  Nested nested = 2;
+// 	  uint32 baz = 3;
+// 	  ...
+// 	}
+//
+// 	message Nested {
+// 	 repeated string bar = 1;
+// 	}
+//
+// This is the filter config in JSON.
+//
+// .. code-block:: json
+//
+// 	{
+// 	  "descriptor_set":{},
+// 	  "extractions_by_method":{
+// 	     "pkg.svc.Method":{
+// 	        "request_field_extractions":{
+// 	           "foo":{
+// 	           },
+// 	           "nested.bar":{
+// 	           }
+// 	           "baz":{
+// 	           }
+// 	        }
+// 	     }
+// 	  }
+// 	}
+//
+// During runtime, the filter receives the following `MethodRequest` message in JSON.
+//
+// .. code-block:: json
+//
+// 	{
+// 	  "foo": "val_foo",
+// 	  "nested": { "bar": ["val_bar1", "val_bar2"]}
+// 	}
+//
+// The filter will write the following dynamic metadata(`envoy.filters.http.grpc_field_extraction`) in JSON.
+//
+// .. code-block:: json
+//
+// 	{
+// 	  "foo":[
+// 	     "val_foo"
+// 	  ],
+// 	  "nested.bar":[
+// 	     "val_bar1", "val_bar2"
+// 	  ]
+// 	  "baz":[
+// 	  ]
+// 	}
 
 message GrpcFieldExtractionConfig {
   // The proto descriptor set binary for the gRPC services.
@@ -124,17 +160,20 @@ message FieldExtractions {
   // The key is the field path within the grpc request.
   // For example, we can define `foo.bar.name` if we want to extract
   // Request.foo.bar.name.
-  // message Request {
-  //   // The namespace in which the Workspace should be created.
-  //   Foo foo = 1;
-  // }
   //
-  // message Foo {
-  //   Bar bar = 1;
-  // }
-  // message Bar {
-  //   string name = 1;
-  // }
+  // .. code-block:: proto
+  //
+  //  message Request {
+  //    Foo foo = 1;
+  //  }
+  //
+  //  message Foo {
+  //    Bar bar = 1;
+  //  }
+  //
+  //  message Bar {
+  //    string name = 1;
+  //  }
   map<string, RequestFieldValueDisposition> request_field_extractions = 1;
 }
 

diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl
@@ -843,7 +843,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         strip_prefix = "grpc-httpjson-transcoding-{version}",
         urls = ["https://github.com/grpc-ecosystem/grpc-httpjson-transcoding/archive/{version}.tar.gz"],
         use_category = ["dataplane_ext"],
-        extensions = ["envoy.filters.http.grpc_json_transcoder"],
+        extensions = ["envoy.filters.http.grpc_json_transcoder", "envoy.filters.http.grpc_field_extraction"],
         release_date = "2023-06-07",
         cpe = "N/A",
         license = "Apache-2.0",
@@ -858,7 +858,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         strip_prefix = "proto-converter-{version}",
         urls = ["https://github.com/grpc-ecosystem/proto-converter/archive/{version}.zip"],
         use_category = ["dataplane_ext"],
-        extensions = ["envoy.filters.http.grpc_json_transcoder"],
+        extensions = ["envoy.filters.http.grpc_json_transcoder", "envoy.filters.http.grpc_field_extraction"],
         release_date = "2023-06-07",
         cpe = "N/A",
         license = "Apache-2.0",
@@ -868,12 +868,12 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_name = "proto-field-extraction",
         project_desc = "Library that supports the extraction from protobuf binary",
         project_url = "https://github.com/grpc-ecosystem/proto-field-extraction",
-        version = "16ab9b2a3b5e73328ec043167094e837e05da98d",
-        sha256 = "75f19dc2b8c79eb41e03371c34b46250d085e47b45387b80df08bfcfb11599e7",
+        version = "2dfe27548e1f21a665f9068b97b2fc5beb678566",
+        sha256 = "ddbbd0dd07012339ac467f5fdac5c294e1efcdc93bb4b7152d468ddbfc9772f0",
         strip_prefix = "proto-field-extraction-{version}",
         urls = ["https://github.com/grpc-ecosystem/proto-field-extraction/archive/{version}.zip"],
         use_category = ["dataplane_ext"],
-        extensions = ["envoy.filters.http.grpc_field_extraction"],
+        extensions = ["envoy.filters.http.grpc_json_transcoder", "envoy.filters.http.grpc_field_extraction"],
         release_date = "2023-07-11",
         cpe = "N/A",
         license = "Apache-2.0",

diff --git a/docs/root/configuration/http/http_filters/grpc_field_extraction_filter.rst b/docs/root/configuration/http/http_filters/grpc_field_extraction_filter.rst
@@ -0,0 +1,8 @@
+.._config_http_filters_grpc_field_extraction:
+
+gRPC Field Extraction
+=====================
+
+* gRPC :ref:`architecture overview <arch_overview_grpc>`
+* This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.http.grpc_field_extraction.v3.GrpcFieldExtractionConfig``.
+* :ref:`v3 API reference <envoy_v3_api_msg_extensions.filters.http.grpc_field_extraction.v3.GrpcFieldExtractionConfig>`.
diff --git a/docs/root/configuration/http/http_filters/http_filters.rst b/docs/root/configuration/http/http_filters/http_filters.rst
@@ -30,6 +30,7 @@ HTTP filters
   gcp_authn_filter
   geoip_filter
   golang_filter
+  grpc_field_extraction_filter
   grpc_http1_bridge_filter
   grpc_http1_reverse_bridge_filter
   grpc_json_transcoder_filter

diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl
@@ -141,6 +141,7 @@ EXTENSIONS = {
     "envoy.filters.http.file_system_buffer":            "//source/extensions/filters/http/file_system_buffer:config",
     "envoy.filters.http.gcp_authn":                     "//source/extensions/filters/http/gcp_authn:config",
     "envoy.filters.http.geoip":                     "//source/extensions/filters/http/geoip:config",
+    "envoy.filters.http.grpc_field_extraction":         "//source/extensions/filters/http/grpc_field_extraction:config",
     "envoy.filters.http.grpc_http1_bridge":             "//source/extensions/filters/http/grpc_http1_bridge:config",
     "envoy.filters.http.grpc_http1_reverse_bridge":     "//source/extensions/filters/http/grpc_http1_reverse_bridge:config",
     "envoy.filters.http.grpc_json_transcoder":          "//source/extensions/filters/http/grpc_json_transcoder:config",

diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml
@@ -342,6 +342,13 @@ envoy.filters.http.gcp_authn:
   status: alpha
   type_urls:
   - envoy.extensions.filters.http.gcp_authn.v3.GcpAuthnFilterConfig
+envoy.filters.http.grpc_field_extraction:
+  categories:
+  - envoy.filters.http
+  security_posture: unknown
+  status: alpha
+  type_urls:
+  - envoy.extensions.filters.http.grpc_field_extraction.v3.GrpcFieldExtractionConfig
 envoy.filters.http.grpc_http1_bridge:
   categories:
   - envoy.filters.http

diff --git a/source/extensions/filters/http/grpc_field_extraction/BUILD b/source/extensions/filters/http/grpc_field_extraction/BUILD
@@ -0,0 +1,78 @@
+load(
+    "//bazel:envoy_build_system.bzl",
+    "envoy_cc_extension",
+    "envoy_cc_library",
+    "envoy_extension_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_extension_package()
+
+envoy_cc_library(
+    name = "extractor",
+    hdrs = ["extractor.h"],
+    external_deps = ["grpc_transcoding"],
+    deps = [
+        "@com_google_absl//absl/status",
+        "@com_google_protofieldextraction//:all_libs",
+        "@envoy_api//envoy/extensions/filters/http/grpc_field_extraction/v3:pkg_cc_proto",
+    ],
+)
+
+envoy_cc_library(
+    name = "extractor_impl",
+    srcs = ["extractor_impl.cc"],
+    hdrs = ["extractor_impl.h"],
+    external_deps = [
+        "grpc_transcoding",
+    ],
+    deps = [
+        "extractor",
+        "//source/common/common:minimal_logger_lib",
+        "@com_google_absl//absl/status",
+        "@com_google_protofieldextraction//:all_libs",
+        "@envoy_api//envoy/extensions/filters/http/grpc_field_extraction/v3:pkg_cc_proto",
+    ],
+)
+
+envoy_cc_extension(
+    name = "filter_config",
+    srcs = ["filter_config.cc"],
+    hdrs = ["filter_config.h"],
+    deps = [
+        ":extractor_impl",
+        "//source/common/grpc:common_lib",
+        "//source/extensions/filters/http/common:factory_base_lib",
+        "@envoy_api//envoy/extensions/filters/http/grpc_field_extraction/v3:pkg_cc_proto",
+    ],
+)
+
+envoy_cc_library(
+    name = "filter",
+    srcs = ["filter.cc"],
+    hdrs = ["filter.h"],
+    external_deps = [
+        "grpc_transcoding",
+    ],
+    deps = [
+        "extractor_impl",
+        "filter_config",
+        "//source/common/http:codes_lib",
+        "//source/extensions/filters/http/common:pass_through_filter_lib",
+        "//source/extensions/filters/http/grpc_field_extraction/message_converter:message_converter_lib",
+        "@envoy_api//envoy/extensions/filters/http/grpc_field_extraction/v3:pkg_cc_proto",
+    ],
+)
+
+envoy_cc_extension(
+    name = "config",
+    srcs = ["config.cc"],
+    hdrs = ["config.h"],
+    deps = [
+        "extractor_impl",
+        "filter",
+        "filter_config",
+        "@envoy_api//envoy/extensions/filters/http/grpc_field_extraction/v3:pkg_cc_proto",
+    ],
+)