Unify OTEL ports with main server (#577)

This commit switches Aperture to use OTEL fork with custom OTLP receiver. Receiver uses Aperture's GRPC server to listen for OTLP data. Included in this PR: * Changes in OTEL configuration. * Changes in Istio Config chart - only one port to be configured. Breaking change. * Changes in operator. Ref: GH-488
fluxninja · Sep 23, 2022 · ca5d31c · ca5d31c
1 parent bf9e6be
commit ca5d31c
Show file tree

Hide file tree

Showing 29 changed files with 62 additions and 411 deletions.
diff --git a/docs/content/references/configuration/agent.md b/docs/content/references/configuration/agent.md
@@ -1276,18 +1276,6 @@ OtelConfig is the configuration for the OTEL collector.
 #### Properties
 
 <dl>
-<dt>grpc_addr</dt>
-<dd>
-
-(string, `hostname_port`, default: `:4317`) GRPC listener addr for OTEL Collector.
-
-</dd>
-<dt>http_addr</dt>
-<dd>
-
-(string, `hostname_port`, default: `:4318`) HTTP listener addr for OTEL Collector.
-
-</dd>
 <dt>batch_postrollup</dt>
 <dd>
 

diff --git a/docs/content/references/configuration/controller.md b/docs/content/references/configuration/controller.md
@@ -1086,18 +1086,6 @@ OtelConfig is the configuration for the OTEL collector.
 #### Properties
 
 <dl>
-<dt>grpc_addr</dt>
-<dd>
-
-(string, `hostname_port`, default: `:4317`) GRPC listener addr for OTEL Collector.
-
-</dd>
-<dt>http_addr</dt>
-<dd>
-
-(string, `hostname_port`, default: `:4318`) HTTP listener addr for OTEL Collector.
-
-</dd>
 <dt>batch_postrollup</dt>
 <dd>
 

diff --git a/docs/gen/config/agent/config-swagger.yaml b/docs/gen/config/agent/config-swagger.yaml
@@ -704,18 +704,6 @@ definitions:
         $ref: '#/definitions/BatchConfig'
       batch_prerollup:
         $ref: '#/definitions/BatchConfig'
-      grpc_addr:
-        description: GRPC listener addr for OTEL Collector.
-        type: string
-        x-go-default: :4317
-        x-go-name: GRPCAddr
-        x-go-validate: hostname_port
-      http_addr:
-        description: HTTP listener addr for OTEL Collector.
-        type: string
-        x-go-default: :4318
-        x-go-name: HTTPAddr
-        x-go-validate: hostname_port
     title: OtelConfig is the configuration for the OTEL collector.
     type: object
     x-go-package: github.com/fluxninja/aperture/pkg/otel

diff --git a/docs/gen/config/controller/config-swagger.yaml b/docs/gen/config/controller/config-swagger.yaml
@@ -626,18 +626,6 @@ definitions:
         $ref: '#/definitions/BatchConfig'
       batch_prerollup:
         $ref: '#/definitions/BatchConfig'
-      grpc_addr:
-        description: GRPC listener addr for OTEL Collector.
-        type: string
-        x-go-default: :4317
-        x-go-name: GRPCAddr
-        x-go-validate: hostname_port
-      http_addr:
-        description: HTTP listener addr for OTEL Collector.
-        type: string
-        x-go-default: :4318
-        x-go-name: HTTPAddr
-        x-go-validate: hostname_port
     title: OtelConfig is the configuration for the OTEL collector.
     type: object
     x-go-package: github.com/fluxninja/aperture/pkg/otel

diff --git a/go.mod b/go.mod
@@ -62,6 +62,7 @@ require (
 	go.opentelemetry.io/collector/pdata v0.60.0
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.34.0
 	go.opentelemetry.io/otel v1.9.0
+	go.opentelemetry.io/proto/otlp v0.18.0
 	go.uber.org/automaxprocs v1.5.1
 	go.uber.org/fx v1.18.1
 	go.uber.org/goleak v1.2.0
@@ -312,4 +313,5 @@ replace (
 	cloud.google.com/go => cloud.google.com/go v0.100.2
 	github.com/go-openapi/analysis => github.com/fluxninja/analysis v0.21.2-fn.patch.1
 	github.com/go-swagger/go-swagger => github.com/fluxninja/go-swagger v0.29.0-fn.patch.8
+	go.opentelemetry.io/collector => github.com/fluxninja/opentelemetry-collector v0.60.0-fn.patch.1
 )
diff --git a/go.sum b/go.sum
@@ -488,6 +488,8 @@ github.com/fluxninja/go-swagger v0.29.0-fn.patch.8 h1:EdIA15vOFLFgxUMXJPtpI2TaTR
 github.com/fluxninja/go-swagger v0.29.0-fn.patch.8/go.mod h1:hHQC4q+Cd26/JhLSXAUauYmDCTAR9u4TIALMYXZbeWM=
 github.com/fluxninja/lumberjack v0.0.0-20220729045908-655029e4d814 h1:AHC1PtYLUw9PWqmgB9VHcxzGEm1GDYGXzC1PRtgIqqs=
 github.com/fluxninja/lumberjack v0.0.0-20220729045908-655029e4d814/go.mod h1:kARLh/xWGaisrwkOGOWKcqk7fagFUFthOTX7mIFVJlw=
+github.com/fluxninja/opentelemetry-collector v0.60.0-fn.patch.1 h1:k7ZO3DOBXLhEVvTJDLWemN2k+zx8GVJn4WZdj4DDEfQ=
+github.com/fluxninja/opentelemetry-collector v0.60.0-fn.patch.1/go.mod h1:n2KBSgs7AakuedVxLR/Tayl3EEztmngrrjZBsYS+qBI=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
@@ -1509,8 +1511,6 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
-go.opentelemetry.io/collector v0.60.0 h1:rHndW/xILGjNoFaYIvwYpngZnRWw1oQT6GLtzxIs7pw=
-go.opentelemetry.io/collector v0.60.0/go.mod h1:n2KBSgs7AakuedVxLR/Tayl3EEztmngrrjZBsYS+qBI=
 go.opentelemetry.io/collector/pdata v0.60.0 h1:jCNR5jtUom2FcUu30h4tw7enZytwGnXX6fs/K2FM/A0=
 go.opentelemetry.io/collector/pdata v0.60.0/go.mod h1:0hqgNMRneVXaLNelv3q0XKJbyBW9aMDwyC15pKd30+E=
 go.opentelemetry.io/collector/semconv v0.60.0 h1:xy6HSukzA5CC8SR4DvFyLd28EFEOnQgxtpU1bSCM0qY=
@@ -1566,6 +1566,8 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe
 go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ=
 go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.opentelemetry.io/proto/otlp v0.16.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
+go.opentelemetry.io/proto/otlp v0.18.0 h1:W5hyXNComRa23tGpKwG+FRAc4rfF6ZUg1JReK+QHS80=
+go.opentelemetry.io/proto/otlp v0.18.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=

diff --git a/manifests/charts/istioconfig/README.md b/manifests/charts/istioconfig/README.md
@@ -10,8 +10,7 @@ This Chart inserts Envoy filters that integrate with Aperture Agent.
 | ------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- |
 | `envoyFilter.name`             | Name of service running aperture-agent                                                                                                                           | `aperture-agent` |
 | `envoyFilter.namespace`        | Namespace where aperture-agent is running                                                                                                                        | `aperture-agent` |
-| `envoyFilter.authzPort`        | Port serving ext authz API                                                                                                                                       | `8080`           |
-| `envoyFilter.otlpPort`         | Port for streaming access logs                                                                                                                                   | `4317`           |
+| `envoyFilter.port`             | Port serving ext authz API and for streaming access logs                                                                                                         | `8080`           |
 | `envoyFilter.authzGrpcTimeout` | Timeout in seconds to authz requests made to aperture-agent. Note: aperture-agent scheduler has max_timeout parameter that must tuned to match the setting here. | `0.5s`           |
 | `envoyFilter.maxRequestBytes`  | Maximum size of request that is sent over ext authz API                                                                                                          | `8192`           |
 

diff --git a/manifests/charts/istioconfig/templates/envoy_filter.yaml b/manifests/charts/istioconfig/templates/envoy_filter.yaml
@@ -62,7 +62,7 @@ spec:
                   log_name: ingress
                   grpc_service:
                     google_grpc:
-                      target_uri: {{ .Values.envoyFilter.name }}.{{ .Values.envoyFilter.namespace }}.svc.cluster.local:{{ .Values.envoyFilter.otlpPort }}
+                      target_uri: {{ .Values.envoyFilter.name }}.{{ .Values.envoyFilter.namespace }}.svc.cluster.local:{{ .Values.envoyFilter.port }}
                       stat_prefix: aperture_access_log
                   transport_api_version: V3
                 body:
@@ -94,7 +94,7 @@ spec:
                   log_name: egress
                   grpc_service:
                     google_grpc:
-                      target_uri: {{ .Values.envoyFilter.name }}.{{ .Values.envoyFilter.namespace }}.svc.cluster.local:{{ .Values.envoyFilter.otlpPort }}
+                      target_uri: {{ .Values.envoyFilter.name }}.{{ .Values.envoyFilter.namespace }}.svc.cluster.local:{{ .Values.envoyFilter.port }}
                       stat_prefix: aperture_access_log
                   transport_api_version: V3
                 body:
@@ -125,7 +125,7 @@ spec:
           failure_mode_allow: true
           grpc_service:
             google_grpc:
-              target_uri: {{ .Values.envoyFilter.name }}.{{ .Values.envoyFilter.namespace }}.svc.cluster.local:{{ .Values.envoyFilter.authzPort }}
+              target_uri: {{ .Values.envoyFilter.name }}.{{ .Values.envoyFilter.namespace }}.svc.cluster.local:{{ .Values.envoyFilter.port }}
               stat_prefix: ext_authz
             timeout: {{ .Values.envoyFilter.authzGrpcTimeout }}
             initial_metadata:
@@ -155,7 +155,7 @@ spec:
           failure_mode_allow: true
           grpc_service:
             google_grpc:
-              target_uri: {{ .Values.envoyFilter.name }}.{{ .Values.envoyFilter.namespace }}.svc.cluster.local:{{ .Values.envoyFilter.authzPort }}
+              target_uri: {{ .Values.envoyFilter.name }}.{{ .Values.envoyFilter.namespace }}.svc.cluster.local:{{ .Values.envoyFilter.port }}
               stat_prefix: ext_authz
             timeout: {{ .Values.envoyFilter.authzGrpcTimeout }}
             initial_metadata:

diff --git a/manifests/charts/istioconfig/values.yaml b/manifests/charts/istioconfig/values.yaml
@@ -8,10 +8,8 @@ envoyFilter:
   name: aperture-agent
   ## @param envoyFilter.namespace Namespace where aperture-agent is running
   namespace: aperture-agent
-  ## @param envoyFilter.authzPort Port serving ext authz API
-  authzPort: 8080
-  ## @param envoyFilter.otlpPort Port for streaming access logs
-  otlpPort: 4317
+  ## @param envoyFilter.port Port serving ext authz API and for streaming access logs
+  port: 8080
   ## @param envoyFilter.authzGrpcTimeout Timeout in seconds to authz requests made to aperture-agent. Note: aperture-agent scheduler has max_timeout parameter that must tuned to match the setting here.
   authzGrpcTimeout: 0.5s
   ## @param envoyFilter.maxRequestBytes Maximum size of request that is sent over ext authz API

diff --git a/operator/config/crd/bases/fluxninja.com_agents.yaml b/operator/config/crd/bases/fluxninja.com_agents.yaml
@@ -1301,12 +1301,6 @@ spec:
                               will be sent regardless of size.
                             type: string
                         type: object
-                      grpc_addr:
-                        description: GRPC listener addr for OTEL Collector.
-                        type: string
-                      http_addr:
-                        description: HTTP listener addr for OTEL Collector.
-                        type: string
                     type: object
                   peer_discovery:
                     description: Peer discovery configuration.

diff --git a/operator/config/crd/bases/fluxninja.com_controllers.yaml b/operator/config/crd/bases/fluxninja.com_controllers.yaml
@@ -1196,12 +1196,6 @@ spec:
                               will be sent regardless of size.
                             type: string
                         type: object
-                      grpc_addr:
-                        description: GRPC listener addr for OTEL Collector.
-                        type: string
-                      http_addr:
-                        description: HTTP listener addr for OTEL Collector.
-                        type: string
                     type: object
                   plugins:
                     description: Plugins configuration.

diff --git a/operator/controllers/agent/config_test.tpl b/operator/controllers/agent/config_test.tpl
@@ -118,8 +118,6 @@ otel:
   batch_prerollup:
     send_batch_size: 15000
     timeout: 1s
-  grpc_addr: :4317
-  http_addr: :4318
 peer_discovery:
   advertisement_addr: ""
 plugins:

diff --git a/operator/controllers/agent/configmaps_test.go b/operator/controllers/agent/configmaps_test.go
@@ -76,8 +76,6 @@ var _ = Describe("ConfigMap for Agent", func() {
 								DisabledPlugins: []string{"aperture-plugin-fluxninja"},
 							},
 							Otel: otel.OtelConfig{
-								GRPCAddr: ":4317",
-								HTTPAddr: ":4318",
 								BatchPrerollup: otel.BatchConfig{
 									Timeout:       config.MakeDuration(1 * time.Second),
 									SendBatchSize: 15000,

diff --git a/operator/controllers/agent/daemonset.go b/operator/controllers/agent/daemonset.go
@@ -58,16 +58,6 @@ func daemonsetForAgent(instance *agentv1alpha1.Agent, log logr.Logger, scheme *r
 		return nil, err
 	}
 
-	otelGRPCPort, err := controllers.GetPort(spec.ConfigSpec.Otel.GRPCAddr)
-	if err != nil {
-		return nil, err
-	}
-
-	otelHTTPPort, err := controllers.GetPort(spec.ConfigSpec.Otel.HTTPAddr)
-	if err != nil {
-		return nil, err
-	}
-
 	distCachePort, err := controllers.GetPort(spec.ConfigSpec.DistCache.BindAddr)
 	if err != nil {
 		return nil, err
@@ -120,16 +110,6 @@ func daemonsetForAgent(instance *agentv1alpha1.Agent, log logr.Logger, scheme *r
 									ContainerPort: serverPort,
 									Protocol:      corev1.ProtocolTCP,
 								},
-								{
-									Name:          controllers.GrpcOtel,
-									ContainerPort: otelGRPCPort,
-									Protocol:      corev1.ProtocolTCP,
-								},
-								{
-									Name:          controllers.HTTPOtel,
-									ContainerPort: otelHTTPPort,
-									Protocol:      corev1.ProtocolTCP,
-								},
 								{
 									Name:          controllers.DistCache,
 									ContainerPort: distCachePort,

diff --git a/operator/controllers/agent/daemonset_test.go b/operator/controllers/agent/daemonset_test.go
@@ -106,10 +106,7 @@ var _ = Describe("Agent Daemonset", func() {
 									Addr: ":80",
 								},
 							},
-							Otel: otel.OtelConfig{
-								GRPCAddr: ":4317",
-								HTTPAddr: ":4318",
-							},
+							Otel: otel.OtelConfig{},
 						},
 						DistCache: distcache.DistCacheConfig{
 							BindAddr:           ":3320",
@@ -209,16 +206,6 @@ var _ = Describe("Agent Daemonset", func() {
 											ContainerPort: 80,
 											Protocol:      corev1.ProtocolTCP,
 										},
-										{
-											Name:          GrpcOtel,
-											ContainerPort: 4317,
-											Protocol:      corev1.ProtocolTCP,
-										},
-										{
-											Name:          HTTPOtel,
-											ContainerPort: 4318,
-											Protocol:      corev1.ProtocolTCP,
-										},
 										{
 											Name:          DistCache,
 											ContainerPort: 3320,
@@ -290,10 +277,7 @@ var _ = Describe("Agent Daemonset", func() {
 									Addr: ":80",
 								},
 							},
-							Otel: otel.OtelConfig{
-								GRPCAddr: ":4317",
-								HTTPAddr: ":4318",
-							},
+							Otel: otel.OtelConfig{},
 						},
 						DistCache: distcache.DistCacheConfig{
 							BindAddr:           ":3320",
@@ -493,16 +477,6 @@ var _ = Describe("Agent Daemonset", func() {
 											ContainerPort: 80,
 											Protocol:      corev1.ProtocolTCP,
 										},
-										{
-											Name:          GrpcOtel,
-											ContainerPort: 4317,
-											Protocol:      corev1.ProtocolTCP,
-										},
-										{
-											Name:          HTTPOtel,
-											ContainerPort: 4318,
-											Protocol:      corev1.ProtocolTCP,
-										},
 										{
 											Name:          DistCache,
 											ContainerPort: 3320,

diff --git a/operator/controllers/agent/services.go b/operator/controllers/agent/services.go
@@ -54,16 +54,6 @@ func serviceForAgent(instance *agentv1alpha1.Agent, log logr.Logger, scheme *run
 		return nil, err
 	}
 
-	otelGRPCPort, err := controllers.GetPort(spec.ConfigSpec.Otel.GRPCAddr)
-	if err != nil {
-		return nil, err
-	}
-
-	otelHTTPPort, err := controllers.GetPort(spec.ConfigSpec.Otel.HTTPAddr)
-	if err != nil {
-		return nil, err
-	}
-
 	distCachePort, err := controllers.GetPort(spec.ConfigSpec.DistCache.BindAddr)
 	if err != nil {
 		return nil, err
@@ -89,18 +79,6 @@ func serviceForAgent(instance *agentv1alpha1.Agent, log logr.Logger, scheme *run
 					Port:       int32(serverPort),
 					TargetPort: intstr.FromString(controllers.Server),
 				},
-				{
-					Name:       controllers.GrpcOtel,
-					Protocol:   corev1.Protocol(controllers.TCP),
-					Port:       int32(otelGRPCPort),
-					TargetPort: intstr.FromString(controllers.GrpcOtel),
-				},
-				{
-					Name:       controllers.HTTPOtel,
-					Protocol:   corev1.Protocol(controllers.TCP),
-					Port:       int32(otelHTTPPort),
-					TargetPort: intstr.FromString(controllers.HTTPOtel),
-				},
 				{
 					Name:       controllers.DistCache,
 					Protocol:   corev1.Protocol(controllers.TCP),