Deploy FLP as a service

This is an intermediate alternative between the Kafka mode and the
Direct mode, that is more suitable for quick install on large clusters
(Kafka mode is a more complex setup, whereas Direct mode isn't suitable
on large clusters due to the memory consumption of FLP)

To use it, set `deploymentModel` to `Service`.

There are potential caveat to check:
- Without sticky session, no guarantee that the agents talk to the same
  FLP instance. I don't think it's an issue in the nominal case, but
might be a problem for conversation tracking?

Author: jotak

api/flowcollector/v1beta2/flowcollector_types.go

@@ -27,8 +27,9 @@ import (
 type FlowCollectorDeploymentModel string
 
 const (
-	DeploymentModelDirect FlowCollectorDeploymentModel = "Direct"
-	DeploymentModelKafka  FlowCollectorDeploymentModel = "Kafka"
+	DeploymentModelDirect  FlowCollectorDeploymentModel = "Direct"
+	DeploymentModelKafka   FlowCollectorDeploymentModel = "Kafka"
+	DeploymentModelService FlowCollectorDeploymentModel = "Service"
 )
 
 // Please notice that the FlowCollectorSpec's properties MUST redefine one of the default
@@ -69,11 +70,12 @@ type FlowCollectorSpec struct {
 	ConsolePlugin FlowCollectorConsolePlugin `json:"consolePlugin,omitempty"`
 
 	// `deploymentModel` defines the desired type of deployment for flow processing. Possible values are:<br>
-	// - `Direct` (default) to make the flow processor listen directly from the agents.<br>
+	// - `Direct` (default) to make the flow processor listen directly from the agents using the host network.<br>
+	// - `Service` to make the flow processor listen as a Kubernetes Service.<br>
 	// - `Kafka` to make flows sent to a Kafka pipeline before consumption by the processor.<br>
 	// Kafka can provide better scalability, resiliency, and high availability (for more details, see https://www.redhat.com/en/topics/integration/what-is-apache-kafka).
 	// +unionDiscriminator
-	// +kubebuilder:validation:Enum:="Direct";"Kafka"
+	// +kubebuilder:validation:Enum:="Direct";"Service";"Kafka"
 	// +kubebuilder:default:=Direct
 	DeploymentModel FlowCollectorDeploymentModel `json:"deploymentModel,omitempty"`
 
@@ -633,15 +635,26 @@ type FlowCollectorFLP struct {
 
 	//+kubebuilder:validation:Minimum=0
 	//+kubebuilder:default:=3
-	// `kafkaConsumerReplicas` defines the number of replicas (pods) to start for `flowlogs-pipeline-transformer`, which consumes Kafka messages.
+	// `kafkaConsumerReplicas` [deprecated (*)] defines the number of replicas (pods) to start for `flowlogs-pipeline-transformer`, which consumes Kafka messages.
 	// This setting is ignored when Kafka is disabled.
+	// Deprecation notice: use `spec.processor.consumerReplicas` instead.
 	KafkaConsumerReplicas *int32 `json:"kafkaConsumerReplicas,omitempty"`
 
-	// `kafkaConsumerAutoscaler` is the spec of a horizontal pod autoscaler to set up for `flowlogs-pipeline-transformer`, which consumes Kafka messages.
+	// `kafkaConsumerAutoscaler` [deprecated (*)] is the spec of a horizontal pod autoscaler to set up for `flowlogs-pipeline-transformer`, which consumes Kafka messages.
 	// This setting is ignored when Kafka is disabled.
+	// Deprecation notice: managed autoscaler will be removed in a future version. You may configure instead an autoscaler of your choice, and set `spec.processor.unmanagedReplicas` to `true`.
 	// +optional
 	KafkaConsumerAutoscaler FlowCollectorHPA `json:"kafkaConsumerAutoscaler,omitempty"`
 
+	//+kubebuilder:validation:Minimum=0
+	// `consumerReplicas` defines the number of replicas (pods) to start for `flowlogs-pipeline`.
+	// This setting is ignored when `spec.deploymentModel` is `Direct` or when `spec.processor.unmanagedReplicas` is `true`.
+	ConsumerReplicas *int32 `json:"consumerReplicas,omitempty"`
+
+	// If `unmanagedReplicas` is `true`, the operator will not reconcile `consumerReplicas`. This is useful when using a pod autoscaler.
+	// +optional
+	UnmanagedReplicas bool `json:"unmanagedReplicas,omitempty"`
+
 	//+kubebuilder:default:=1000
 	// +optional
 	// `kafkaConsumerQueueCapacity` defines the capacity of the internal message queue used in the Kafka consumer client. Ignored when not using Kafka.
@@ -1032,6 +1045,10 @@ type FlowCollectorConsolePlugin struct {
 	// `replicas` defines the number of replicas (pods) to start.
 	Replicas *int32 `json:"replicas,omitempty"`
 
+	// If `unmanagedReplicas` is `true`, the operator will not reconcile `replicas`. This is useful when using a pod autoscaler.
+	// +optional
+	UnmanagedReplicas bool `json:"unmanagedReplicas,omitempty"`
+
 	//+kubebuilder:validation:Enum=IfNotPresent;Always;Never
 	//+kubebuilder:default:=IfNotPresent
 	// `imagePullPolicy` is the Kubernetes pull policy for the image defined above
@@ -1048,7 +1065,8 @@ type FlowCollectorConsolePlugin struct {
 	// `logLevel` for the console plugin backend
 	LogLevel string `json:"logLevel,omitempty"`
 
-	// `autoscaler` spec of a horizontal pod autoscaler to set up for the plugin Deployment.
+	// `autoscaler` [deprecated (*)] spec of a horizontal pod autoscaler to set up for the plugin Deployment.
+	// Deprecation notice: managed autoscaler will be removed in a future version. You may configure instead an autoscaler of your choice, and set `spec.consolePlugin.unmanagedReplicas` to `true`.
 	// +optional
 	Autoscaler FlowCollectorHPA `json:"autoscaler,omitempty"`
 

api/flowcollector/v1beta2/zz_generated.deepcopy.go

@@ -2750,8 +2750,9 @@ spec:
                         type: object
                     type: object
                   autoscaler:
-                    description: '`autoscaler` spec of a horizontal pod autoscaler
-                      to set up for the plugin Deployment.'
+                    description: |-
+                      `autoscaler` [deprecated (*)] spec of a horizontal pod autoscaler to set up for the plugin Deployment.
+                      Deprecation notice: managed autoscaler will be removed in a future version. You may configure instead an autoscaler of your choice, and set `spec.consolePlugin.unmanagedReplicas` to `true`.
                     properties:
                       maxReplicas:
                         default: 3
@@ -3211,16 +3212,22 @@ spec:
                           More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
                         type: object
                     type: object
+                  unmanagedReplicas:
+                    description: If `unmanagedReplicas` is `true`, the operator will
+                      not reconcile `replicas`. This is useful when using a pod autoscaler.
+                    type: boolean
                 type: object
               deploymentModel:
                 default: Direct
                 description: |-
                   `deploymentModel` defines the desired type of deployment for flow processing. Possible values are:<br>
-                  - `Direct` (default) to make the flow processor listen directly from the agents.<br>
+                  - `Direct` (default) to make the flow processor listen directly from the agents using the host network.<br>
+                  - `Service` to make the flow processor listen as a Kubernetes Service.<br>
                   - `Kafka` to make flows sent to a Kafka pipeline before consumption by the processor.<br>
                   Kafka can provide better scalability, resiliency, and high availability (for more details, see https://www.redhat.com/en/topics/integration/what-is-apache-kafka).
                 enum:
                 - Direct
+                - Service
                 - Kafka
                 type: string
               exporters:
@@ -5366,6 +5373,13 @@ spec:
                       in the flows data. This is useful in a multi-cluster context.
                       When using OpenShift, leave empty to make it automatically determined.'
                     type: string
+                  consumerReplicas:
+                    description: |-
+                      `consumerReplicas` defines the number of replicas (pods) to start for `flowlogs-pipeline`.
+                      This setting is ignored when `spec.deploymentModel` is `Direct` or when `spec.processor.unmanagedReplicas` is `true`.
+                    format: int32
+                    minimum: 0
+                    type: integer
                   deduper:
                     description: '`deduper` allows you to sample or drop flows identified
                       as duplicates, in order to save on resource usage.'
@@ -5434,8 +5448,9 @@ spec:
                     type: string
                   kafkaConsumerAutoscaler:
                     description: |-
-                      `kafkaConsumerAutoscaler` is the spec of a horizontal pod autoscaler to set up for `flowlogs-pipeline-transformer`, which consumes Kafka messages.
+                      `kafkaConsumerAutoscaler` [deprecated (*)] is the spec of a horizontal pod autoscaler to set up for `flowlogs-pipeline-transformer`, which consumes Kafka messages.
                       This setting is ignored when Kafka is disabled.
+                      Deprecation notice: managed autoscaler will be removed in a future version. You may configure instead an autoscaler of your choice, and set `spec.processor.unmanagedReplicas` to `true`.
                     properties:
                       maxReplicas:
                         default: 3
@@ -5749,8 +5764,9 @@ spec:
                   kafkaConsumerReplicas:
                     default: 3
                     description: |-
-                      `kafkaConsumerReplicas` defines the number of replicas (pods) to start for `flowlogs-pipeline-transformer`, which consumes Kafka messages.
+                      `kafkaConsumerReplicas` [deprecated (*)] defines the number of replicas (pods) to start for `flowlogs-pipeline-transformer`, which consumes Kafka messages.
                       This setting is ignored when Kafka is disabled.
+                      Deprecation notice: use `spec.processor.consumerReplicas` instead.
                     format: int32
                     minimum: 0
                     type: integer
@@ -6052,6 +6068,11 @@ spec:
                           external traffic: flows that are not labeled for those subnets are external to the cluster. Enabled by default on OpenShift.
                         type: boolean
                     type: object
+                  unmanagedReplicas:
+                    description: If `unmanagedReplicas` is `true`, the operator will
+                      not reconcile `consumerReplicas`. This is useful when using
+                      a pod autoscaler.
+                    type: boolean
                 type: object
               prometheus:
                 description: '`prometheus` defines Prometheus settings, such as querier

bundle/manifests/flows.netobserv.io_flowcollectors.yaml

@@ -253,7 +253,7 @@ metadata:
     categories: Monitoring, Networking, Observability
     console.openshift.io/plugins: '["netobserv-plugin"]'
     containerImage: quay.io/netobserv/network-observability-operator:1.9.1-community
-    createdAt: "2025-07-21T09:56:28Z"
+    createdAt: "2025-09-05T12:57:16Z"
     description: Network flows collector and monitoring solution
     operatorframework.io/initialization-resource: '{"apiVersion":"flows.netobserv.io/v1beta2",
       "kind":"FlowCollector","metadata":{"name":"cluster"},"spec": {}}'
@@ -568,6 +568,8 @@ spec:
         path: consolePlugin.portNaming.enable
       - displayName: Port names
         path: consolePlugin.portNaming.portNames
+      - displayName: Unmanaged replicas
+        path: consolePlugin.unmanagedReplicas
       - displayName: Address
         path: kafka.address
       - displayName: Topic
@@ -606,6 +608,8 @@ spec:
         path: networkPolicy.additionalNamespaces
       - displayName: Enable
         path: networkPolicy.enable
+      - displayName: Consumer replicas
+        path: processor.consumerReplicas
       - displayName: Deduper
         path: processor.deduper
       - displayName: Mode
@@ -626,6 +630,8 @@ spec:
         path: processor.subnetLabels
       - displayName: Custom labels
         path: processor.subnetLabels.customLabels
+      - displayName: Unmanaged replicas
+        path: processor.unmanagedReplicas
       - displayName: Prometheus
         path: prometheus
       - displayName: Querier
@@ -724,7 +730,7 @@ spec:
 
     - Quick filters (`spec.consolePlugin.quickFilters`): configure preset filters to be displayed in the Console plugin. They offer a way to quickly switch from filters to others, such as showing / hiding pods network, or infrastructure network, or application network, etc. They can be tuned to reflect the different workloads running on your cluster. For a list of available filters, [check this page](https://github.com/netobserv/network-observability-operator/blob/1.9.1-community/docs/QuickFilters.md).
 
-    - Kafka (`spec.deploymentModel: KAFKA` and `spec.kafka`): when enabled, integrates the flow collection pipeline with Kafka, by splitting ingestion from transformation (kube enrichment, derived metrics, ...). Kafka can provide better scalability, resiliency and high availability ([view more details](https://www.redhat.com/en/topics/integration/what-is-apache-kafka)). Assumes Kafka is already deployed and a topic is created.
+    - Kafka (`spec.deploymentModel: Kafka` and `spec.kafka`): when enabled, integrates the flow collection pipeline with Kafka, by splitting ingestion from transformation (kube enrichment, derived metrics, ...). Kafka can provide better scalability, resiliency and high availability ([view more details](https://www.redhat.com/en/topics/integration/what-is-apache-kafka)). Assumes Kafka is already deployed and a topic is created.
 
     - Exporters (`spec.exporters`) an optional list of exporters to which to send enriched flows. KAFKA and IPFIX exporters are supported. This allows you to define any custom storage or processing that can read from Kafka or use the IPFIX standard.
 
@@ -1070,7 +1076,7 @@ spec:
                 - name: RELATED_IMAGE_CONSOLE_PLUGIN
                   value: quay.io/netobserv/network-observability-console-plugin:v1.9.1-community
                 - name: RELATED_IMAGE_CONSOLE_PLUGIN_COMPAT
-                  value: quay.io/netobserv/network-observability-console-plugin-pf4:v1.8.2-community
+                  value: quay.io/netobserv/network-observability-console-plugin:v1.9.1-community
                 - name: DOWNSTREAM_DEPLOYMENT
                   value: "false"
                 - name: PROFILING_BIND_ADDRESS
@@ -1215,9 +1221,7 @@ spec:
   - image: quay.io/netobserv/flowlogs-pipeline:v1.9.1-community
     name: flowlogs-pipeline
   - image: quay.io/netobserv/network-observability-console-plugin:v1.9.1-community
-    name: console-plugin
-  - image: quay.io/netobserv/network-observability-console-plugin-pf4:v1.8.2-community
-    name: console-plugin-compat
+    name: ""
   version: 1.9.1-community
   webhookdefinitions:
   - admissionReviewVersions:

bundle/manifests/netobserv-operator.clusterserviceversion.yaml

@@ -2557,7 +2557,9 @@ spec:
                           type: object
                       type: object
                     autoscaler:
-                      description: '`autoscaler` spec of a horizontal pod autoscaler to set up for the plugin Deployment.'
+                      description: |-
+                        `autoscaler` [deprecated (*)] spec of a horizontal pod autoscaler to set up for the plugin Deployment.
+                        Deprecation notice: managed autoscaler will be removed in a future version. You may configure instead an autoscaler of your choice, and set `spec.consolePlugin.unmanagedReplicas` to `true`.
                       properties:
                         maxReplicas:
                           default: 3
@@ -3006,16 +3008,21 @@ spec:
                             More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
                           type: object
                       type: object
+                    unmanagedReplicas:
+                      description: If `unmanagedReplicas` is `true`, the operator will not reconcile `replicas`. This is useful when using a pod autoscaler.
+                      type: boolean
                   type: object
                 deploymentModel:
                   default: Direct
                   description: |-
                     `deploymentModel` defines the desired type of deployment for flow processing. Possible values are:<br>
-                    - `Direct` (default) to make the flow processor listen directly from the agents.<br>
+                    - `Direct` (default) to make the flow processor listen directly from the agents using the host network.<br>
+                    - `Service` to make the flow processor listen as a Kubernetes Service.<br>
                     - `Kafka` to make flows sent to a Kafka pipeline before consumption by the processor.<br>
                     Kafka can provide better scalability, resiliency, and high availability (for more details, see https://www.redhat.com/en/topics/integration/what-is-apache-kafka).
                   enum:
                     - Direct
+                    - Service
                     - Kafka
                   type: string
                 exporters:
@@ -4946,6 +4953,13 @@ spec:
                       default: ""
                       description: '`clusterName` is the name of the cluster to appear in the flows data. This is useful in a multi-cluster context. When using OpenShift, leave empty to make it automatically determined.'
                       type: string
+                    consumerReplicas:
+                      description: |-
+                        `consumerReplicas` defines the number of replicas (pods) to start for `flowlogs-pipeline`.
+                        This setting is ignored when `spec.deploymentModel` is `Direct` or when `spec.processor.unmanagedReplicas` is `true`.
+                      format: int32
+                      minimum: 0
+                      type: integer
                     deduper:
                       description: '`deduper` allows you to sample or drop flows identified as duplicates, in order to save on resource usage.'
                       properties:
@@ -5004,8 +5018,9 @@ spec:
                       type: string
                     kafkaConsumerAutoscaler:
                       description: |-
-                        `kafkaConsumerAutoscaler` is the spec of a horizontal pod autoscaler to set up for `flowlogs-pipeline-transformer`, which consumes Kafka messages.
+                        `kafkaConsumerAutoscaler` [deprecated (*)] is the spec of a horizontal pod autoscaler to set up for `flowlogs-pipeline-transformer`, which consumes Kafka messages.
                         This setting is ignored when Kafka is disabled.
+                        Deprecation notice: managed autoscaler will be removed in a future version. You may configure instead an autoscaler of your choice, and set `spec.processor.unmanagedReplicas` to `true`.
                       properties:
                         maxReplicas:
                           default: 3
@@ -5312,8 +5327,9 @@ spec:
                     kafkaConsumerReplicas:
                       default: 3
                       description: |-
-                        `kafkaConsumerReplicas` defines the number of replicas (pods) to start for `flowlogs-pipeline-transformer`, which consumes Kafka messages.
+                        `kafkaConsumerReplicas` [deprecated (*)] defines the number of replicas (pods) to start for `flowlogs-pipeline-transformer`, which consumes Kafka messages.
                         This setting is ignored when Kafka is disabled.
+                        Deprecation notice: use `spec.processor.consumerReplicas` instead.
                       format: int32
                       minimum: 0
                       type: integer
@@ -5597,6 +5613,9 @@ spec:
                             external traffic: flows that are not labeled for those subnets are external to the cluster. Enabled by default on OpenShift.
                           type: boolean
                       type: object
+                    unmanagedReplicas:
+                      description: If `unmanagedReplicas` is `true`, the operator will not reconcile `consumerReplicas`. This is useful when using a pod autoscaler.
+                      type: boolean
                   type: object
                 prometheus:
                   description: '`prometheus` defines Prometheus settings, such as querier configuration used to fetch metrics from the Console plugin.'