diff --git a/charts/spark-operator-chart/README.md b/charts/spark-operator-chart/README.md index cf0420edf..d94bc1720 100644 --- a/charts/spark-operator-chart/README.md +++ b/charts/spark-operator-chart/README.md @@ -86,6 +86,7 @@ See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall) for command docum | controller.replicas | int | `1` | Number of replicas of controller. | | controller.workers | int | `10` | Reconcile concurrency, higher values might increase memory usage. | | controller.logLevel | string | `"info"` | Configure the verbosity of logging, can be one of `debug`, `info`, `error`. | +| controller.maxTrackedExecutorPerApp | int | `1000` | Specifies the maximum number of Executor pods that can be tracked by the controller per SparkApplication. | | controller.uiService.enable | bool | `true` | Specifies whether to create service for Spark web UI. | | controller.uiIngress.enable | bool | `false` | Specifies whether to create ingress for Spark web UI. `controller.uiService.enable` must be `true` to enable ingress. | | controller.uiIngress.urlFormat | string | `""` | Ingress URL format. Required if `controller.uiIngress.enable` is true. | diff --git a/charts/spark-operator-chart/templates/controller/deployment.yaml b/charts/spark-operator-chart/templates/controller/deployment.yaml index 4f9251f5a..aae6e275b 100644 --- a/charts/spark-operator-chart/templates/controller/deployment.yaml +++ b/charts/spark-operator-chart/templates/controller/deployment.yaml @@ -94,6 +94,9 @@ spec: {{- if .Values.controller.pprof.enable }} - --pprof-bind-address=:{{ .Values.controller.pprof.port }} {{- end }} + {{- if .Values.controller.maxTrackedExecutorPerApp }} + - --max-tracked-executor-per-app={{ .Values.controller.maxTrackedExecutorPerApp }} + {{- end }} {{- if or .Values.prometheus.metrics.enable .Values.controller.pprof.enable }} ports: {{- if .Values.controller.pprof.enable }} diff --git a/charts/spark-operator-chart/tests/controller/deployment_test.yaml b/charts/spark-operator-chart/tests/controller/deployment_test.yaml index 57b557757..79faae8f0 100644 --- a/charts/spark-operator-chart/tests/controller/deployment_test.yaml +++ b/charts/spark-operator-chart/tests/controller/deployment_test.yaml @@ -592,4 +592,13 @@ tests: content: name: pprof-test containerPort: 12345 - count: 1 \ No newline at end of file + count: 1 + + - it: Should contain `--max-tracked-executor-per-app` arg if `controller.maxTrackedExecutorPerApp` is set + set: + controller: + maxTrackedExecutorPerApp: 123 + asserts: + - contains: + path: spec.template.spec.containers[?(@.name=="spark-operator-controller")].args + content: --max-tracked-executor-per-app=123 diff --git a/charts/spark-operator-chart/values.yaml b/charts/spark-operator-chart/values.yaml index 94100dfb2..1c7b585b3 100644 --- a/charts/spark-operator-chart/values.yaml +++ b/charts/spark-operator-chart/values.yaml @@ -51,6 +51,9 @@ controller: # -- Configure the verbosity of logging, can be one of `debug`, `info`, `error`. logLevel: info + # -- Specifies the maximum number of Executor pods that can be tracked by the controller per SparkApplication. + maxTrackedExecutorPerApp: 1000 + uiService: # -- Specifies whether to create service for Spark web UI. enable: true