Add option for configuring HPA with custom/external metrics

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## master / unreleased
 
+* [ENHANCEMENT] Add option to autoscale on multiple metrics and custom metrics metrics #467
 * [DEPENDENCY] Update quay.io/cortexproject/cortex Docker tag to v1.15.0 #453
 * [DEPENDENCY] Update quay.io/cortexproject/cortex Docker tag to v1.15.1 #456
 * [BUGFIX] Correctly interpret .Capabilities.KubeVersion when it looks like a prerelease #457

README.md

@@ -265,6 +265,7 @@ Kubernetes: `^1.19.0-0`
 | distributor.​affinity.​podAntiAffinity.​preferredDuringSchedulingIgnoredDuringExecution[0].​weight | int | `100` |  |
 | distributor.​annotations | object | `{}` |  |
 | distributor.​autoscaling.​behavior | object | `{}` | Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior |
+| distributor.​autoscaling.​customMetrics | list | `[]` | Optional custom and external metrics for the distributor pods to scale on In order to use this option , define a list of of specific following either: - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects  For example:  - type: External   external:     metric:       name: queue_messages_ready       selector:         matchLabels:           queue: "worker_tasks"     target:       type: AverageValue       averageValue: 30 |
 | distributor.​autoscaling.​enabled | bool | `false` | Creates a HorizontalPodAutoscaler for the distributor pods. |
 | distributor.​autoscaling.​maxReplicas | int | `30` |  |
 | distributor.​autoscaling.​minReplicas | int | `2` |  |
@@ -326,6 +327,7 @@ Kubernetes: `^1.19.0-0`
 | ingester.​autoscaling.​behavior.​scaleDown.​policies | list | `[{"periodSeconds":1800,"type":"Pods","value":1}]` | see https://cortexmetrics.io/docs/guides/ingesters-scaling-up-and-down/#scaling-down for scaledown details |
 | ingester.​autoscaling.​behavior.​scaleDown.​stabilizationWindowSeconds | int | `3600` | uses metrics from the past 1h to make scaleDown decisions |
 | ingester.​autoscaling.​behavior.​scaleUp.​policies | list | `[{"periodSeconds":1800,"type":"Pods","value":1}]` | This default scaleup policy allows adding 1 pod every 30 minutes. Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior |
+| ingester.​autoscaling.​customMetrics | list | `[]` | Optional custom and external metrics for the ingester pods to scale on In order to use this option , define a list of of specific following either: - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects  For example:  - type: External   external:     metric:       name: queue_messages_ready       selector:         matchLabels:           queue: "worker_tasks"     target:       type: AverageValue       averageValue: 30 |
 | ingester.​autoscaling.​enabled | bool | `false` |  |
 | ingester.​autoscaling.​maxReplicas | int | `30` |  |
 | ingester.​autoscaling.​minReplicas | int | `3` |  |
@@ -424,6 +426,7 @@ Kubernetes: `^1.19.0-0`
 | nginx.​affinity | object | `{}` |  |
 | nginx.​annotations | object | `{}` |  |
 | nginx.​autoscaling.​behavior | object | `{}` | Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior |
+| nginx.​autoscaling.​customMetrics | list | `[]` | Optional custom and external metrics for the nginx pods to scale on In order to use this option , define a list of of specific following either: - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects  For example:  - type: External   external:     metric:       name: queue_messages_ready       selector:         matchLabels:           queue: "worker_tasks"     target:       type: AverageValue       averageValue: 30 |
 | nginx.​autoscaling.​enabled | bool | `false` | Creates a HorizontalPodAutoscaler for the nginx pods. |
 | nginx.​autoscaling.​maxReplicas | int | `30` |  |
 | nginx.​autoscaling.​minReplicas | int | `2` |  |
@@ -571,6 +574,7 @@ Kubernetes: `^1.19.0-0`
 | querier.​affinity.​podAntiAffinity.​preferredDuringSchedulingIgnoredDuringExecution[0].​weight | int | `100` |  |
 | querier.​annotations | object | `{}` |  |
 | querier.​autoscaling.​behavior | object | `{}` | Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior |
+| querier.​autoscaling.​customMetrics | list | `[]` | Optional custom and external metrics for the querier pods to scale on In order to use this option , define a list of of specific following either: - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects  For example:  - type: External   external:     metric:       name: queue_messages_ready       selector:         matchLabels:           queue: "worker_tasks"     target:       type: AverageValue       averageValue: 30 |
 | querier.​autoscaling.​enabled | bool | `false` | Creates a HorizontalPodAutoscaler for the querier pods. |
 | querier.​autoscaling.​maxReplicas | int | `30` |  |
 | querier.​autoscaling.​minReplicas | int | `2` |  |
@@ -713,6 +717,7 @@ Kubernetes: `^1.19.0-0`
 | ruler.​affinity | object | `{}` |  |
 | ruler.​annotations | object | `{}` |  |
 | ruler.​autoscaling.​behavior | object | `{}` | Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior |
+| ruler.​autoscaling.​customMetrics | list | `[]` | Optional custom and external metrics for the ruler pods to scale on In order to use this option , define a list of of specific following either: - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects  For example:  - type: External   external:     metric:       name: queue_messages_ready       selector:         matchLabels:           queue: "worker_tasks"     target:       type: AverageValue       averageValue: 30 |
 | ruler.​autoscaling.​enabled | bool | `false` | Creates a HorizontalPodAutoscaler for the ruler. |
 | ruler.​autoscaling.​maxReplicas | int | `30` |  |
 | ruler.​autoscaling.​minReplicas | int | `2` |  |
@@ -791,6 +796,7 @@ Kubernetes: `^1.19.0-0`
 | store_gateway.​affinity.​podAntiAffinity.​preferredDuringSchedulingIgnoredDuringExecution[0].​weight | int | `100` |  |
 | store_gateway.​annotations | object | `{}` |  |
 | store_gateway.​autoscaling.​behavior | object | `{}` | Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior |
+| store_gateway.​autoscaling.​customMetrics | list | `[]` | Optional custom and external metrics for the store gateway pods to scale on In order to use this option , define a list of of specific following either: - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects  For example:  - type: External   external:     metric:       name: queue_messages_ready       selector:         matchLabels:           queue: "worker_tasks"     target:       type: AverageValue       averageValue: 30 |
 | store_gateway.​autoscaling.​enabled | bool | `false` |  |
 | store_gateway.​autoscaling.​maxReplicas | int | `30` |  |
 | store_gateway.​autoscaling.​minReplicas | int | `3` |  |

templates/distributor/distributor-hpa.yaml

@@ -31,6 +31,9 @@ spec:
           type: Utilization
           averageUtilization: {{ . }}
   {{- end }}
+  {{- with .customMetrics }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   {{- with .behavior }}
   behavior:
     {{- toYaml . | nindent 4 }}

templates/ingester/ingester-hpa.yaml

@@ -21,6 +21,9 @@ spec:
         target:
           type: Utilization
           averageUtilization: {{ .targetMemoryUtilizationPercentage }}
+  {{- with .customMetrics }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   {{- with .behavior }}
   behavior:
     {{- toYaml . | nindent 4 }}

templates/nginx/nginx-hpa.yaml

@@ -31,6 +31,9 @@ spec:
           type: Utilization
           averageUtilization: {{ . }}
   {{- end }}
+  {{- with .customMetrics }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   {{- with .behavior }}
   behavior:
     {{- toYaml . | nindent 4 }}

templates/querier/querier-hpa.yaml

@@ -31,6 +31,9 @@ spec:
           type: Utilization
           averageUtilization: {{ . }}
   {{- end }}
+  {{- with .customMetrics }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   {{- with .behavior }}
   behavior:
     {{- toYaml . | nindent 4 }}

templates/ruler/ruler-hpa.yaml

@@ -31,6 +31,9 @@ spec:
           type: Utilization
           averageUtilization: {{ . }}
   {{- end }}
+  {{- with .customMetrics }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   {{- with .behavior }}
   behavior:
     {{- toYaml . | nindent 4 }}

templates/store-gateway/store-gateway-hpa.yaml

@@ -21,6 +21,9 @@ spec:
         target:
           type: Utilization
           averageUtilization: {{ .targetMemoryUtilizationPercentage }}
+  {{- with .customMetrics }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   {{- with .behavior }}
   behavior:
     {{- toYaml . | nindent 4 }}

values.yaml

@@ -372,6 +372,24 @@ distributor:
     targetMemoryUtilizationPercentage: 0  # 80
     # -- Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior
     behavior: {}
+    # -- Optional custom and external metrics for the distributor pods to scale on
+    # In order to use this option , define a list of of specific following either:
+    # - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics
+    # - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects
+    #
+    # For example:
+    #
+    # - type: External
+    #   external:
+    #     metric:
+    #       name: queue_messages_ready
+    #       selector:
+    #         matchLabels:
+    #           queue: "worker_tasks"
+    #     target:
+    #       type: AverageValue
+    #       averageValue: 30
+    customMetrics: []
 
   persistentVolume:
     subPath:
@@ -496,6 +514,24 @@ ingester:
           - type: Pods
             value: 1
             periodSeconds: 1800
+    # -- Optional custom and external metrics for the ingester pods to scale on
+    # In order to use this option , define a list of of specific following either:
+    # - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics
+    # - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects
+    #
+    # For example:
+    #
+    # - type: External
+    #   external:
+    #     metric:
+    #       name: queue_messages_ready
+    #       selector:
+    #         matchLabels:
+    #           queue: "worker_tasks"
+    #     target:
+    #       type: AverageValue
+    #       averageValue: 30
+    customMetrics: []
 
   lifecycle:
     # -- The /shutdown preStop hook is recommended as part of the ingester
@@ -624,6 +660,24 @@ ruler:
     targetMemoryUtilizationPercentage: 80
     # -- Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior
     behavior: {}
+    # -- Optional custom and external metrics for the ruler pods to scale on
+    # In order to use this option , define a list of of specific following either:
+    # - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics
+    # - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects
+    #
+    # For example:
+    #
+    # - type: External
+    #   external:
+    #     metric:
+    #       name: queue_messages_ready
+    #       selector:
+    #         matchLabels:
+    #           queue: "worker_tasks"
+    #     target:
+    #       type: AverageValue
+    #       averageValue: 30
+    customMetrics: []
 
   validation:
     # -- Checks that the ruler is compatible with horizontal scaling,
@@ -773,6 +827,24 @@ querier:
     targetMemoryUtilizationPercentage: 0  # 80
     # -- Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior
     behavior: {}
+    # -- Optional custom and external metrics for the querier pods to scale on
+    # In order to use this option , define a list of of specific following either:
+    # - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics
+    # - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects
+    #
+    # For example:
+    #
+    # - type: External
+    #   external:
+    #     metric:
+    #       name: queue_messages_ready
+    #       selector:
+    #         matchLabels:
+    #           queue: "worker_tasks"
+    #     target:
+    #       type: AverageValue
+    #       averageValue: 30
+    customMetrics: []
 
   persistentVolume:
     subPath:
@@ -1293,6 +1365,24 @@ nginx:
     targetMemoryUtilizationPercentage: 0  # 80
     # -- Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior
     behavior: {}
+    # -- Optional custom and external metrics for the nginx pods to scale on
+    # In order to use this option , define a list of of specific following either:
+    # - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics
+    # - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects
+    #
+    # For example:
+    #
+    # - type: External
+    #   external:
+    #     metric:
+    #       name: queue_messages_ready
+    #       selector:
+    #         matchLabels:
+    #           queue: "worker_tasks"
+    #     target:
+    #       type: AverageValue
+    #       averageValue: 30
+    customMetrics: []
 
 store_gateway:
   enabled: true
@@ -1354,6 +1444,24 @@ store_gateway:
     targetMemoryUtilizationPercentage: 80
     # -- Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior
     behavior: {}
+    # -- Optional custom and external metrics for the store gateway pods to scale on
+    # In order to use this option , define a list of of specific following either:
+    # - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics
+    # - https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects
+    #
+    # For example:
+    #
+    # - type: External
+    #   external:
+    #     metric:
+    #       name: queue_messages_ready
+    #       selector:
+    #         matchLabels:
+    #           queue: "worker_tasks"
+    #     target:
+    #       type: AverageValue
+    #       averageValue: 30
+    customMetrics: []
 
   persistentVolume:
     # -- If true Store-gateway will create/use a Persistent Volume Claim