From fe86e67febff0fc9edf0262c088f23d9520727bd Mon Sep 17 00:00:00 2001 From: zehweh Date: Thu, 1 Aug 2019 08:26:34 +0000 Subject: [PATCH 1/2] Add resource limits/requests to helm charts --- .../templates/datanode-daemonset.yaml | 2 + .../templates/journalnode-statefulset.yaml | 2 + charts/hdfs-k8s/values.yaml | 41 +++++++++++++++++++ .../templates/namenode-statefulset.yaml | 2 + .../templates/namenode-statefulset.yaml | 2 + 5 files changed, 49 insertions(+) diff --git a/charts/hdfs-datanode-k8s/templates/datanode-daemonset.yaml b/charts/hdfs-datanode-k8s/templates/datanode-daemonset.yaml index 09445ed..73a7ecd 100644 --- a/charts/hdfs-datanode-k8s/templates/datanode-daemonset.yaml +++ b/charts/hdfs-datanode-k8s/templates/datanode-daemonset.yaml @@ -108,6 +108,8 @@ spec: periodSeconds: 30 securityContext: privileged: true + resources: +{{ toYaml .Values.resources | indent 12 }} volumeMounts: - name: dn-scripts mountPath: /dn-scripts diff --git a/charts/hdfs-journalnode-k8s/templates/journalnode-statefulset.yaml b/charts/hdfs-journalnode-k8s/templates/journalnode-statefulset.yaml index 22a4a2b..3398093 100644 --- a/charts/hdfs-journalnode-k8s/templates/journalnode-statefulset.yaml +++ b/charts/hdfs-journalnode-k8s/templates/journalnode-statefulset.yaml @@ -98,6 +98,8 @@ spec: name: jn - containerPort: 8480 name: http + resources: +{{ toYaml .Values.resources | indent 12 }} volumeMounts: # Mount a subpath of the volume so that the journal subdir would be # a brand new empty dir. This way, we won't get affected by diff --git a/charts/hdfs-k8s/values.yaml b/charts/hdfs-k8s/values.yaml index 77ca3fe..4afee94 100644 --- a/charts/hdfs-k8s/values.yaml +++ b/charts/hdfs-k8s/values.yaml @@ -48,6 +48,16 @@ hdfs-journalnode-k8s: accessMode: ReadWriteOnce size: 20Gi + resources: {} + ## Optionally specify how much CPU and memory (RAM) each container needs. + ## + # limits: + # cpu: 1 + # memory: 512Mi + # requests: + # cpu: 100m + # memory: 128Mi + ## Node labels and tolerations for pod assignment nodeSelector: {} tolerations: [] @@ -89,6 +99,16 @@ hdfs-namenode-k8s: accessMode: ReadWriteOnce size: 100Gi + + resources: {} + ## Optionally specify how much CPU and memory (RAM) each container needs. + ## + # limits: + # cpu: 1 + # memory: 512Mi + # requests: + # cpu: 100m + # memory: 128Mi ## Whether or not to use hostNetwork in namenode pods. Disabling this will break ## data locality as namenode will see pod virtual IPs and fails to equate them with @@ -110,6 +130,16 @@ hdfs-simple-namenode-k8s: ## volume. nameNodeHostPath: /hdfs-name + resources: {} + ## Optionally specify how much CPU and memory (RAM) each container needs. + ## + # limits: + # cpu: 1 + # memory: 512Mi + # requests: + # cpu: 100m + # memory: 128Mi + ## Node labels and tolerations for pod assignment nodeSelector: {} tolerations: [] @@ -119,6 +149,17 @@ hdfs-simple-namenode-k8s: ## hdfs-datanode-k8s: ## ------------------------------------------------------------------------------ hdfs-datanode-k8s: + resources: {} + ## Optionally specify how much CPU and memory (RAM) each container needs. + ## + # limits: + # cpu: 1 + # memory: 512Mi + # requests: + # cpu: 100m + # memory: 128Mi + + ## Node labels and tolerations for pod assignment nodeSelector: {} tolerations: [] diff --git a/charts/hdfs-namenode-k8s/templates/namenode-statefulset.yaml b/charts/hdfs-namenode-k8s/templates/namenode-statefulset.yaml index 44e8fc6..df046b3 100644 --- a/charts/hdfs-namenode-k8s/templates/namenode-statefulset.yaml +++ b/charts/hdfs-namenode-k8s/templates/namenode-statefulset.yaml @@ -197,6 +197,8 @@ spec: name: fs - containerPort: 50070 name: http + resources: +{{ toYaml .Values.resources | indent 12 }} volumeMounts: - name: nn-scripts mountPath: /nn-scripts diff --git a/charts/hdfs-simple-namenode-k8s/templates/namenode-statefulset.yaml b/charts/hdfs-simple-namenode-k8s/templates/namenode-statefulset.yaml index ab92efa..fa36757 100644 --- a/charts/hdfs-simple-namenode-k8s/templates/namenode-statefulset.yaml +++ b/charts/hdfs-simple-namenode-k8s/templates/namenode-statefulset.yaml @@ -66,6 +66,8 @@ spec: ports: - containerPort: 8020 name: fs + resources: +{{ toYaml .Values.resources | indent 12 }} volumeMounts: - name: hdfs-name mountPath: /hadoop/dfs/name From 14dd80c0c12676cfea2b2a0710ab2736f0105686 Mon Sep 17 00:00:00 2001 From: zehweh Date: Tue, 27 Aug 2019 12:04:15 +0000 Subject: [PATCH 2/2] Add the possibility to set env variables --- .../templates/datanode-daemonset.yaml | 3 +++ .../templates/journalnode-statefulset.yaml | 3 +++ charts/hdfs-k8s/values.yaml | 14 +++++++++++++- .../templates/namenode-statefulset.yaml | 3 +++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/charts/hdfs-datanode-k8s/templates/datanode-daemonset.yaml b/charts/hdfs-datanode-k8s/templates/datanode-daemonset.yaml index 73a7ecd..9238dfd 100644 --- a/charts/hdfs-datanode-k8s/templates/datanode-daemonset.yaml +++ b/charts/hdfs-datanode-k8s/templates/datanode-daemonset.yaml @@ -94,6 +94,9 @@ spec: - name: JSVC_HOME value: /jsvc-home {{- end }} +{{- if .Values.env }} +{{ toYaml .Values.env | indent 12 }} +{{- end }} livenessProbe: exec: command: diff --git a/charts/hdfs-journalnode-k8s/templates/journalnode-statefulset.yaml b/charts/hdfs-journalnode-k8s/templates/journalnode-statefulset.yaml index 3398093..f95afd0 100644 --- a/charts/hdfs-journalnode-k8s/templates/journalnode-statefulset.yaml +++ b/charts/hdfs-journalnode-k8s/templates/journalnode-statefulset.yaml @@ -91,6 +91,9 @@ spec: env: - name: HADOOP_CUSTOM_CONF_DIR value: /etc/hadoop-custom-conf +{{- if .Values.env }} +{{ toYaml .Values.env | indent 12 }} +{{- end }} command: ["/entrypoint.sh"] args: ["/opt/hadoop-2.7.2/bin/hdfs", "--config", "/etc/hadoop", "journalnode"] ports: diff --git a/charts/hdfs-k8s/values.yaml b/charts/hdfs-k8s/values.yaml index 4afee94..8ebb8eb 100644 --- a/charts/hdfs-k8s/values.yaml +++ b/charts/hdfs-k8s/values.yaml @@ -4,7 +4,7 @@ zookeeper: ## Configure Zookeeper resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ - resources: ~ + resources: {} ## The JVM heap size to allocate to Zookeeper env: @@ -58,6 +58,10 @@ hdfs-journalnode-k8s: # cpu: 100m # memory: 128Mi + env: [] + ## Optional environment variables + + ## Node labels and tolerations for pod assignment nodeSelector: {} tolerations: [] @@ -110,6 +114,9 @@ hdfs-namenode-k8s: # cpu: 100m # memory: 128Mi + env: [] + ## Optional environment variables + ## Whether or not to use hostNetwork in namenode pods. Disabling this will break ## data locality as namenode will see pod virtual IPs and fails to equate them with ## cluster node physical IPs associated with data nodes. @@ -140,6 +147,9 @@ hdfs-simple-namenode-k8s: # cpu: 100m # memory: 128Mi + env: [] + ## Optional environment variables + ## Node labels and tolerations for pod assignment nodeSelector: {} tolerations: [] @@ -159,6 +169,8 @@ hdfs-datanode-k8s: # cpu: 100m # memory: 128Mi + env: [] + ## Optional environment variables ## Node labels and tolerations for pod assignment nodeSelector: {} diff --git a/charts/hdfs-namenode-k8s/templates/namenode-statefulset.yaml b/charts/hdfs-namenode-k8s/templates/namenode-statefulset.yaml index df046b3..d9c2134 100644 --- a/charts/hdfs-namenode-k8s/templates/namenode-statefulset.yaml +++ b/charts/hdfs-namenode-k8s/templates/namenode-statefulset.yaml @@ -188,6 +188,9 @@ spec: value: {{ template "namenode-pod-0" . }} - name: NAMENODE_POD_1 value: {{ template "namenode-pod-1" . }} +{{- if .Values.env }} +{{ toYaml .Values.env | indent 12 }} +{{- end }} command: ['/bin/sh', '-c'] # The start script is provided by a config map. args: