From b2964c9b89678a5817275e19a4b814e2d8b58bca Mon Sep 17 00:00:00 2001 From: Samuel Alfageme Date: Thu, 17 Sep 2020 15:03:33 +0200 Subject: [PATCH 1/4] useStatefulSet switch in revad for stateful workloads This is specially useful when running into sharded scenarios as e.g. different storage providers to map users with different rules. Ref. https://github.com/cs3org/reva/pull/1142 This commit contains a preliminary version, where the "real" uniqueness of each replica in the set (in terms of config) is yet to be implemented. First version will simply rely on the hostname as selector inside the 'configFiles' value for a user-provided toml file. --- revad/Chart.yaml | 2 +- revad/README.md | 6 +++++ revad/templates/deployment.yaml | 39 +++++++++++++++++++++++++++------ revad/templates/service.yaml | 4 ++++ revad/values.yaml | 2 ++ 5 files changed, 45 insertions(+), 8 deletions(-) diff --git a/revad/Chart.yaml b/revad/Chart.yaml index 1214a46..d5cbd6d 100644 --- a/revad/Chart.yaml +++ b/revad/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: revad description: The Reva daemon (revad) helm chart type: application -version: 1.3.1 +version: 1.4.0 appVersion: v1.7.0 kubeVersion: ">= 1.14.0" icon: https://reva.link/logo.svg diff --git a/revad/README.md b/revad/README.md index e39630f..dee56e7 100644 --- a/revad/README.md +++ b/revad/README.md @@ -41,6 +41,7 @@ The following configurations may be set. It is recommended to use `values.yaml` | `extraVolumeMounts` | Array of additional volume mounts. | `[]` | | `extraVolumes` | Array of additional volumes. | `[]` | | `emptyDir.sizeLimit` | `emptyDir` `sizeLimit` if a Persistent Volume is not used | `""` | +| `useStatefulSet` | If true, Revad will be deployed using a `StatefulSet` rather than the usual `Deployment`. | `false` | | `persistentVolume.enabled` | If true, Revad will create a Persistent Volume Claim. | `false` | | `persistentVolume.accessModes` | Revad data Persistent Volume access modes. | `[ReadWriteOnce]` | | `persistentVolume.annotations` | Revad data Persistent Volume annotations. | `{}` | @@ -71,3 +72,8 @@ The following configurations may be set. It is recommended to use `values.yaml` $ helm install custom-reva cs3org/revad \ --set-file configFiles.revad\\.toml=custom-config.toml ``` + +### Stateful deployment + +```console +``` diff --git a/revad/templates/deployment.yaml b/revad/templates/deployment.yaml index d412c7e..ebf6ff9 100644 --- a/revad/templates/deployment.yaml +++ b/revad/templates/deployment.yaml @@ -1,10 +1,13 @@ apiVersion: apps/v1 -kind: Deployment +kind: {{ if .Values.useStatefulSet }}{{ "StatefulSet" }}{{- else }}{{ "Deployment" }}{{- end }} metadata: name: {{ include "revad.fullname" . }} labels: {{- include "revad.labels" . | nindent 4 }} spec: + {{- if .Values.useStatefulSet }} + serviceName: {{ include "revad.fullname" . }} + {{- end }} replicas: {{ .Values.replicaCount }} selector: matchLabels: @@ -61,18 +64,40 @@ spec: - name: {{ include "revad.fullname" . }}-configfiles configMap: name: {{ template "revad.fullname" . }}-config + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} + {{- if .Values.persistentVolume.enabled }} + {{- if .Values.useStatefulSet }} + volumeClaimTemplates: + - metadata: + name: {{ include "revad.fullname" . }}-datadir + {{- if .Values.persistentVolume.annotations }} + annotations: + {{- toYaml .Values.persistentVolume.annotations | indent 10 }} + {{- end }} + spec: + accessModes: + {{- toYaml .Values.persistentVolume.accessModes | indent 10 }} + resources: + requests: + storage: "{{ .Values.persistentVolume.size }}" + {{- if (eq "-" .Values.persistentVolume.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.persistentVolume.storageClass }}" + {{- end }} + {{- else }} - name: {{ include "revad.fullname" . }}-datadir - {{- if .Values.persistentVolume.enabled }} persistentVolumeClaim: claimName: {{ if .Values.persistentVolume.existingClaim }}{{ .Values.persistentVolume.existingClaim }}{{- else }}{{ template "revad.fullname" . }}{{- end }} - {{- else }} + {{- end }} + {{- else }} + - name: {{ include "revad.fullname" . }}-datadir emptyDir: {{- if .Values.emptyDir.sizeLimit }} sizeLimit: {{ .Values.emptyDir.sizeLimit }} {{- else }} {} {{- end -}} - {{- end -}} - {{- if .Values.extraVolumes }} - {{ toYaml .Values.extraVolumes | nindent 8 }} - {{- end }} + {{- end }} diff --git a/revad/templates/service.yaml b/revad/templates/service.yaml index 5fba114..7616ea2 100644 --- a/revad/templates/service.yaml +++ b/revad/templates/service.yaml @@ -5,7 +5,11 @@ metadata: labels: {{- include "revad.labels" . | nindent 4 }} spec: + {{- if .Values.useStatefulSet }} + clusterIP: None + {{- else }} type: {{ .Values.service.type }} + {{- end }} ports: {{- if .Values.service.http }} - port: {{ .Values.service.http.port }} diff --git a/revad/values.yaml b/revad/values.yaml index 2fa54a6..51d771b 100644 --- a/revad/values.yaml +++ b/revad/values.yaml @@ -18,6 +18,8 @@ extraVolumes: [] emptyDir: sizeLimit: "" +useStatefulSet: false + persistentVolume: ## If true, revad will create/use a Persistent Volume Claim ## If false, use emptyDir From 8807458729b72a579e3d3fc430db207b8ec22c4f Mon Sep 17 00:00:00 2001 From: Samuel Alfageme Date: Thu, 17 Sep 2020 16:55:58 +0200 Subject: [PATCH 2/4] Use '.toml' as revad config --- revad/templates/deployment.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/revad/templates/deployment.yaml b/revad/templates/deployment.yaml index ebf6ff9..2a0a111 100644 --- a/revad/templates/deployment.yaml +++ b/revad/templates/deployment.yaml @@ -34,7 +34,11 @@ spec: - /go/bin/revad args: - "-c" + {{- if .Values.useStatefulSet }} + - "/etc/revad/${HOSTNAME}.toml" + {{- else }} - "/etc/revad/revad.toml" + {{- end }} - "-p" - "/var/run/revad.pid" volumeMounts: From 0c322796e52d252d295ce98e7c2de6ee9a5eac2e Mon Sep 17 00:00:00 2001 From: Samuel Alfageme Date: Fri, 18 Sep 2020 09:48:08 +0200 Subject: [PATCH 3/4] Fetch the pod's HOSTNAME via downward API and pass it as env.var ... when the pod is deployed as part of an StatefulSet, so it can determine its identity on startup and pick the right config file from the '*-configfiles' mount. --- revad/templates/deployment.yaml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/revad/templates/deployment.yaml b/revad/templates/deployment.yaml index 2a0a111..16d7003 100644 --- a/revad/templates/deployment.yaml +++ b/revad/templates/deployment.yaml @@ -35,7 +35,7 @@ spec: args: - "-c" {{- if .Values.useStatefulSet }} - - "/etc/revad/${HOSTNAME}.toml" + - "/etc/revad/$(HOSTNAME).toml" {{- else }} - "/etc/revad/revad.toml" {{- end }} @@ -50,8 +50,14 @@ spec: {{- if .Values.extraVolumeMounts }} {{ toYaml .Values.extraVolumeMounts | nindent 12 }} {{- end }} - {{- if or .Values.env .Values.extraEnv }} + {{- if or .Values.useStatefulSet (or .Values.env .Values.extraEnv) }} env: + {{- if .Values.useStatefulSet }} + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + {{- end }} {{- range $name, $value := .Values.env }} - name: "{{ $name }}" value: "{{ $value }}" @@ -62,7 +68,7 @@ spec: {{- end }} {{- if .Values.envFrom }} envFrom: - {{ toYaml .Values.envFrom | indent 12 }} + {{- toYaml .Values.envFrom | nindent 12 }} {{- end }} volumes: - name: {{ include "revad.fullname" . }}-configfiles @@ -78,13 +84,13 @@ spec: name: {{ include "revad.fullname" . }}-datadir {{- if .Values.persistentVolume.annotations }} annotations: - {{- toYaml .Values.persistentVolume.annotations | indent 10 }} + {{- toYaml .Values.persistentVolume.annotations | nindent 10 }} {{- end }} spec: accessModes: - {{- toYaml .Values.persistentVolume.accessModes | indent 10 }} + {{- toYaml .Values.persistentVolume.accessModes | nindent 10 }} resources: - requests: + requests: storage: "{{ .Values.persistentVolume.size }}" {{- if (eq "-" .Values.persistentVolume.storageClass) }} storageClassName: "" From a02f7c27c574429c24a484a0cde588698a76eaca Mon Sep 17 00:00:00 2001 From: Samuel Alfageme Sainz Date: Tue, 11 May 2021 15:49:32 +0200 Subject: [PATCH 4/4] revad: ci, test StatefulSet and Deployment configs on CI --- revad/.helmignore | 2 ++ revad/ci/statefulset-values.yaml | 2 ++ revad/ci/values.yaml | 0 3 files changed, 4 insertions(+) create mode 100644 revad/ci/statefulset-values.yaml create mode 100644 revad/ci/values.yaml diff --git a/revad/.helmignore b/revad/.helmignore index 0e8a0eb..05e36ed 100644 --- a/revad/.helmignore +++ b/revad/.helmignore @@ -21,3 +21,5 @@ .idea/ *.tmproj .vscode/ +# chart-testing-action configurations +ci/ diff --git a/revad/ci/statefulset-values.yaml b/revad/ci/statefulset-values.yaml new file mode 100644 index 0000000..83cd478 --- /dev/null +++ b/revad/ci/statefulset-values.yaml @@ -0,0 +1,2 @@ +useStatefulSet: true +replicaCount: 3 diff --git a/revad/ci/values.yaml b/revad/ci/values.yaml new file mode 100644 index 0000000..e69de29