[v17] Make teleport container do a graceful shutdown (#48942)

* Container image should try a graceful shutdown by default

* Make grace termination period configurable in teleport-kube-agent

* re-render docs

Author: hugoShaka

build.assets/charts/Dockerfile

@@ -1,3 +1,8 @@
+# DEPRECATED: Images from this dockerfile are not published for v15 and above
+# https://goteleport.com/docs/changelog/#heavy-container-images-are-discontinued
+# Teleport images are built from Dockerfile-distroless
+# TODO(hugoShaka): cleanup the Makefile docker/image targets and remove this file.
+
 # Stage to build the image, without FIPS entrypoint argument
 FROM ubuntu:20.04 AS teleport
 
@@ -67,6 +72,10 @@ RUN --mount=target=/ctx \
 # Used to track whether a Teleport agent was installed using this method.
 ENV TELEPORT_INSTALL_METHOD_DOCKERFILE=true
 
+# Attempt a graceful shutdown by default
+# See https://goteleport.com/docs/reference/signals/ for signal reference.
+STOPSIGNAL SIGQUIT
+
 # By setting this entry point, we expose make target as command.
 ENTRYPOINT ["/usr/bin/dumb-init", "teleport", "start", "-c", "/etc/teleport/teleport.yaml"]
 

build.assets/charts/Dockerfile-distroless

@@ -30,4 +30,7 @@ FROM $BASE_IMAGE
 COPY --from=teleport /opt/staging /
 COPY --from=staging /opt/staging/root /
 COPY --from=staging /opt/staging/status /var/lib/dpkg/status.d
+# Attempt a graceful shutdown by default
+# See https://goteleport.com/docs/reference/signals/ for signal reference.
+STOPSIGNAL SIGQUIT
 ENTRYPOINT ["/usr/bin/dumb-init", "/usr/local/bin/teleport", "start", "-c", "/etc/teleport/teleport.yaml"]

build.assets/charts/Dockerfile-distroless-fips

@@ -30,4 +30,7 @@ FROM $BASE_IMAGE
 COPY --from=teleport /opt/staging /
 COPY --from=staging /opt/staging/root /
 COPY --from=staging /opt/staging/status /var/lib/dpkg/status.d
+# Attempt a graceful shutdown by default
+# See https://goteleport.com/docs/reference/signals/ for signal reference.
+STOPSIGNAL SIGQUIT
 ENTRYPOINT ["/usr/bin/dumb-init", "/usr/local/bin/teleport", "start", "-c", "/etc/teleport/teleport.yaml", "--fips"]

docs/pages/includes/helm-reference/zz_generated.teleport-kube-agent.mdx

@@ -631,6 +631,21 @@ teleportConfig:
         "*":"*"
 ```
 
+## `terminationGracePeriodSeconds`
+
+| Type | Default |
+|------|---------|
+| `integer` | `30` |
+
+`terminationGracePeriodSeconds` is the time the pod has to do a graceful shutdown.
+If teleport has not existed after this delay, the process gets killed.
+Teleport will wait until every connection backed by the agent is over before exiting.
+If you want to reduce the disruption of rolling out agents at the price of a slower rollout, you can increase this
+value to an hour.
+
+See the [Kubernetes Pod Lifecycle docs](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-termination)
+for more details.
+
 ## `tls`
 
 `tls` contains settings for mounting your own TLS material in the agent pod.

examples/chart/teleport-kube-agent/templates/statefulset.yaml

@@ -49,6 +49,9 @@ spec:
       {{- if .Values.podSecurityContext }}
       securityContext: {{- toYaml .Values.podSecurityContext | nindent 8}}
       {{- end }}
+      {{- if .Values.terminationGracePeriodSeconds }}
+      terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
+      {{- end }}
       {{- if or .Values.affinity (gt (int $replicaCount) 1) }}
       affinity:
         {{- if .Values.affinity }}

examples/chart/teleport-kube-agent/tests/__snapshot__/statefulset_test.yaml.snap

@@ -63,6 +63,7 @@ sets Pod annotations when specified:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -135,6 +136,7 @@ sets Pod labels when specified:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -231,6 +233,7 @@ sets StatefulSet labels when specified:
           securityContext:
             fsGroup: 9807
           serviceAccountName: RELEASE-NAME
+          terminationGracePeriodSeconds: 30
           volumes:
           - configMap:
               name: RELEASE-NAME
@@ -335,6 +338,7 @@ should add insecureSkipProxyTLSVerify to args when set in values:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -407,6 +411,7 @@ should add volumeClaimTemplate for data volume when using StatefulSet and action
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -499,6 +504,7 @@ should add volumeClaimTemplate for data volume when using StatefulSet and is Fre
           securityContext:
             fsGroup: 9807
           serviceAccountName: RELEASE-NAME
+          terminationGracePeriodSeconds: 30
           volumes:
           - configMap:
               name: RELEASE-NAME
@@ -581,6 +587,7 @@ should add volumeMount for data volume when using StatefulSet:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -653,6 +660,7 @@ should expose diag port:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -725,6 +733,7 @@ should generate Statefulset when storage is disabled and mode is a Upgrade:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -811,6 +820,7 @@ should have multiple replicas when replicaCount is set (using .replicaCount, dep
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -895,6 +905,7 @@ should have multiple replicas when replicaCount is set (using highAvailability.r
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -967,6 +978,7 @@ should have one replica when replicaCount is not set:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -1039,6 +1051,7 @@ should install Statefulset when storage is disabled and mode is a Fresh Install:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -1115,6 +1128,7 @@ should mount extraVolumes and extraVolumeMounts:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -1193,6 +1207,7 @@ should mount jamfCredentialsSecret if it already exists and when role is jamf:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -1273,6 +1288,7 @@ should mount jamfCredentialsSecret.name when role is jamf:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -1355,6 +1371,7 @@ should mount tls.existingCASecretName and set environment when set in values:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -1439,6 +1456,7 @@ should mount tls.existingCASecretName and set extra environment when set in valu
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -1516,6 +1534,7 @@ should not add emptyDir for data when using StatefulSet:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -1626,6 +1645,7 @@ should provision initContainer correctly when set in values:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -1718,6 +1738,7 @@ should set affinity when set in values:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -1790,6 +1811,7 @@ should set default serviceAccountName when not set in values:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -1875,6 +1897,7 @@ should set environment when extraEnv set in values:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -1947,6 +1970,7 @@ should set image and tag correctly:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -2019,6 +2043,7 @@ should set imagePullPolicy when set in values:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -2093,6 +2118,7 @@ should set nodeSelector if set in values:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -2177,6 +2203,7 @@ should set preferred affinity when more than one replica is used:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -2249,6 +2276,7 @@ should set probeTimeoutSeconds when set in values:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -2331,6 +2359,7 @@ should set required affinity when highAvailability.requireAntiAffinity is set:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -2410,6 +2439,7 @@ should set resources when set in values:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -2482,6 +2512,7 @@ should set serviceAccountName when set in values:
     securityContext:
       fsGroup: 9807
     serviceAccountName: teleport-kube-agent-sa
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -2554,6 +2585,7 @@ should set storage.requests when set in values and action is an Upgrade:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -2626,6 +2658,7 @@ should set storage.storageClassName when set in values and action is an Upgrade:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     volumes:
     - configMap:
         name: RELEASE-NAME
@@ -2698,6 +2731,7 @@ should set tolerations when set in values:
     securityContext:
       fsGroup: 9807
     serviceAccountName: RELEASE-NAME
+    terminationGracePeriodSeconds: 30
     tolerations:
     - effect: NoExecute
       key: dedicated

examples/chart/teleport-kube-agent/tests/statefulset_test.yaml

@@ -814,3 +814,13 @@ tests:
               hostnames:
                 - "foo.remote"
                 - "bar.remote"
+  - it: should set the terminationGracePeriodSeconds when specified
+    template: statefulset.yaml
+    values:
+      - ../.lint/stateful.yaml
+    set:
+      terminationGracePeriodSeconds: 3600
+    asserts:
+      - equal:
+          path: spec.template.spec.terminationGracePeriodSeconds
+          value: 3600

examples/chart/teleport-kube-agent/values.schema.json

@@ -252,6 +252,11 @@
             "type": "object",
             "default": {}
         },
+        "terminationGracePeriodSeconds": {
+            "$id": "#/properties/terminationGracePeriodSeconds",
+            "type": "integer",
+            "default": 30
+        },
         "tls": {
             "$id": "#/properties/tls",
             "type": "object",

examples/chart/teleport-kube-agent/values.yaml

@@ -538,6 +538,16 @@ insecureSkipProxyTLSVerify: false
 # ```
 teleportConfig: {}
 
+# terminationGracePeriodSeconds(integer) -- is the time the pod has to do a graceful shutdown.
+# If teleport has not existed after this delay, the process gets killed.
+# Teleport will wait until every connection backed by the agent is over before exiting.
+# If you want to reduce the disruption of rolling out agents at the price of a slower rollout, you can increase this
+# value to an hour.
+#
+# See the [Kubernetes Pod Lifecycle docs](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-termination)
+# for more details.
+terminationGracePeriodSeconds: 30
+
 # tls -- contains settings for mounting your own TLS material in the agent pod.
 # The agent does not expose a TLS server, so this is only used to trust CAs.
 tls: