Add a certificate rotation command

Makefile

@@ -101,7 +101,8 @@ run-e2e-tests:
 	KUBECONFIG=${kubeconfig} PLATFORM=${platform} go test -mod=$(MOD) -tags="$(platform),poste2e" -covermode=atomic -buildmode=exe -v -count=1 -timeout=17m ./test/...
 	# This is a test that should be run in the end to reduce the disruption to other tests because
 	# it will delete a node.
-	KUBECONFIG=${kubeconfig} PLATFORM=${platform} go test -mod=$(MOD) -tags="$(platform),disruptivee2e" -covermode=atomic -buildmode=exe -v -count=1 ./test/...
+	# The timeout is made longer as it will perform actions that will take longer than the default 10 minutes.
+	KUBECONFIG=${kubeconfig} PLATFORM=${platform} go test -timeout 30m -mod=$(MOD) -tags="$(platform),disruptivee2e" -covermode=atomic -buildmode=exe -v -count=1 ./test/...
 
 .PHONY: all
 all: build test

assets/charts/control-plane/kube-apiserver/templates/kube-apiserver-ds.yaml

@@ -24,6 +24,8 @@ spec:
       annotations:
         checkpointer.alpha.coreos.com/checkpoint: "true"
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
+        # Automatically rolls update when secret changes.
+        checksum/secret: {{ include (print $.Template.BasePath "/kube-apiserver-secret.yaml") . | sha256sum }}
     spec:
 {{- template "containers" . }}
 {{- end }}

assets/charts/control-plane/kube-apiserver/templates/kube-apiserver.yaml

@@ -25,6 +25,8 @@ spec:
       annotations:
         checkpointer.alpha.coreos.com/checkpoint: "true"
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
+        # Automatically rolls update when secret changes.
+        checksum/secret: {{ include (print $.Template.BasePath "/kube-apiserver-secret.yaml") . | sha256sum }}
     spec:
 {{- template "containers" . }}
 {{- end }}

assets/charts/control-plane/kubelet/container-image/Dockerfile

@@ -1,3 +1,7 @@
 FROM quay.io/poseidon/kubelet:v1.20.4
 
+RUN clean-install \
+    sed \
+    bash
+
 COPY ./iscsiadm /bin/iscsiadm

assets/charts/control-plane/kubelet/container-image/README.md

@@ -2,6 +2,8 @@
 
 This image is built on top of the Typhoon Kubelet image. This image includes a wrapper script that runs `isciadm` in the host namespace. This ensures that all the `isciadm` libraries are loaded from the host.
 
+We also use `sed` and `bash` of the image to run the `ca-syncer` init container.
+
 ## Updating Image Tag
 
 In this directory run the following command to update the Kubernetes version:

assets/charts/control-plane/kubelet/templates/kubelet-ds.yaml

@@ -17,7 +17,21 @@ spec:
         tier: node
         k8s-app: kubelet
     spec:
-      automountServiceAccountToken: false
+      initContainers:
+      - name: ca-syncer
+        image: {{ .Values.image }}
+        command:
+        - bash
+        - -c
+        - |
+          sed -i "s/^    certificate-authority-data:.*$/    certificate-authority-data: {{ required "kubernetesCACert can't be empty" .Values.kubernetesCACert }}/g" /var/lib/kubelet/kubeconfig /etc/kubernetes/kubeconfig
+        volumeMounts:
+        - name: var-lib-kubelet
+          mountPath: /var/lib/kubelet
+          readOnly: false
+        - name: etc-kubernetes
+          mountPath: /etc/kubernetes
+          readOnly: false
       containers:
       - name: kubelet
         image: {{ .Values.image }}

assets/charts/control-plane/kubelet/values.yaml

@@ -3,3 +3,4 @@ clusterDNS: 10.0.0.10
 clusterDomain: cluster.local
 enableTLSBootstrap: true
 cloudProvider:
+kubernetesCACert: ""

assets/charts/control-plane/kubernetes/templates/kube-controller-manager-ds.yaml

@@ -23,6 +23,8 @@ spec:
         k8s-app: kube-controller-manager
       annotations:
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
+        # Automatically rolls update when secret changes.
+        checksum/secret: {{ include (print $.Template.BasePath "/kube-controller-manager-secret.yaml") . | sha256sum }}
     spec:
 {{- template "controller-manager-containers" . }}
 {{- end }}

assets/charts/control-plane/kubernetes/templates/kube-controller-manager.yaml

@@ -24,6 +24,8 @@ spec:
         k8s-app: kube-controller-manager
       annotations:
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
+        # Automatically rolls update when secret changes.
+        checksum/secret: {{ include (print $.Template.BasePath "/kube-controller-manager-secret.yaml") . | sha256sum }}
     spec:
 {{- template "controller-manager-containers" . }}
 {{- end }}

assets/terraform-modules/aws/flatcar-linux/kubernetes/ssh.tf

@@ -57,9 +57,13 @@ resource "null_resource" "copy-controller-secrets" {
 
   provisioner "remote-exec" {
     inline = [
+      "set -e",
       "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig",
       "sudo chown root:root /etc/kubernetes/kubeconfig",
-      "sudo chmod 644 /etc/kubernetes/kubeconfig",
+      "sudo chmod 600 /etc/kubernetes/kubeconfig",
+      # Using "etcd/." copies the etcd/ folder recursively in an idempotent
+      # way. See https://unix.stackexchange.com/a/228637 for details.
+      "[ -d /etc/ssl/etcd ] && sudo cp -R /etc/ssl/etcd/. /etc/ssl/etcd.old && sudo rm -rf /etc/ssl/etcd",
       "sudo mkdir -p /etc/ssl/etcd/etcd",
       "sudo mv etcd-client* /etc/ssl/etcd/",
       "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt",
@@ -70,11 +74,15 @@ resource "null_resource" "copy-controller-secrets" {
       "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key",
       "sudo chown -R etcd:etcd /etc/ssl/etcd",
       "sudo chmod -R 500 /etc/ssl/etcd",
+      "sudo systemctl restart etcd",
     ]
   }
 
   triggers = {
-    controller_id = aws_instance.controllers[count.index].id
+    controller_id    = aws_instance.controllers[count.index].id
+    etcd_ca_cert     = module.bootkube.etcd_ca_cert
+    etcd_server_cert = module.bootkube.etcd_server_cert
+    etcd_peer_cert   = module.bootkube.etcd_peer_cert
   }
 }
 

assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/ssh.tf

@@ -64,6 +64,13 @@ resource "null_resource" "copy-controller-secrets" {
 
   provisioner "remote-exec" {
     inline = [
+      "set -e",
+      "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig",
+      "sudo chown root:root /etc/kubernetes/kubeconfig",
+      "sudo chmod 600 /etc/kubernetes/kubeconfig",
+      # Using "etcd/." copies the etcd/ folder recursively in an idempotent
+      # way. See https://unix.stackexchange.com/a/228637 for details.
+      "[ -d /etc/ssl/etcd ] && sudo cp -R /etc/ssl/etcd/. /etc/ssl/etcd.old && sudo rm -rf /etc/ssl/etcd",
       "sudo mkdir -p /etc/ssl/etcd/etcd",
       "sudo mv etcd-client* /etc/ssl/etcd/",
       "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt",
@@ -74,9 +81,15 @@ resource "null_resource" "copy-controller-secrets" {
       "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key",
       "sudo chown -R etcd:etcd /etc/ssl/etcd",
       "sudo chmod -R 500 /etc/ssl/etcd",
-      "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig",
+      "sudo systemctl restart etcd",
     ]
   }
+
+  triggers = {
+    etcd_ca_cert     = module.bootkube.etcd_ca_cert
+    etcd_server_cert = module.bootkube.etcd_server_cert
+    etcd_peer_cert   = module.bootkube.etcd_peer_cert
+  }
 }
 
 # Secure copy kubeconfig to all workers. Activates kubelet.service

assets/terraform-modules/bootkube/assets.tf

@@ -109,6 +109,7 @@ locals {
     cluster_domain_suffix  = var.cluster_domain_suffix
     enable_tls_bootstrap   = var.enable_tls_bootstrap
     cloud_provider         = var.cloud_provider
+    kubernetes_ca_cert     = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
   })
 
   kubeconfig_kubelet_content = templatefile("${path.module}/resources/kubeconfig-kubelet", {

assets/terraform-modules/bootkube/resources/charts/kubelet.yaml

@@ -3,3 +3,4 @@ clusterDNS: ${cluster_dns_service_ip}
 clusterDomain: ${cluster_domain_suffix}
 enableTLSBootstrap: ${enable_tls_bootstrap}
 cloudProvider: ${cloud_provider}
+kubernetesCACert: ${kubernetes_ca_cert}

assets/terraform-modules/packet/flatcar-linux/kubernetes/outputs.tf

@@ -35,10 +35,18 @@ output "calico_values" {
   value = module.bootkube.calico_values
 }
 
+output "calico-host-protection_values" {
+  value = local_file.calico_host_protection.content
+}
+
 output "lokomotive_values" {
   value = module.bootkube.lokomotive_values
 }
 
+output "packet-ccm_values" {
+  value = local_file.packet-ccm.content
+}
+
 output "bootstrap-secrets_values" {
   value = module.bootkube.bootstrap-secrets_values
 }

assets/terraform-modules/packet/flatcar-linux/kubernetes/ssh.tf

@@ -46,6 +46,10 @@ resource "null_resource" "copy-controller-secrets" {
 
   provisioner "remote-exec" {
     inline = [
+      "set -e",
+      # Using "etcd/." copies the etcd/ folder recursively in an idempotent
+      # way. See https://unix.stackexchange.com/a/228637 for details.
+      "[ -d /etc/ssl/etcd ] && sudo cp -R /etc/ssl/etcd/. /etc/ssl/etcd.old && sudo rm -rf /etc/ssl/etcd",
       "sudo mkdir -p /etc/ssl/etcd/etcd",
       "sudo mv etcd-client* /etc/ssl/etcd/",
       "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt",
@@ -56,11 +60,14 @@ resource "null_resource" "copy-controller-secrets" {
       "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key",
       "sudo chown -R etcd:etcd /etc/ssl/etcd",
       "sudo chmod -R 500 /etc/ssl/etcd",
+      "sudo systemctl restart etcd",
     ]
   }
 
   triggers = {
-    controller_id = packet_device.controllers[count.index].id
+    etcd_ca_cert     = module.bootkube.etcd_ca_cert
+    etcd_server_cert = module.bootkube.etcd_server_cert
+    etcd_peer_cert   = module.bootkube.etcd_peer_cert
   }
 }
 

assets/terraform-modules/platforms/tinkerbell/controllers/ssh.tf

@@ -51,6 +51,13 @@ resource "null_resource" "copy-controller-secrets" {
 
   provisioner "remote-exec" {
     inline = [
+      "set -e",
+      "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig",
+      "sudo chown root:root /etc/kubernetes/kubeconfig",
+      "sudo chmod 600 /etc/kubernetes/kubeconfig",
+      # Using "etcd/." copies the etcd/ folder recursively in an idempotent
+      # way. See https://unix.stackexchange.com/a/228637 for details.
+      "[ -d /etc/ssl/etcd ] && sudo cp -R /etc/ssl/etcd/. /etc/ssl/etcd.old && sudo rm -rf /etc/ssl/etcd",
       "sudo mkdir -p /etc/ssl/etcd/etcd",
       "sudo mv etcd-client* /etc/ssl/etcd/",
       "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt",
@@ -61,9 +68,15 @@ resource "null_resource" "copy-controller-secrets" {
       "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key",
       "sudo chown -R etcd:etcd /etc/ssl/etcd",
       "sudo chmod -R 500 /etc/ssl/etcd",
-      "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig",
+      "sudo systemctl restart etcd",
     ]
   }
+
+  triggers = {
+    etcd_ca_cert     = module.bootkube.etcd_ca_cert
+    etcd_server_cert = module.bootkube.etcd_server_cert
+    etcd_peer_cert   = module.bootkube.etcd_peer_cert
+  }
 }
 
 # Secure copy bootkube assets to ONE controller and start bootkube to perform

ci/scripts/deploy-cluster.sh

@@ -93,6 +93,7 @@ resource_dir=$(pwd)/..
 cd "ci/$platform"
 cat "$platform-cluster.lokocfg.envsubst" | envsubst '$AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY $PUB_KEY $CLUSTER_ID $AWS_DEFAULT_REGION $AWS_DNS_ZONE $AWS_DNS_ZONE_ID $PACKET_PROJECT_ID $EMAIL $GITHUB_CLIENT_ID $GITHUB_CLIENT_SECRET $DEX_STATIC_CLIENT_CLUSTERAUTH_ID $DEX_STATIC_CLIENT_CLUSTERAUTH_SECRET $GANGWAY_REDIRECT_URL $GANGWAY_SESSION_KEY $DEX_INGRESS_HOST $GANGWAY_INGRESS_HOST $ISSUER_HOST $REDIRECT_URI $API_SERVER_URL $AUTHORIZE_URL $TOKEN_URL $PACKET_LOCATION $ARM_SUBSCRIPTION_ID $ARM_TENANT_ID $ARM_CLIENT_ID $ARM_CLIENT_SECRET' >"$platform-cluster.lokocfg"
 
+export LOKOCFG_LOCATION="$(pwd)/$platform-cluster.lokocfg"
 export KUBECONFIG=$HOME/lokoctl-assets/cluster-assets/auth/kubeconfig
 echo "export KUBECONFIG=$KUBECONFIG" >> ~/.bashrc
 

cli/cmd/cluster-certificate-rotate.go

@@ -0,0 +1,57 @@
+// Copyright 2021 The Lokomotive Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/kinvolk/lokomotive/cli/cmd/cluster"
+)
+
+var clusterCertificateRotateCmd = &cobra.Command{
+	Use:   "rotate",
+	Short: "Rotate certificates of a cluster",
+	Long: `Rotate certificates of a cluster.
+Rotate will replace all certificates inside a cluster with new ones.
+This can be used to renew all certificates with a longer validity.`,
+	Run: runClusterCertificateRotate,
+}
+
+func init() { //nolint:gochecknoinits
+	clusterCertificateCmd.AddCommand(clusterCertificateRotateCmd)
+
+	pf := clusterCertificateRotateCmd.PersistentFlags()
+	pf.BoolVarP(&verbose, "verbose", "v", false, "Show output from Terraform")
+}
+
+func runClusterCertificateRotate(cmd *cobra.Command, args []string) {
+	contextLogger := log.WithFields(log.Fields{
+		"command": "lokoctl cluster certificate rotate",
+		"args":    args,
+	})
+
+	options := cluster.CertificateRotateOptions{
+		Confirm:    confirm,
+		Verbose:    verbose,
+		ConfigPath: viper.GetString("lokocfg"),
+		ValuesPath: viper.GetString("lokocfg-vars"),
+	}
+
+	if err := cluster.RotateCertificates(contextLogger, options); err != nil {
+		contextLogger.Fatalf("Rotating Certificates failed: %v", err)
+	}
+}

cli/cmd/cluster-certificate.go

@@ -0,0 +1,28 @@
+// Copyright 2021 The Lokomotive Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var clusterCertificateCmd = &cobra.Command{
+	Use:   "certificate",
+	Short: "Manage cluster certificates",
+}
+
+func init() { //nolint:gochecknoinits
+	clusterCmd.AddCommand(clusterCertificateCmd)
+}

cli/cmd/cluster/apply.go

@@ -129,21 +129,8 @@ func Apply(contextLogger *log.Entry, options ApplyOptions) error {
 	if exists && !c.platform.Meta().Managed {
 		fmt.Printf("\nEnsuring that cluster controlplane is up to date.\n")
 
-		cu := controlplaneUpdater{
-			kubeconfig:    kubeconfig,
-			assetDir:      c.assetDir,
-			contextLogger: *contextLogger,
-			ex:            c.terraformExecutor,
-		}
-
-		if err := c.unpackControlplaneCharts(); err != nil {
-			return fmt.Errorf("unpacking controlplane assets: %w", err)
-		}
-
-		for _, c := range charts {
-			if err := cu.upgradeComponent(c.Name, c.Namespace); err != nil {
-				return fmt.Errorf("upgrading controlplane component %q: %w", c.Name, err)
-			}
+		if err := c.upgradeControlPlane(contextLogger, kubeconfig); err != nil {
+			return fmt.Errorf("running controlplane upgrade: %v", err)
 		}
 	}