cli: remove helm management from join-config (#2251)

* Replace UpdateAttestationConfig with ApplyJoinConfig

* Dont set up join-config over Helm, it is now only managed by our CLI directly during init and upgrade

* Remove measurementSalt and attestationConfig parsing from helm, they were only needed for the JoinConfig

* Add migration step to remove join-config from Helm management

* Update attestation config trouble shooting tip

---------

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

cli/internal/cmd/init.go

@@ -39,6 +39,7 @@ import (
 	"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
 	"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix"
 	"github.com/edgelesssys/constellation/v2/cli/internal/helm"
+	"github.com/edgelesssys/constellation/v2/cli/internal/kubecmd"
 	"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
 	"github.com/edgelesssys/constellation/v2/internal/config"
@@ -129,13 +130,20 @@ func runInitialize(cmd *cobra.Command, _ []string) error {
 		return fmt.Errorf("creating Helm installer: %w", err)
 	}
 	i := newInitCmd(tfClient, helmInstaller, fileHandler, spinner, &kubeconfigMerger{log: log}, log)
+
 	fetcher := attestationconfigapi.NewFetcher()
-	return i.initialize(cmd, newDialer, license.NewClient(), fetcher)
+	newAttestationApplier := func(w io.Writer, kubeConfig string, log debugLog) (attestationConfigApplier, error) {
+		return kubecmd.New(w, kubeConfig, log)
+	}
+
+	return i.initialize(cmd, newDialer, license.NewClient(), fetcher, newAttestationApplier)
 }
 
 // initialize initializes a Constellation.
-func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.Validator) *dialer.Dialer,
+func (i *initCmd) initialize(
+	cmd *cobra.Command, newDialer func(validator atls.Validator) *dialer.Dialer,
 	quotaChecker license.QuotaChecker, configFetcher attestationconfigapi.Fetcher,
+	newAttestationApplier func(io.Writer, string, debugLog) (attestationConfigApplier, error),
 ) error {
 	flags, err := i.evalFlagArgs(cmd)
 	if err != nil {
@@ -249,6 +257,14 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V
 		return err
 	}
 
+	attestationApplier, err := newAttestationApplier(cmd.OutOrStdout(), constants.AdminConfFilename, i.log)
+	if err != nil {
+		return err
+	}
+	if err := attestationApplier.ApplyJoinConfig(cmd.Context(), conf.GetAttestationConfig(), measurementSalt); err != nil {
+		return fmt.Errorf("applying attestation config: %w", err)
+	}
+
 	helmLoader := helm.NewLoader(provider, k8sVersion, clusterName)
 	i.log.Debugf("Created new Helm loader")
 	output, err := i.clusterShower.ShowCluster(cmd.Context(), conf.GetProvider())
@@ -609,3 +625,7 @@ func (e *nonRetriableError) Unwrap() error {
 type initializer interface {
 	Install(ctx context.Context, releases *helm.Releases) error
 }
+
+type attestationConfigApplier interface {
+	ApplyJoinConfig(ctx context.Context, newAttestConfig config.AttestationCfg, measurementSalt []byte) error
+}

cli/internal/cmd/init_test.go

@@ -187,7 +187,15 @@ func TestInitialize(t *testing.T) {
 			defer cancel()
 			cmd.SetContext(ctx)
 			i := newInitCmd(&stubShowCluster{}, &stubHelmInstaller{}, fileHandler, &nopSpinner{}, nil, logger.NewTest(t))
-			err := i.initialize(cmd, newDialer, &stubLicenseClient{}, stubAttestationFetcher{})
+			err := i.initialize(
+				cmd,
+				newDialer,
+				&stubLicenseClient{},
+				stubAttestationFetcher{},
+				func(io.Writer, string, debugLog) (attestationConfigApplier, error) {
+					return &stubAttestationApplier{}, nil
+				},
+			)
 
 			if tc.wantErr {
 				assert.Error(err)
@@ -486,7 +494,15 @@ func TestAttestation(t *testing.T) {
 	cmd.SetContext(ctx)
 
 	i := newInitCmd(nil, nil, fileHandler, &nopSpinner{}, nil, logger.NewTest(t))
-	err := i.initialize(cmd, newDialer, &stubLicenseClient{}, stubAttestationFetcher{})
+	err := i.initialize(
+		cmd,
+		newDialer,
+		&stubLicenseClient{},
+		stubAttestationFetcher{},
+		func(io.Writer, string, debugLog) (attestationConfigApplier, error) {
+			return &stubAttestationApplier{}, nil
+		},
+	)
 	assert.Error(err)
 	// make sure the error is actually a TLS handshake error
 	assert.Contains(err.Error(), "transport: authentication handshake failed")
@@ -666,3 +682,11 @@ func (s *stubShowCluster) ShowCluster(_ context.Context, csp cloudprovider.Provi
 	}
 	return res, nil
 }
+
+type stubAttestationApplier struct {
+	applyErr error
+}
+
+func (a *stubAttestationApplier) ApplyJoinConfig(_ context.Context, _ config.AttestationCfg, _ []byte) error {
+	return a.applyErr
+}

cli/internal/cmd/miniup.go

@@ -10,12 +10,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 
 	"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
 	"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix"
 	"github.com/edgelesssys/constellation/v2/cli/internal/featureset"
 	"github.com/edgelesssys/constellation/v2/cli/internal/helm"
+	"github.com/edgelesssys/constellation/v2/cli/internal/kubecmd"
 	"github.com/edgelesssys/constellation/v2/cli/internal/libvirt"
 	"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
 	"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
@@ -215,8 +217,12 @@ func (m *miniUpCmd) initializeMiniCluster(cmd *cobra.Command, fileHandler file.H
 		return fmt.Errorf("creating Terraform client: %w", err)
 	}
 
+	newAttestationApplier := func(w io.Writer, kubeConfig string, log debugLog) (attestationConfigApplier, error) {
+		return kubecmd.New(w, kubeConfig, log)
+	}
+
 	i := newInitCmd(tfClient, helmInstaller, fileHandler, spinner, &kubeconfigMerger{log: log}, log)
-	if err := i.initialize(cmd, newDialer, license.NewClient(), m.configFetcher); err != nil {
+	if err := i.initialize(cmd, newDialer, license.NewClient(), m.configFetcher, newAttestationApplier); err != nil {
 		return err
 	}
 	m.log.Debugf("Initialized mini cluster")

cli/internal/cmd/upgradeapply.go

@@ -29,6 +29,7 @@ import (
 	"github.com/edgelesssys/constellation/v2/internal/file"
 	"github.com/edgelesssys/constellation/v2/internal/kms/uri"
 	"github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl"
+	"github.com/edgelesssys/constellation/v2/internal/semver"
 	"github.com/edgelesssys/constellation/v2/internal/versions"
 	"github.com/rogpeppe/go-internal/diff"
 	"github.com/spf13/afero"
@@ -153,7 +154,15 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command) error {
 	}
 	conf.UpdateMAAURL(idFile.AttestationURL)
 
-	if err := u.confirmAttestationConfigUpgrade(cmd, conf.GetAttestationConfig(), flags); err != nil {
+	// Apply migrations necessary for the upgrade
+	if err := migrateFrom2_10(cmd.Context(), u.kubeUpgrader); err != nil {
+		return fmt.Errorf("applying migration for upgrading from v2.10: %w", err)
+	}
+	if err := migrateFrom2_11(cmd.Context(), u.kubeUpgrader); err != nil {
+		return fmt.Errorf("applying migration for upgrading from v2.11: %w", err)
+	}
+
+	if err := u.confirmAndUpgradeAttestationConfig(cmd, conf.GetAttestationConfig(), idFile.MeasurementSalt, flags); err != nil {
 		return fmt.Errorf("upgrading measurements: %w", err)
 	}
 
@@ -177,29 +186,30 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command) error {
 		return fmt.Errorf("extending cert SANs: %w", err)
 	}
 
-	if conf.GetProvider() == cloudprovider.Azure || conf.GetProvider() == cloudprovider.GCP || conf.GetProvider() == cloudprovider.AWS {
-		var upgradeErr *compatibility.InvalidUpgradeError
-		err = u.handleServiceUpgrade(cmd, conf, idFile, tfOutput, validK8sVersion, flags)
-		switch {
-		case errors.As(err, &upgradeErr):
-			cmd.PrintErrln(err)
-		case err == nil:
-			cmd.Println("Successfully upgraded Constellation services.")
-		case err != nil:
-			return fmt.Errorf("upgrading services: %w", err)
-		}
-
-		err = u.kubeUpgrader.UpgradeNodeVersion(cmd.Context(), conf, flags.force)
-		switch {
-		case errors.Is(err, kubecmd.ErrInProgress):
-			cmd.PrintErrln("Skipping image and Kubernetes upgrades. Another upgrade is in progress.")
-		case errors.As(err, &upgradeErr):
-			cmd.PrintErrln(err)
-		case err != nil:
-			return fmt.Errorf("upgrading NodeVersion: %w", err)
-		}
-	} else {
+	if conf.GetProvider() != cloudprovider.Azure && conf.GetProvider() != cloudprovider.GCP && conf.GetProvider() != cloudprovider.AWS {
 		cmd.PrintErrln("WARNING: Skipping service and image upgrades, which are currently only supported for AWS, Azure, and GCP.")
+		return nil
+	}
+
+	var upgradeErr *compatibility.InvalidUpgradeError
+	err = u.handleServiceUpgrade(cmd, conf, idFile, tfOutput, validK8sVersion, flags)
+	switch {
+	case errors.As(err, &upgradeErr):
+		cmd.PrintErrln(err)
+	case err == nil:
+		cmd.Println("Successfully upgraded Constellation services.")
+	case err != nil:
+		return fmt.Errorf("upgrading services: %w", err)
+	}
+
+	err = u.kubeUpgrader.UpgradeNodeVersion(cmd.Context(), conf, flags.force)
+	switch {
+	case errors.Is(err, kubecmd.ErrInProgress):
+		cmd.PrintErrln("Skipping image and Kubernetes upgrades. Another upgrade is in progress.")
+	case errors.As(err, &upgradeErr):
+		cmd.PrintErrln(err)
+	case err != nil:
+		return fmt.Errorf("upgrading NodeVersion: %w", err)
 	}
 
 	return nil
@@ -338,9 +348,11 @@ func validK8sVersion(cmd *cobra.Command, version string, yes bool) (validVersion
 	return validVersion, nil
 }
 
-// confirmAttestationConfigUpgrade checks if the locally configured measurements are different from the cluster's measurements.
+// confirmAndUpgradeAttestationConfig checks if the locally configured measurements are different from the cluster's measurements.
 // If so the function will ask the user to confirm (if --yes is not set) and upgrade the cluster's config.
-func (u *upgradeApplyCmd) confirmAttestationConfigUpgrade(cmd *cobra.Command, newConfig config.AttestationCfg, flags upgradeApplyFlags) error {
+func (u *upgradeApplyCmd) confirmAndUpgradeAttestationConfig(
+	cmd *cobra.Command, newConfig config.AttestationCfg, measurementSalt []byte, flags upgradeApplyFlags,
+) error {
 	clusterAttestationConfig, err := u.kubeUpgrader.GetClusterAttestationConfig(cmd.Context(), newConfig.GetVariant())
 	if err != nil {
 		return fmt.Errorf("getting cluster attestation config: %w", err)
@@ -371,9 +383,10 @@ func (u *upgradeApplyCmd) confirmAttestationConfigUpgrade(cmd *cobra.Command, ne
 		}
 	}
 
-	if err := u.kubeUpgrader.UpdateAttestationConfig(cmd.Context(), newConfig); err != nil {
+	if err := u.kubeUpgrader.ApplyJoinConfig(cmd.Context(), newConfig, measurementSalt); err != nil {
 		return fmt.Errorf("updating attestation config: %w", err)
 	}
+	cmd.Println("Successfully update the cluster's attestation config")
 	return nil
 }
 
@@ -413,6 +426,34 @@ func (u *upgradeApplyCmd) handleServiceUpgrade(cmd *cobra.Command, conf *config.
 	return err
 }
 
+// migrateFrom2_10 applies migrations necessary for upgrading from v2.10 to v2.11
+// TODO(v2.11): Remove this function after v2.11 is released.
+func migrateFrom2_10(ctx context.Context, kubeUpgrader kubernetesUpgrader) error {
+	// Sanity check to make sure we only run migrations on upgrades with CLI version 2.10 < v < 2.12
+	if !constants.BinaryVersion().MajorMinorEqual(semver.NewFromInt(2, 11, 0, "")) {
+		return nil
+	}
+
+	if err := kubeUpgrader.RemoveAttestationConfigHelmManagement(ctx); err != nil {
+		return fmt.Errorf("removing helm management from attestation config: %w", err)
+	}
+	return nil
+}
+
+// migrateFrom2_11 applies migrations necessary for upgrading from v2.11 to v2.12
+// TODO(v2.12): Remove this function after v2.12 is released.
+func migrateFrom2_11(ctx context.Context, kubeUpgrader kubernetesUpgrader) error {
+	// Sanity check to make sure we only run migrations on upgrades with CLI version 2.11 < v < 2.13
+	if !constants.BinaryVersion().MajorMinorEqual(semver.NewFromInt(2, 12, 0, "")) {
+		return nil
+	}
+
+	if err := kubeUpgrader.RemoveHelmKeepAnnotation(ctx); err != nil {
+		return fmt.Errorf("removing helm keep annotation: %w", err)
+	}
+	return nil
+}
+
 func parseUpgradeApplyFlags(cmd *cobra.Command) (upgradeApplyFlags, error) {
 	workDir, err := cmd.Flags().GetString("workspace")
 	if err != nil {
@@ -493,7 +534,11 @@ type kubernetesUpgrader interface {
 	UpgradeNodeVersion(ctx context.Context, conf *config.Config, force bool) error
 	ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error
 	GetClusterAttestationConfig(ctx context.Context, variant variant.Variant) (config.AttestationCfg, error)
-	UpdateAttestationConfig(ctx context.Context, newAttestConfig config.AttestationCfg) error
+	ApplyJoinConfig(ctx context.Context, newAttestConfig config.AttestationCfg, measurementSalt []byte) error
+	// TODO(v2.11): Remove this function after v2.11 is released.
+	RemoveAttestationConfigHelmManagement(ctx context.Context) error
+	// TODO(v2.12): Remove this function after v2.12 is released.
+	RemoveHelmKeepAnnotation(ctx context.Context) error
 }
 
 type helmUpgrader interface {

cli/internal/cmd/upgradeapply_test.go

@@ -197,7 +197,7 @@ func (u stubKubernetesUpgrader) UpgradeNodeVersion(_ context.Context, _ *config.
 	return u.nodeVersionErr
 }
 
-func (u stubKubernetesUpgrader) UpdateAttestationConfig(_ context.Context, _ config.AttestationCfg) error {
+func (u stubKubernetesUpgrader) ApplyJoinConfig(_ context.Context, _ config.AttestationCfg, _ []byte) error {
 	return nil
 }
 
@@ -209,6 +209,16 @@ func (u stubKubernetesUpgrader) ExtendClusterConfigCertSANs(_ context.Context, _
 	return nil
 }
 
+// TODO(v2.11): Remove this function.
+func (u stubKubernetesUpgrader) RemoveAttestationConfigHelmManagement(_ context.Context) error {
+	return nil
+}
+
+// TODO(v2.12): Remove this function.
+func (u stubKubernetesUpgrader) RemoveHelmKeepAnnotation(_ context.Context) error {
+	return nil
+}
+
 type stubTerraformUpgrader struct {
 	terraformDiff     bool
 	planTerraformErr  error

cli/internal/helm/BUILD.bazel

@@ -226,7 +226,6 @@ go_library(
         "charts/edgeless/constellation-services/charts/join-service/Chart.yaml",
         "charts/edgeless/constellation-services/charts/join-service/templates/clusterrole.yaml",
         "charts/edgeless/constellation-services/charts/join-service/templates/clusterrolebinding.yaml",
-        "charts/edgeless/constellation-services/charts/join-service/templates/configmap.yaml",
         "charts/edgeless/constellation-services/charts/join-service/templates/daemonset.yaml",
         "charts/edgeless/constellation-services/charts/join-service/templates/service.yaml",
         "charts/edgeless/constellation-services/charts/join-service/templates/serviceaccount.yaml",
@@ -468,7 +467,6 @@ go_test(
     deps = [
         "//cli/internal/clusterid",
         "//cli/internal/terraform",
-        "//internal/attestation/idkeydigest",
         "//internal/attestation/measurements",
         "//internal/cloud/azureshared",
         "//internal/cloud/cloudprovider",

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/values.schema.json

@@ -3,33 +3,33 @@
     "properties": {
         "csp": {
             "description": "CSP to which the chart is deployed.",
-            "enum": ["AWS", "Azure", "GCP", "OpenStack", "QEMU"]
-        },
-        "attestationConfig": {
-            "description": "JSON-string to describe the config to use for attestation validation.",
-            "type": "string",
-            "examples": ["{'measurements':{'1':{'expected':'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA','warnOnly':true},'15':{'expected':'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=','warnOnly':true}}}"]
+            "enum": [
+                "AWS",
+                "Azure",
+                "GCP",
+                "OpenStack",
+                "QEMU"
+            ]
         },
         "image": {
             "description": "Container image to use for the spawned pods.",
             "type": "string",
-            "examples": ["ghcr.io/edgelesssys/constellation/join-service:latest"]
-        },
-        "measurementSalt": {
-            "description": "Salt used to generate node measurements",
-            "type": "string",
-            "examples": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"]
+            "examples": [
+                "ghcr.io/edgelesssys/constellation/join-service:latest"
+            ]
         },
         "attestationVariant": {
             "description": "Attestation variant to use for aTLS connections.",
             "type": "string",
-            "examples": ["azure-sev-snp", "azure-trusted-launch", "gcp-sev-es"]
+            "examples": [
+                "azure-sev-snp",
+                "azure-trusted-launch",
+                "gcp-sev-es"
+            ]
         }
     },
     "required": [
         "csp",
-        "attestationConfig",
-        "measurementSalt",
         "image",
         "attestationVariant"
     ],

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/values.yaml

@@ -1,5 +1,4 @@
 csp: "gcp"
 attestationVariant: ""
-measurementSalt: ""
 joinServicePort: 9090
 joinServiceNodePort: 30090