Skip to content

Commit

Permalink
cli: remove helm management from join-config (#2251)
Browse files Browse the repository at this point in the history
* Replace UpdateAttestationConfig with ApplyJoinConfig

* Dont set up join-config over Helm, it is now only managed by our CLI directly during init and upgrade

* Remove measurementSalt and attestationConfig parsing from helm, they were only needed for the JoinConfig

* Add migration step to remove join-config from Helm management

* Update attestation config trouble shooting tip

---------

Signed-off-by: Daniel Weiße <dw@edgeless.systems>
  • Loading branch information
daniel-weisse authored Aug 23, 2023
1 parent c42e81b commit 053aa60
Show file tree
Hide file tree
Showing 21 changed files with 325 additions and 195 deletions.
24 changes: 22 additions & 2 deletions cli/internal/cmd/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/cli/internal/kubecmd"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config"
Expand Down Expand Up @@ -129,13 +130,20 @@ func runInitialize(cmd *cobra.Command, _ []string) error {
return fmt.Errorf("creating Helm installer: %w", err)
}
i := newInitCmd(tfClient, helmInstaller, fileHandler, spinner, &kubeconfigMerger{log: log}, log)

fetcher := attestationconfigapi.NewFetcher()
return i.initialize(cmd, newDialer, license.NewClient(), fetcher)
newAttestationApplier := func(w io.Writer, kubeConfig string, log debugLog) (attestationConfigApplier, error) {
return kubecmd.New(w, kubeConfig, log)
}

return i.initialize(cmd, newDialer, license.NewClient(), fetcher, newAttestationApplier)
}

// initialize initializes a Constellation.
func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.Validator) *dialer.Dialer,
func (i *initCmd) initialize(
cmd *cobra.Command, newDialer func(validator atls.Validator) *dialer.Dialer,
quotaChecker license.QuotaChecker, configFetcher attestationconfigapi.Fetcher,
newAttestationApplier func(io.Writer, string, debugLog) (attestationConfigApplier, error),
) error {
flags, err := i.evalFlagArgs(cmd)
if err != nil {
Expand Down Expand Up @@ -249,6 +257,14 @@ func (i *initCmd) initialize(cmd *cobra.Command, newDialer func(validator atls.V
return err
}

attestationApplier, err := newAttestationApplier(cmd.OutOrStdout(), constants.AdminConfFilename, i.log)
if err != nil {
return err
}
if err := attestationApplier.ApplyJoinConfig(cmd.Context(), conf.GetAttestationConfig(), measurementSalt); err != nil {
return fmt.Errorf("applying attestation config: %w", err)
}

helmLoader := helm.NewLoader(provider, k8sVersion, clusterName)
i.log.Debugf("Created new Helm loader")
output, err := i.clusterShower.ShowCluster(cmd.Context(), conf.GetProvider())
Expand Down Expand Up @@ -609,3 +625,7 @@ func (e *nonRetriableError) Unwrap() error {
type initializer interface {
Install(ctx context.Context, releases *helm.Releases) error
}

type attestationConfigApplier interface {
ApplyJoinConfig(ctx context.Context, newAttestConfig config.AttestationCfg, measurementSalt []byte) error
}
28 changes: 26 additions & 2 deletions cli/internal/cmd/init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,15 @@ func TestInitialize(t *testing.T) {
defer cancel()
cmd.SetContext(ctx)
i := newInitCmd(&stubShowCluster{}, &stubHelmInstaller{}, fileHandler, &nopSpinner{}, nil, logger.NewTest(t))
err := i.initialize(cmd, newDialer, &stubLicenseClient{}, stubAttestationFetcher{})
err := i.initialize(
cmd,
newDialer,
&stubLicenseClient{},
stubAttestationFetcher{},
func(io.Writer, string, debugLog) (attestationConfigApplier, error) {
return &stubAttestationApplier{}, nil
},
)

if tc.wantErr {
assert.Error(err)
Expand Down Expand Up @@ -486,7 +494,15 @@ func TestAttestation(t *testing.T) {
cmd.SetContext(ctx)

i := newInitCmd(nil, nil, fileHandler, &nopSpinner{}, nil, logger.NewTest(t))
err := i.initialize(cmd, newDialer, &stubLicenseClient{}, stubAttestationFetcher{})
err := i.initialize(
cmd,
newDialer,
&stubLicenseClient{},
stubAttestationFetcher{},
func(io.Writer, string, debugLog) (attestationConfigApplier, error) {
return &stubAttestationApplier{}, nil
},
)
assert.Error(err)
// make sure the error is actually a TLS handshake error
assert.Contains(err.Error(), "transport: authentication handshake failed")
Expand Down Expand Up @@ -666,3 +682,11 @@ func (s *stubShowCluster) ShowCluster(_ context.Context, csp cloudprovider.Provi
}
return res, nil
}

type stubAttestationApplier struct {
applyErr error
}

func (a *stubAttestationApplier) ApplyJoinConfig(_ context.Context, _ config.AttestationCfg, _ []byte) error {
return a.applyErr
}
8 changes: 7 additions & 1 deletion cli/internal/cmd/miniup.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,14 @@ import (
"context"
"errors"
"fmt"
"io"
"net"

"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix"
"github.com/edgelesssys/constellation/v2/cli/internal/featureset"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/cli/internal/kubecmd"
"github.com/edgelesssys/constellation/v2/cli/internal/libvirt"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
Expand Down Expand Up @@ -215,8 +217,12 @@ func (m *miniUpCmd) initializeMiniCluster(cmd *cobra.Command, fileHandler file.H
return fmt.Errorf("creating Terraform client: %w", err)
}

newAttestationApplier := func(w io.Writer, kubeConfig string, log debugLog) (attestationConfigApplier, error) {
return kubecmd.New(w, kubeConfig, log)
}

i := newInitCmd(tfClient, helmInstaller, fileHandler, spinner, &kubeconfigMerger{log: log}, log)
if err := i.initialize(cmd, newDialer, license.NewClient(), m.configFetcher); err != nil {
if err := i.initialize(cmd, newDialer, license.NewClient(), m.configFetcher, newAttestationApplier); err != nil {
return err
}
m.log.Debugf("Initialized mini cluster")
Expand Down
99 changes: 72 additions & 27 deletions cli/internal/cmd/upgradeapply.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
"github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/rogpeppe/go-internal/diff"
"github.com/spf13/afero"
Expand Down Expand Up @@ -153,7 +154,15 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command) error {
}
conf.UpdateMAAURL(idFile.AttestationURL)

if err := u.confirmAttestationConfigUpgrade(cmd, conf.GetAttestationConfig(), flags); err != nil {
// Apply migrations necessary for the upgrade
if err := migrateFrom2_10(cmd.Context(), u.kubeUpgrader); err != nil {
return fmt.Errorf("applying migration for upgrading from v2.10: %w", err)
}
if err := migrateFrom2_11(cmd.Context(), u.kubeUpgrader); err != nil {
return fmt.Errorf("applying migration for upgrading from v2.11: %w", err)
}

if err := u.confirmAndUpgradeAttestationConfig(cmd, conf.GetAttestationConfig(), idFile.MeasurementSalt, flags); err != nil {
return fmt.Errorf("upgrading measurements: %w", err)
}

Expand All @@ -177,29 +186,30 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command) error {
return fmt.Errorf("extending cert SANs: %w", err)
}

if conf.GetProvider() == cloudprovider.Azure || conf.GetProvider() == cloudprovider.GCP || conf.GetProvider() == cloudprovider.AWS {
var upgradeErr *compatibility.InvalidUpgradeError
err = u.handleServiceUpgrade(cmd, conf, idFile, tfOutput, validK8sVersion, flags)
switch {
case errors.As(err, &upgradeErr):
cmd.PrintErrln(err)
case err == nil:
cmd.Println("Successfully upgraded Constellation services.")
case err != nil:
return fmt.Errorf("upgrading services: %w", err)
}

err = u.kubeUpgrader.UpgradeNodeVersion(cmd.Context(), conf, flags.force)
switch {
case errors.Is(err, kubecmd.ErrInProgress):
cmd.PrintErrln("Skipping image and Kubernetes upgrades. Another upgrade is in progress.")
case errors.As(err, &upgradeErr):
cmd.PrintErrln(err)
case err != nil:
return fmt.Errorf("upgrading NodeVersion: %w", err)
}
} else {
if conf.GetProvider() != cloudprovider.Azure && conf.GetProvider() != cloudprovider.GCP && conf.GetProvider() != cloudprovider.AWS {
cmd.PrintErrln("WARNING: Skipping service and image upgrades, which are currently only supported for AWS, Azure, and GCP.")
return nil
}

var upgradeErr *compatibility.InvalidUpgradeError
err = u.handleServiceUpgrade(cmd, conf, idFile, tfOutput, validK8sVersion, flags)
switch {
case errors.As(err, &upgradeErr):
cmd.PrintErrln(err)
case err == nil:
cmd.Println("Successfully upgraded Constellation services.")
case err != nil:
return fmt.Errorf("upgrading services: %w", err)
}

err = u.kubeUpgrader.UpgradeNodeVersion(cmd.Context(), conf, flags.force)
switch {
case errors.Is(err, kubecmd.ErrInProgress):
cmd.PrintErrln("Skipping image and Kubernetes upgrades. Another upgrade is in progress.")
case errors.As(err, &upgradeErr):
cmd.PrintErrln(err)
case err != nil:
return fmt.Errorf("upgrading NodeVersion: %w", err)
}

return nil
Expand Down Expand Up @@ -338,9 +348,11 @@ func validK8sVersion(cmd *cobra.Command, version string, yes bool) (validVersion
return validVersion, nil
}

// confirmAttestationConfigUpgrade checks if the locally configured measurements are different from the cluster's measurements.
// confirmAndUpgradeAttestationConfig checks if the locally configured measurements are different from the cluster's measurements.
// If so the function will ask the user to confirm (if --yes is not set) and upgrade the cluster's config.
func (u *upgradeApplyCmd) confirmAttestationConfigUpgrade(cmd *cobra.Command, newConfig config.AttestationCfg, flags upgradeApplyFlags) error {
func (u *upgradeApplyCmd) confirmAndUpgradeAttestationConfig(
cmd *cobra.Command, newConfig config.AttestationCfg, measurementSalt []byte, flags upgradeApplyFlags,
) error {
clusterAttestationConfig, err := u.kubeUpgrader.GetClusterAttestationConfig(cmd.Context(), newConfig.GetVariant())
if err != nil {
return fmt.Errorf("getting cluster attestation config: %w", err)
Expand Down Expand Up @@ -371,9 +383,10 @@ func (u *upgradeApplyCmd) confirmAttestationConfigUpgrade(cmd *cobra.Command, ne
}
}

if err := u.kubeUpgrader.UpdateAttestationConfig(cmd.Context(), newConfig); err != nil {
if err := u.kubeUpgrader.ApplyJoinConfig(cmd.Context(), newConfig, measurementSalt); err != nil {
return fmt.Errorf("updating attestation config: %w", err)
}
cmd.Println("Successfully update the cluster's attestation config")
return nil
}

Expand Down Expand Up @@ -413,6 +426,34 @@ func (u *upgradeApplyCmd) handleServiceUpgrade(cmd *cobra.Command, conf *config.
return err
}

// migrateFrom2_10 applies migrations necessary for upgrading from v2.10 to v2.11
// TODO(v2.11): Remove this function after v2.11 is released.
func migrateFrom2_10(ctx context.Context, kubeUpgrader kubernetesUpgrader) error {
// Sanity check to make sure we only run migrations on upgrades with CLI version 2.10 < v < 2.12
if !constants.BinaryVersion().MajorMinorEqual(semver.NewFromInt(2, 11, 0, "")) {
return nil
}

if err := kubeUpgrader.RemoveAttestationConfigHelmManagement(ctx); err != nil {
return fmt.Errorf("removing helm management from attestation config: %w", err)
}
return nil
}

// migrateFrom2_11 applies migrations necessary for upgrading from v2.11 to v2.12
// TODO(v2.12): Remove this function after v2.12 is released.
func migrateFrom2_11(ctx context.Context, kubeUpgrader kubernetesUpgrader) error {
// Sanity check to make sure we only run migrations on upgrades with CLI version 2.11 < v < 2.13
if !constants.BinaryVersion().MajorMinorEqual(semver.NewFromInt(2, 12, 0, "")) {
return nil
}

if err := kubeUpgrader.RemoveHelmKeepAnnotation(ctx); err != nil {
return fmt.Errorf("removing helm keep annotation: %w", err)
}
return nil
}

func parseUpgradeApplyFlags(cmd *cobra.Command) (upgradeApplyFlags, error) {
workDir, err := cmd.Flags().GetString("workspace")
if err != nil {
Expand Down Expand Up @@ -493,7 +534,11 @@ type kubernetesUpgrader interface {
UpgradeNodeVersion(ctx context.Context, conf *config.Config, force bool) error
ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error
GetClusterAttestationConfig(ctx context.Context, variant variant.Variant) (config.AttestationCfg, error)
UpdateAttestationConfig(ctx context.Context, newAttestConfig config.AttestationCfg) error
ApplyJoinConfig(ctx context.Context, newAttestConfig config.AttestationCfg, measurementSalt []byte) error
// TODO(v2.11): Remove this function after v2.11 is released.
RemoveAttestationConfigHelmManagement(ctx context.Context) error
// TODO(v2.12): Remove this function after v2.12 is released.
RemoveHelmKeepAnnotation(ctx context.Context) error
}

type helmUpgrader interface {
Expand Down
12 changes: 11 additions & 1 deletion cli/internal/cmd/upgradeapply_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ func (u stubKubernetesUpgrader) UpgradeNodeVersion(_ context.Context, _ *config.
return u.nodeVersionErr
}

func (u stubKubernetesUpgrader) UpdateAttestationConfig(_ context.Context, _ config.AttestationCfg) error {
func (u stubKubernetesUpgrader) ApplyJoinConfig(_ context.Context, _ config.AttestationCfg, _ []byte) error {
return nil
}

Expand All @@ -209,6 +209,16 @@ func (u stubKubernetesUpgrader) ExtendClusterConfigCertSANs(_ context.Context, _
return nil
}

// TODO(v2.11): Remove this function.
func (u stubKubernetesUpgrader) RemoveAttestationConfigHelmManagement(_ context.Context) error {
return nil
}

// TODO(v2.12): Remove this function.
func (u stubKubernetesUpgrader) RemoveHelmKeepAnnotation(_ context.Context) error {
return nil
}

type stubTerraformUpgrader struct {
terraformDiff bool
planTerraformErr error
Expand Down
2 changes: 0 additions & 2 deletions cli/internal/helm/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,6 @@ go_library(
"charts/edgeless/constellation-services/charts/join-service/Chart.yaml",
"charts/edgeless/constellation-services/charts/join-service/templates/clusterrole.yaml",
"charts/edgeless/constellation-services/charts/join-service/templates/clusterrolebinding.yaml",
"charts/edgeless/constellation-services/charts/join-service/templates/configmap.yaml",
"charts/edgeless/constellation-services/charts/join-service/templates/daemonset.yaml",
"charts/edgeless/constellation-services/charts/join-service/templates/service.yaml",
"charts/edgeless/constellation-services/charts/join-service/templates/serviceaccount.yaml",
Expand Down Expand Up @@ -468,7 +467,6 @@ go_test(
deps = [
"//cli/internal/clusterid",
"//cli/internal/terraform",
"//internal/attestation/idkeydigest",
"//internal/attestation/measurements",
"//internal/cloud/azureshared",
"//internal/cloud/cloudprovider",
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -3,33 +3,33 @@
"properties": {
"csp": {
"description": "CSP to which the chart is deployed.",
"enum": ["AWS", "Azure", "GCP", "OpenStack", "QEMU"]
},
"attestationConfig": {
"description": "JSON-string to describe the config to use for attestation validation.",
"type": "string",
"examples": ["{'measurements':{'1':{'expected':'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA','warnOnly':true},'15':{'expected':'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=','warnOnly':true}}}"]
"enum": [
"AWS",
"Azure",
"GCP",
"OpenStack",
"QEMU"
]
},
"image": {
"description": "Container image to use for the spawned pods.",
"type": "string",
"examples": ["ghcr.io/edgelesssys/constellation/join-service:latest"]
},
"measurementSalt": {
"description": "Salt used to generate node measurements",
"type": "string",
"examples": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"]
"examples": [
"ghcr.io/edgelesssys/constellation/join-service:latest"
]
},
"attestationVariant": {
"description": "Attestation variant to use for aTLS connections.",
"type": "string",
"examples": ["azure-sev-snp", "azure-trusted-launch", "gcp-sev-es"]
"examples": [
"azure-sev-snp",
"azure-trusted-launch",
"gcp-sev-es"
]
}
},
"required": [
"csp",
"attestationConfig",
"measurementSalt",
"image",
"attestationVariant"
],
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
csp: "gcp"
attestationVariant: ""
measurementSalt: ""
joinServicePort: 9090
joinServiceNodePort: 30090
Loading

0 comments on commit 053aa60

Please sign in to comment.