From f0d80287598614d6c58266f60704f89bbf3f8654 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Thu, 11 Jan 2024 20:28:38 -0800 Subject: [PATCH 01/12] feat: containerd configuration for mirror registry --- api/v1alpha1/clusterconfig_types.go | 43 +++-------- api/v1alpha1/zz_generated.deepcopy.go | 52 +++++--------- .../aws/mutation/metapatch_handler_test.go | 2 - .../docker/mutation/metapatch_handler_test.go | 2 - .../credential_provider_config_files.go | 1 + .../imageregistries/credentials/inject.go | 60 +++++++++------- .../imageregistries/credentials/mirror.go | 71 +++++++++++++++++++ .../credentials/templates/hosts.toml.gotmpl | 5 ++ .../credentials/tests/generate_patches.go | 24 +++---- .../credentials/variables_test.go | 20 +++--- 10 files changed, 160 insertions(+), 120 deletions(-) create mode 100644 pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go create mode 100644 pkg/handlers/generic/mutation/imageregistries/credentials/templates/hosts.toml.gotmpl diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index c04ba04e1..6eb3beaca 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -237,52 +237,31 @@ func (ExtraAPIServerCertSANs) VariableSchema() clusterv1.VariableSchema { } } -type ImageRegistries struct { - // +optional - ImageRegistryCredentials ImageRegistryCredentials `json:"credentials,omitempty"` -} +type ImageRegistries []ImageRegistry func (ImageRegistries) VariableSchema() clusterv1.VariableSchema { + resourceSchema := ImageRegistry{}.VariableSchema().OpenAPIV3Schema return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ Description: "Configuration for image registries.", - Type: "object", - Properties: map[string]clusterv1.JSONSchemaProps{ - "credentials": ImageRegistryCredentials{}.VariableSchema().OpenAPIV3Schema, - }, - }, - } -} - -type ImageRegistryCredentials []ImageRegistryCredentialsResource - -func (ImageRegistryCredentials) VariableSchema() clusterv1.VariableSchema { - resourceSchema := ImageRegistryCredentialsResource{}.VariableSchema().OpenAPIV3Schema - - return clusterv1.VariableSchema{ - OpenAPIV3Schema: clusterv1.JSONSchemaProps{ - Description: "Image registry credentials to set up on all Nodes in the cluster. " + - "Enabling this will configure the Kubelets with " + - "https://kubernetes.io/docs/tasks/administer-cluster/kubelet-credential-provider/.", - Type: "array", - Items: &resourceSchema, + Type: "array", + Items: &resourceSchema, }, } } -// ImageRegistryCredentialsResource required for providing credentials for an image registry URL. -type ImageRegistryCredentialsResource struct { +type ImageRegistry struct { // Registry URL. URL string `json:"url"` - // The Secret containing the registry credentials. - // The Secret should have keys 'username' and 'password'. + // The Secret containing the registry credentials and CA certificate + // The Secret should have keys 'username', 'password' and 'caCert' // This credentials Secret is not required for some registries, e.g. ECR. // +optional - Secret *corev1.ObjectReference `json:"secretRef,omitempty"` + CredentialsSecret *corev1.ObjectReference `json:"secretRef,omitempty"` } -func (ImageRegistryCredentialsResource) VariableSchema() clusterv1.VariableSchema { +func (ImageRegistry) VariableSchema() clusterv1.VariableSchema { return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ Type: "object", @@ -292,8 +271,8 @@ func (ImageRegistryCredentialsResource) VariableSchema() clusterv1.VariableSchem Type: "string", }, "secretRef": { - Description: "The Secret containing the registry credentials. " + - "The Secret should have keys 'username' and 'password'. " + + Description: "The Secret containing the registry credentials and CA certificates " + + "The Secret should have keys 'username', 'password' and 'ca.crt' " + "This credentials Secret is not required for some registries, e.g. ECR.", Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 27897b8dd..8e6dff6a0 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -435,7 +435,13 @@ func (in *GenericClusterConfig) DeepCopyInto(out *GenericClusterConfig) { *out = make(ExtraAPIServerCertSANs, len(*in)) copy(*out, *in) } - in.ImageRegistries.DeepCopyInto(&out.ImageRegistries) + if in.ImageRegistries != nil { + in, out := &in.ImageRegistries, &out.ImageRegistries + *out = make(ImageRegistries, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Addons != nil { in, out := &in.Addons, &out.Addons *out = new(Addons) @@ -504,64 +510,42 @@ func (in *Image) DeepCopy() *Image { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ImageRegistries) DeepCopyInto(out *ImageRegistries) { - *out = *in - if in.ImageRegistryCredentials != nil { - in, out := &in.ImageRegistryCredentials, &out.ImageRegistryCredentials - *out = make(ImageRegistryCredentials, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistries. -func (in *ImageRegistries) DeepCopy() *ImageRegistries { - if in == nil { - return nil - } - out := new(ImageRegistries) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in ImageRegistryCredentials) DeepCopyInto(out *ImageRegistryCredentials) { +func (in ImageRegistries) DeepCopyInto(out *ImageRegistries) { { in := &in - *out = make(ImageRegistryCredentials, len(*in)) + *out = make(ImageRegistries, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistryCredentials. -func (in ImageRegistryCredentials) DeepCopy() ImageRegistryCredentials { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistries. +func (in ImageRegistries) DeepCopy() ImageRegistries { if in == nil { return nil } - out := new(ImageRegistryCredentials) + out := new(ImageRegistries) in.DeepCopyInto(out) return *out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ImageRegistryCredentialsResource) DeepCopyInto(out *ImageRegistryCredentialsResource) { +func (in *ImageRegistry) DeepCopyInto(out *ImageRegistry) { *out = *in - if in.Secret != nil { - in, out := &in.Secret, &out.Secret + if in.CredentialsSecret != nil { + in, out := &in.CredentialsSecret, &out.CredentialsSecret *out = new(v1.ObjectReference) **out = **in } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistryCredentialsResource. -func (in *ImageRegistryCredentialsResource) DeepCopy() *ImageRegistryCredentialsResource { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistry. +func (in *ImageRegistry) DeepCopy() *ImageRegistry { if in == nil { return nil } - out := new(ImageRegistryCredentialsResource) + out := new(ImageRegistry) in.DeepCopyInto(out) return out } diff --git a/pkg/handlers/aws/mutation/metapatch_handler_test.go b/pkg/handlers/aws/mutation/metapatch_handler_test.go index 87a288229..a1a2ddc84 100644 --- a/pkg/handlers/aws/mutation/metapatch_handler_test.go +++ b/pkg/handlers/aws/mutation/metapatch_handler_test.go @@ -35,7 +35,6 @@ import ( "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/httpproxy" httpproxytests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/httpproxy/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries" - imageregistrycredentials "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials" imageregistrycredentialstests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository" kubernetesimagerepositorytests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository/tests" @@ -155,7 +154,6 @@ func TestGeneratePatches(t *testing.T) { mgr.GetClient(), clusterconfig.MetaVariableName, imageregistries.VariableName, - imageregistrycredentials.VariableName, ) amitests.TestControlPlaneGeneratePatches( diff --git a/pkg/handlers/docker/mutation/metapatch_handler_test.go b/pkg/handlers/docker/mutation/metapatch_handler_test.go index a74a7cc94..f6334174d 100644 --- a/pkg/handlers/docker/mutation/metapatch_handler_test.go +++ b/pkg/handlers/docker/mutation/metapatch_handler_test.go @@ -25,7 +25,6 @@ import ( "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/httpproxy" httpproxytests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/httpproxy/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries" - imageregistrycredentials "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials" imageregistrycredentialstests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository" kubernetesimagerepositorytests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository/tests" @@ -112,6 +111,5 @@ func TestGeneratePatches(t *testing.T) { mgr.GetClient(), clusterconfig.MetaVariableName, imageregistries.VariableName, - imageregistrycredentials.VariableName, ) } diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go index fe3d5c6c2..0bb923069 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go @@ -45,6 +45,7 @@ type providerConfig struct { URL string Username string Password string + CACert string } func (c providerConfig) isCredentialsEmpty() bool { diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go index a1ac10c1e..a841ea875 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go @@ -26,11 +26,6 @@ import ( "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries" ) -const ( - // VariableName is the external patch variable name. - VariableName = "credentials" -) - type imageRegistriesPatchHandler struct { client ctrlclient.Client @@ -45,7 +40,6 @@ func NewPatch( cl, clusterconfig.MetaVariableName, imageregistries.VariableName, - VariableName, ) } @@ -75,7 +69,7 @@ func (h *imageRegistriesPatchHandler) Mutate( "holderRef", holderRef, ) - imageRegistryCredentials, found, err := variables.Get[v1alpha1.ImageRegistryCredentials]( + imageRegistries, found, err := variables.Get[v1alpha1.ImageRegistries]( vars, h.variableName, h.variableFieldPath..., @@ -89,11 +83,11 @@ func (h *imageRegistriesPatchHandler) Mutate( } // TODO: Add support for multiple registries. - if len(imageRegistryCredentials) > 1 { - return fmt.Errorf("multiple Image Registry Credentials are not supported at this time") + if len(imageRegistries) > 1 { + return fmt.Errorf("multiple Image Registry are not supported at this time") } - credentials := imageRegistryCredentials[0] + imageRegistry := imageRegistries[0] log = log.WithValues( "variableName", @@ -101,19 +95,19 @@ func (h *imageRegistriesPatchHandler) Mutate( "variableFieldPath", h.variableFieldPath, "variableValue", - credentials, + imageRegistry, ) if err := patches.MutateIfApplicable( obj, vars, &holderRef, selectors.ControlPlane(), log, func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { registryWithOptionalCredentials, generateErr := registryWithOptionalCredentialsFromImageRegistryCredentials( - ctx, h.client, credentials, obj, + ctx, h.client, imageRegistry, obj, ) if generateErr != nil { return generateErr } - files, commands, generateErr := generateFilesAndCommands(registryWithOptionalCredentials, obj.GetName()) + files, commands, generateErr := generateFilesAndCommands(registryWithOptionalCredentials, imageRegistry, obj.GetName()) if generateErr != nil { return generateErr } @@ -169,12 +163,12 @@ func (h *imageRegistriesPatchHandler) Mutate( obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log, func(obj *bootstrapv1.KubeadmConfigTemplate) error { registryWithOptionalCredentials, generateErr := registryWithOptionalCredentialsFromImageRegistryCredentials( - ctx, h.client, credentials, obj, + ctx, h.client, imageRegistry, obj, ) if generateErr != nil { return generateErr } - files, commands, generateErr := generateFilesAndCommands(registryWithOptionalCredentials, obj.GetName()) + files, commands, generateErr := generateFilesAndCommands(registryWithOptionalCredentials, imageRegistry, obj.GetName()) if generateErr != nil { return generateErr } @@ -217,23 +211,23 @@ func (h *imageRegistriesPatchHandler) Mutate( func registryWithOptionalCredentialsFromImageRegistryCredentials( ctx context.Context, c ctrlclient.Client, - credentials v1alpha1.ImageRegistryCredentialsResource, + imageRegistry v1alpha1.ImageRegistry, obj ctrlclient.Object, ) (providerConfig, error) { registryWithOptionalCredentials := providerConfig{ - URL: credentials.URL, + URL: imageRegistry.URL, } secret, err := secretForImageRegistryCredentials( ctx, c, - credentials, + imageRegistry, obj.GetNamespace(), ) if err != nil { return providerConfig{}, fmt.Errorf( - "error getting secret %s/%s from Image Registry Credentials variable: %w", + "error getting secret %s/%s from Image Registry variable: %w", obj.GetNamespace(), - credentials.Secret, + imageRegistry.CredentialsSecret, err, ) } @@ -241,6 +235,7 @@ func registryWithOptionalCredentialsFromImageRegistryCredentials( if secret != nil { registryWithOptionalCredentials.Username = string(secret.Data["username"]) registryWithOptionalCredentials.Password = string(secret.Data["password"]) + registryWithOptionalCredentials.CACert = string(secret.Data[secretKeyForMirrorCACert]) } return registryWithOptionalCredentials, nil @@ -248,12 +243,13 @@ func registryWithOptionalCredentialsFromImageRegistryCredentials( func generateFilesAndCommands( registryWithOptionalCredentials providerConfig, + imageRegistry v1alpha1.ImageRegistry, objName string, ) ([]bootstrapv1.File, []string, error) { files, commands, err := templateFilesAndCommandsForInstallKubeletCredentialProviders() if err != nil { return nil, nil, fmt.Errorf( - "error generating insall files and commands for Image Registry Credentials variable: %w", + "error generating install files and commands for Image Registry Credentials variable: %w", err, ) } @@ -271,6 +267,18 @@ func generateFilesAndCommands( files, generateCredentialsSecretFile(registryWithOptionalCredentials, objName)...) + // Generate default registry mirror file + mirrorHostFiles, err := generateDefaultRegistryMirrorFile(registryWithOptionalCredentials) + if err != nil { + return nil, nil, err + } + files = append( + files, + mirrorHostFiles..., + ) + // generate CA certificate file for registry mirror + files = append(files, generateMirrorCACertFile(registryWithOptionalCredentials, imageRegistry)...) + return files, commands, err } @@ -307,20 +315,20 @@ func createSecretIfNeeded( func secretForImageRegistryCredentials( ctx context.Context, c ctrlclient.Reader, - credentials v1alpha1.ImageRegistryCredentialsResource, + registry v1alpha1.ImageRegistry, objectNamespace string, ) (*corev1.Secret, error) { - if credentials.Secret == nil { + if registry.CredentialsSecret == nil { return nil, nil } namespace := objectNamespace - if credentials.Secret.Namespace != "" { - namespace = credentials.Secret.Namespace + if registry.CredentialsSecret.Namespace != "" { + namespace = registry.CredentialsSecret.Namespace } key := ctrlclient.ObjectKey{ - Name: credentials.Secret.Name, + Name: registry.CredentialsSecret.Name, Namespace: namespace, } secret := &corev1.Secret{} diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go b/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go new file mode 100644 index 000000000..10af34cb9 --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go @@ -0,0 +1,71 @@ +package credentials + +import ( + "bytes" + _ "embed" + "fmt" + "text/template" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" +) + +const ( + mirrorCACertPathOnRemote = "/etc/certs/mirror.pem" + defaultRegistryMirrorConfigPathOnRemote = "/etc/containerd/certs.d/_default/hosts.toml" + secretKeyForMirrorCACert = "ca.crt" +) + +//go:embed templates/hosts.toml.gotmpl +var defaultRegistryMirrorPatch []byte + +// Default Mirror for all registries. Use a mirror regardless of the intended registry. +// The upstream registry will be automatically used after all defined mirrors have been tried. +// reference: https://github.com/containerd/containerd/blob/main/docs/hosts.md#setup-default-mirror-for-all-registries +func generateDefaultRegistryMirrorFile(config providerConfig) ([]cabpkv1.File, error) { + t, err := template.New("").Parse(string(defaultRegistryMirrorPatch)) + if err != nil { + return nil, fmt.Errorf("fail to parse go template for registry mirror: %w", err) + } + templateInput := struct { + URL string + CACertPath string + }{ + URL: config.URL, + } + // CA cert is optional for mirror registry. + // i.e. registry is using signed certificates. Insecure registry will not be allowed. + if config.CACert != "" { + templateInput.CACertPath = mirrorCACertPathOnRemote + } + + var b bytes.Buffer + err = t.Execute(&b, templateInput) + if err != nil { + return nil, fmt.Errorf("failed executing template for registry mirror: %w", err) + } + return []cabpkv1.File{ + { + Path: defaultRegistryMirrorConfigPathOnRemote, + Content: b.String(), + Permissions: "0700", + }, + }, nil +} + +func generateMirrorCACertFile(config providerConfig, registry v1alpha1.ImageRegistry) []cabpkv1.File { + if config.CACert == "" { + return nil + } + return []cabpkv1.File{ + { + Path: mirrorCACertPathOnRemote, + ContentFrom: &cabpkv1.FileSource{ + Secret: cabpkv1.SecretFileSource{ + Name: registry.CredentialsSecret.Name, + Key: secretKeyForMirrorCACert, + }, + }, + }, + } +} diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/hosts.toml.gotmpl b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/hosts.toml.gotmpl new file mode 100644 index 000000000..51bc3c5f3 --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/hosts.toml.gotmpl @@ -0,0 +1,5 @@ +[host."{{ .URL }}"] + capabilities = ["pull", "resolve"] + {{- if .CACertPath }} + ca = "{{ .CACertPath }}" + {{- end }} \ No newline at end of file diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go index 5fc1704d8..8da3ad2ab 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go @@ -75,8 +75,8 @@ func TestGeneratePatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistryCredentials{ - v1alpha1.ImageRegistryCredentialsResource{ + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }, }, @@ -130,10 +130,10 @@ func TestGeneratePatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistryCredentials{ - v1alpha1.ImageRegistryCredentialsResource{ + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ URL: "https://my-registry.io", - Secret: &corev1.ObjectReference{ + CredentialsSecret: &corev1.ObjectReference{ Name: validSecretName, }, }, @@ -191,8 +191,8 @@ func TestGeneratePatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistryCredentials{ - v1alpha1.ImageRegistryCredentialsResource{ + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }, }, @@ -246,10 +246,10 @@ func TestGeneratePatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistryCredentials{ - v1alpha1.ImageRegistryCredentialsResource{ + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ URL: "https://my-registry.io", - Secret: &corev1.ObjectReference{ + CredentialsSecret: &corev1.ObjectReference{ Name: validSecretName, }, }, @@ -307,8 +307,8 @@ func TestGeneratePatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistryCredentials{ - v1alpha1.ImageRegistryCredentialsResource{ + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ URL: "https://my-registry.io", }, }, diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go index ef7bff7b5..7e9c81de6 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go @@ -24,11 +24,9 @@ func TestVariableValidation(t *testing.T) { capitest.VariableTestDef{ Name: "without a Secret", Vals: v1alpha1.GenericClusterConfig{ - ImageRegistries: v1alpha1.ImageRegistries{ - ImageRegistryCredentials: []v1alpha1.ImageRegistryCredentialsResource{ - { - URL: "http://a.b.c.example.com", - }, + ImageRegistries: []v1alpha1.ImageRegistry{ + { + URL: "http://a.b.c.example.com", }, }, }, @@ -36,13 +34,11 @@ func TestVariableValidation(t *testing.T) { capitest.VariableTestDef{ Name: "with a Secret", Vals: v1alpha1.GenericClusterConfig{ - ImageRegistries: v1alpha1.ImageRegistries{ - ImageRegistryCredentials: []v1alpha1.ImageRegistryCredentialsResource{ - { - URL: "http://a.b.c.example.com", - Secret: &corev1.ObjectReference{ - Name: "a.b.c.example.com-creds", - }, + ImageRegistries: []v1alpha1.ImageRegistry{ + { + URL: "http://a.b.c.example.com", + CredentialsSecret: &corev1.ObjectReference{ + Name: "a.b.c.example.com-creds", }, }, }, From a1cb78c11dbb3effe9436a2c360ee0e091d4ed62 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Fri, 12 Jan 2024 00:21:22 -0800 Subject: [PATCH 02/12] test: unit tests for registry mirror configuration --- .../imageregistries/credentials/inject.go | 17 ++- .../imageregistries/credentials/mirror.go | 16 ++- .../credentials/mirror_test.go | 129 ++++++++++++++++++ .../credentials/templates/hosts.toml.gotmpl | 2 +- .../credentials/tests/generate_patches.go | 30 +++- 5 files changed, 176 insertions(+), 18 deletions(-) create mode 100644 pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go index a841ea875..7147414e5 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go @@ -107,7 +107,10 @@ func (h *imageRegistriesPatchHandler) Mutate( if generateErr != nil { return generateErr } - files, commands, generateErr := generateFilesAndCommands(registryWithOptionalCredentials, imageRegistry, obj.GetName()) + files, commands, generateErr := generateFilesAndCommands( + registryWithOptionalCredentials, + imageRegistry, + obj.GetName()) if generateErr != nil { return generateErr } @@ -168,7 +171,10 @@ func (h *imageRegistriesPatchHandler) Mutate( if generateErr != nil { return generateErr } - files, commands, generateErr := generateFilesAndCommands(registryWithOptionalCredentials, imageRegistry, obj.GetName()) + files, commands, generateErr := generateFilesAndCommands( + registryWithOptionalCredentials, + imageRegistry, + obj.GetName()) if generateErr != nil { return generateErr } @@ -272,12 +278,11 @@ func generateFilesAndCommands( if err != nil { return nil, nil, err } + files = append(files, mirrorHostFiles...) + // generate CA certificate file for registry mirror files = append( files, - mirrorHostFiles..., - ) - // generate CA certificate file for registry mirror - files = append(files, generateMirrorCACertFile(registryWithOptionalCredentials, imageRegistry)...) + generateMirrorCACertFile(registryWithOptionalCredentials, imageRegistry)...) return files, commands, err } diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go b/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go index 10af34cb9..9443ca290 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go @@ -1,3 +1,6 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + package credentials import ( @@ -6,8 +9,9 @@ import ( "fmt" "text/template" - "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" ) const ( @@ -48,18 +52,22 @@ func generateDefaultRegistryMirrorFile(config providerConfig) ([]cabpkv1.File, e { Path: defaultRegistryMirrorConfigPathOnRemote, Content: b.String(), - Permissions: "0700", + Permissions: "0600", }, }, nil } -func generateMirrorCACertFile(config providerConfig, registry v1alpha1.ImageRegistry) []cabpkv1.File { +func generateMirrorCACertFile( + config providerConfig, + registry v1alpha1.ImageRegistry, +) []cabpkv1.File { if config.CACert == "" { return nil } return []cabpkv1.File{ { - Path: mirrorCACertPathOnRemote, + Path: mirrorCACertPathOnRemote, + Permissions: "0600", ContentFrom: &cabpkv1.FileSource{ Secret: cabpkv1.SecretFileSource{ Name: registry.CredentialsSecret.Name, diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go new file mode 100644 index 000000000..0460d4add --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go @@ -0,0 +1,129 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" +) + +func Test_generateDefaultRegistryMirrorFile(t *testing.T) { + t.Parallel() + tests := []struct { + name string + config providerConfig + want []cabpkv1.File + wantErr error + }{ + { + name: "ECR image registry and no CA certificate", + config: providerConfig{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + want: []cabpkv1.File{ + { + Path: "/etc/containerd/certs.d/_default/hosts.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[host."https://123456789.dkr.ecr.us-east-1.amazonaws.com"] + capabilities = ["pull", "resolve"] +`, + }, + }, + wantErr: nil, + }, + { + name: "image registry with CA certificates", + config: providerConfig{ + URL: "https://myregistry.com", + CACert: "mycacert", + }, + want: []cabpkv1.File{ + { + Path: "/etc/containerd/certs.d/_default/hosts.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[host."https://myregistry.com"] + capabilities = ["pull", "resolve"] + ca = "/etc/certs/mirror.pem" +`, + }, + }, + wantErr: nil, + }, + } + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + file, err := generateDefaultRegistryMirrorFile(tt.config) + assert.ErrorIs(t, err, tt.wantErr) + assert.Equal(t, tt.want, file) + }) + } +} + +func Test_generateMirrorCACertFile(t *testing.T) { + t.Parallel() + tests := []struct { + name string + config providerConfig + registry v1alpha1.ImageRegistry + want []cabpkv1.File + }{ + { + name: "Mirror registry with no CA certificate", + config: providerConfig{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + }, + registry: v1alpha1.ImageRegistry{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + }, + want: nil, + }, + { + name: "Mirror registry with CA certificate", + config: providerConfig{ + URL: "https://myregistry.com", + CACert: "mycacert", + }, + registry: v1alpha1.ImageRegistry{ + URL: "https://myregistry.com", + CredentialsSecret: &v1.ObjectReference{ + Name: "my-registry-credentials-secret", + }, + }, + want: []cabpkv1.File{ + { + Path: "/etc/certs/mirror.pem", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + ContentFrom: &cabpkv1.FileSource{ + Secret: cabpkv1.SecretFileSource{ + Name: "my-registry-credentials-secret", + Key: "ca.crt", + }, + }, + }, + }, + }, + } + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + file := generateMirrorCACertFile(tt.config, tt.registry) + assert.Equal(t, tt.want, file) + }) + } +} diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/hosts.toml.gotmpl b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/hosts.toml.gotmpl index 51bc3c5f3..46237f6e3 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/hosts.toml.gotmpl +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/hosts.toml.gotmpl @@ -2,4 +2,4 @@ capabilities = ["pull", "resolve"] {{- if .CACertPath }} ca = "{{ .CACertPath }}" - {{- end }} \ No newline at end of file + {{- end }} diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go index 8da3ad2ab..46eb674d5 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go @@ -36,12 +36,9 @@ func TestGeneratePatches( // Server side apply does not work with the fake client, hack around it by pre-creating empty Secrets // https://github.com/kubernetes-sigs/controller-runtime/issues/2341 - require.NoError( - t, - fakeClient.Create( - context.Background(), - newTestSecret(validSecretName, request.Namespace), - ), + fakeClient.Create( + context.Background(), + newRegistryCredentialsSecret(validSecretName, request.Namespace), ) require.NoError( t, @@ -98,6 +95,9 @@ func TestGeneratePatches( gomega.HaveKeyWithValue( "path", "/etc/kubernetes/dynamic-credential-provider-config.yaml", ), + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), ), }, { @@ -159,6 +159,12 @@ func TestGeneratePatches( gomega.HaveKeyWithValue( "path", "/etc/kubernetes/static-image-credentials.json", ), + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/certs/mirror.pem", + ), ), }, { @@ -222,6 +228,9 @@ func TestGeneratePatches( gomega.HaveKeyWithValue( "path", "/etc/kubernetes/dynamic-credential-provider-config.yaml", ), + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), ), }, { @@ -283,6 +292,12 @@ func TestGeneratePatches( gomega.HaveKeyWithValue( "path", "/etc/kubernetes/static-image-credentials.json", ), + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/certs/mirror.pem", + ), ), }, { @@ -321,10 +336,11 @@ func TestGeneratePatches( ) } -func newTestSecret(name, namespace string) *corev1.Secret { +func newRegistryCredentialsSecret(name, namespace string) *corev1.Secret { secretData := map[string][]byte{ "username": []byte("myuser"), "password": []byte("mypassword"), + "ca.crt": []byte("myCACert"), } return &corev1.Secret{ TypeMeta: metav1.TypeMeta{ From 287b9acce75bef0948d1ddd11d6119057891d8ea Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Fri, 12 Jan 2024 14:59:42 -0800 Subject: [PATCH 03/12] fix: set mirror for image registry --- api/v1alpha1/clusterconfig_types.go | 105 +++++++-- api/v1alpha1/zz_generated.deepcopy.go | 53 ++++- .../credential_provider_config_files.go | 1 - .../imageregistries/credentials/inject.go | 36 ++- .../imageregistries/credentials/mirror.go | 75 +++++- .../credentials/mirror_test.go | 18 +- .../credentials/tests/generate_patches.go | 222 +++++++++++++++--- .../credentials/variables_test.go | 25 +- 8 files changed, 450 insertions(+), 85 deletions(-) diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index 6eb3beaca..c92bb203c 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -237,51 +237,66 @@ func (ExtraAPIServerCertSANs) VariableSchema() clusterv1.VariableSchema { } } -type ImageRegistries []ImageRegistry +type ImageCredentials struct { + // The Secret containing the registry credentials and CA certificate + // The Secret should have keys 'username', 'password' and 'caCert' + // This credentials Secret is not required for some registries, e.g. ECR. + // +optional + SecretRef *corev1.ObjectReference `json:"secretRef,omitempty"` +} -func (ImageRegistries) VariableSchema() clusterv1.VariableSchema { - resourceSchema := ImageRegistry{}.VariableSchema().OpenAPIV3Schema +func (ImageCredentials) VariableSchema() clusterv1.VariableSchema { return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ - Description: "Configuration for image registries.", - Type: "array", - Items: &resourceSchema, + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "secretRef": { + Description: "The Secret containing the registry credentials. " + + "The Secret should have keys 'username', 'password'. " + + "This credentials Secret is not required for some registries, e.g. ECR.", + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "name": { + Description: "The name of the Secret containing the registry credentials.", + Type: "string", + }, + "namespace": { + Description: "The namespace of the Secret containing the registry credentials. " + + "Defaults to the namespace of the KubeadmControlPlaneTemplate and KubeadmConfigTemplate" + + " that reference this variable.", + Type: "string", + }, + }, + }, + }, }, } } -type ImageRegistry struct { - // Registry URL. - URL string `json:"url"` - - // The Secret containing the registry credentials and CA certificate - // The Secret should have keys 'username', 'password' and 'caCert' - // This credentials Secret is not required for some registries, e.g. ECR. +type RegistryMirror struct { + // The secret containing CA certificate for the registry mirror. + // The secret should have 'ca.crt' key // +optional - CredentialsSecret *corev1.ObjectReference `json:"secretRef,omitempty"` + SecretRef *corev1.ObjectReference `json:"secretRef,omitempty"` } -func (ImageRegistry) VariableSchema() clusterv1.VariableSchema { +func (RegistryMirror) VariableSchema() clusterv1.VariableSchema { return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ - "url": { - Description: "Registry URL.", - Type: "string", - }, "secretRef": { - Description: "The Secret containing the registry credentials and CA certificates " + - "The Secret should have keys 'username', 'password' and 'ca.crt' " + - "This credentials Secret is not required for some registries, e.g. ECR.", + Description: "The Secret containing the registry CA certificate. " + + "The Secret should have keys 'ca.crt'. " + + "This credentials Secret is not required for public registries.", Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ "name": { - Description: "The name of the Secret containing the registry credentials.", + Description: "The name of the Secret containing the registry CA certificate.", Type: "string", }, "namespace": { - Description: "The namespace of the Secret containing the registry credentials. " + + Description: "The namespace of the Secret containing the registry CA certificate. " + "Defaults to the namespace of the KubeadmControlPlaneTemplate and KubeadmConfigTemplate" + " that reference this variable.", Type: "string", @@ -289,11 +304,53 @@ func (ImageRegistry) VariableSchema() clusterv1.VariableSchema { }, }, }, + }, + } +} + +type ImageRegistry struct { + // Registry URL. + URL string `json:"url"` + + // Credentials for the image registry + // +optional + Credentials *ImageCredentials `json:"credentials,omitempty"` + + // Use this registry as a mirror + // +optional + Mirror *RegistryMirror `json:"mirror,omitempty"` +} + +func (ImageRegistry) VariableSchema() clusterv1.VariableSchema { + return clusterv1.VariableSchema{ + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "url": { + Description: "Registry URL.", + Type: "string", + }, + "credentials": ImageCredentials{}.VariableSchema().OpenAPIV3Schema, + "mirror": RegistryMirror{}.VariableSchema().OpenAPIV3Schema, + }, Required: []string{"url"}, }, } } +type ImageRegistries []ImageRegistry + +func (ImageRegistries) VariableSchema() clusterv1.VariableSchema { + resourceSchema := ImageRegistry{}.VariableSchema().OpenAPIV3Schema + return clusterv1.VariableSchema{ + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Description: "Configuration for image registries.", + Type: "array", + Items: &resourceSchema, + }, + } +} + func init() { SchemeBuilder.Register(&ClusterConfig{}) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 8e6dff6a0..c9fa1330a 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -509,6 +509,26 @@ func (in *Image) DeepCopy() *Image { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImageCredentials) DeepCopyInto(out *ImageCredentials) { + *out = *in + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(v1.ObjectReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageCredentials. +func (in *ImageCredentials) DeepCopy() *ImageCredentials { + if in == nil { + return nil + } + out := new(ImageCredentials) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in ImageRegistries) DeepCopyInto(out *ImageRegistries) { { @@ -533,10 +553,15 @@ func (in ImageRegistries) DeepCopy() ImageRegistries { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ImageRegistry) DeepCopyInto(out *ImageRegistry) { *out = *in - if in.CredentialsSecret != nil { - in, out := &in.CredentialsSecret, &out.CredentialsSecret - *out = new(v1.ObjectReference) - **out = **in + if in.Credentials != nil { + in, out := &in.Credentials, &out.Credentials + *out = new(ImageCredentials) + (*in).DeepCopyInto(*out) + } + if in.Mirror != nil { + in, out := &in.Mirror, &out.Mirror + *out = new(RegistryMirror) + (*in).DeepCopyInto(*out) } } @@ -645,6 +670,26 @@ func (in *ObjectMeta) DeepCopy() *ObjectMeta { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RegistryMirror) DeepCopyInto(out *RegistryMirror) { + *out = *in + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(v1.ObjectReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RegistryMirror. +func (in *RegistryMirror) DeepCopy() *RegistryMirror { + if in == nil { + return nil + } + out := new(RegistryMirror) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecurityGroup) DeepCopyInto(out *SecurityGroup) { *out = *in diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go index 0bb923069..fe3d5c6c2 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go @@ -45,7 +45,6 @@ type providerConfig struct { URL string Username string Password string - CACert string } func (c providerConfig) isCredentialsEmpty() bool { diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go index 7147414e5..751ac104f 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go @@ -107,8 +107,18 @@ func (h *imageRegistriesPatchHandler) Mutate( if generateErr != nil { return generateErr } + mirrorConfig, err := mirrorFromImageRegistry( + ctx, + h.client, + imageRegistry, + obj, + ) + if err != nil { + return err + } files, commands, generateErr := generateFilesAndCommands( registryWithOptionalCredentials, + mirrorConfig, imageRegistry, obj.GetName()) if generateErr != nil { @@ -171,8 +181,18 @@ func (h *imageRegistriesPatchHandler) Mutate( if generateErr != nil { return generateErr } + mirrorConfig, err := mirrorFromImageRegistry( + ctx, + h.client, + imageRegistry, + obj, + ) + if err != nil { + return err + } files, commands, generateErr := generateFilesAndCommands( registryWithOptionalCredentials, + mirrorConfig, imageRegistry, obj.GetName()) if generateErr != nil { @@ -233,7 +253,7 @@ func registryWithOptionalCredentialsFromImageRegistryCredentials( return providerConfig{}, fmt.Errorf( "error getting secret %s/%s from Image Registry variable: %w", obj.GetNamespace(), - imageRegistry.CredentialsSecret, + imageRegistry.Credentials.SecretRef.Name, err, ) } @@ -241,7 +261,6 @@ func registryWithOptionalCredentialsFromImageRegistryCredentials( if secret != nil { registryWithOptionalCredentials.Username = string(secret.Data["username"]) registryWithOptionalCredentials.Password = string(secret.Data["password"]) - registryWithOptionalCredentials.CACert = string(secret.Data[secretKeyForMirrorCACert]) } return registryWithOptionalCredentials, nil @@ -249,6 +268,7 @@ func registryWithOptionalCredentialsFromImageRegistryCredentials( func generateFilesAndCommands( registryWithOptionalCredentials providerConfig, + mirrorConfig mirrorConfig, imageRegistry v1alpha1.ImageRegistry, objName string, ) ([]bootstrapv1.File, []string, error) { @@ -274,7 +294,7 @@ func generateFilesAndCommands( generateCredentialsSecretFile(registryWithOptionalCredentials, objName)...) // Generate default registry mirror file - mirrorHostFiles, err := generateDefaultRegistryMirrorFile(registryWithOptionalCredentials) + mirrorHostFiles, err := generateDefaultRegistryMirrorFile(mirrorConfig) if err != nil { return nil, nil, err } @@ -282,7 +302,7 @@ func generateFilesAndCommands( // generate CA certificate file for registry mirror files = append( files, - generateMirrorCACertFile(registryWithOptionalCredentials, imageRegistry)...) + generateMirrorCACertFile(mirrorConfig, imageRegistry)...) return files, commands, err } @@ -323,17 +343,17 @@ func secretForImageRegistryCredentials( registry v1alpha1.ImageRegistry, objectNamespace string, ) (*corev1.Secret, error) { - if registry.CredentialsSecret == nil { + if registry.Credentials == nil || registry.Credentials.SecretRef == nil { return nil, nil } namespace := objectNamespace - if registry.CredentialsSecret.Namespace != "" { - namespace = registry.CredentialsSecret.Namespace + if registry.Credentials.SecretRef.Namespace != "" { + namespace = registry.Credentials.SecretRef.Namespace } key := ctrlclient.ObjectKey{ - Name: registry.CredentialsSecret.Name, + Name: registry.Credentials.SecretRef.Name, Namespace: namespace, } secret := &corev1.Secret{} diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go b/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go index 9443ca290..070440986 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go @@ -5,11 +5,14 @@ package credentials import ( "bytes" + "context" _ "embed" "fmt" "text/template" + corev1 "k8s.io/api/core/v1" cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" ) @@ -23,10 +26,72 @@ const ( //go:embed templates/hosts.toml.gotmpl var defaultRegistryMirrorPatch []byte +type mirrorConfig struct { + URL string + CACert string +} + +func mirrorFromImageRegistry( + ctx context.Context, + c ctrlclient.Client, + imageRegistry v1alpha1.ImageRegistry, + obj ctrlclient.Object, +) (mirrorConfig, error) { + mirrorWithOptionalCACert := mirrorConfig{ + URL: imageRegistry.URL, + } + secret, err := secretForMirrorCACert( + ctx, + c, + imageRegistry, + obj.GetNamespace(), + ) + if err != nil { + return mirrorConfig{}, fmt.Errorf( + "error getting secret %s/%s from Image Registry variable: %w", + obj.GetNamespace(), + imageRegistry.Mirror.SecretRef.Name, + err, + ) + } + + if secret != nil { + mirrorWithOptionalCACert.CACert = string(secret.Data[secretKeyForMirrorCACert]) + } + + return mirrorWithOptionalCACert, nil +} + +// secretForMirrorCACert returns the Secret for the given mirror's CA certificate. +// Returns nil if the secret field is empty. +func secretForMirrorCACert( + ctx context.Context, + c ctrlclient.Reader, + registry v1alpha1.ImageRegistry, + objectNamespace string, +) (*corev1.Secret, error) { + if registry.Mirror == nil || registry.Mirror.SecretRef == nil { + return nil, nil + } + + namespace := objectNamespace + if registry.Mirror.SecretRef.Namespace != "" { + namespace = registry.Mirror.SecretRef.Namespace + } + + key := ctrlclient.ObjectKey{ + Name: registry.Mirror.SecretRef.Name, + Namespace: namespace, + } + secret := &corev1.Secret{} + err := c.Get(ctx, key, secret) + return secret, err +} + // Default Mirror for all registries. Use a mirror regardless of the intended registry. // The upstream registry will be automatically used after all defined mirrors have been tried. // reference: https://github.com/containerd/containerd/blob/main/docs/hosts.md#setup-default-mirror-for-all-registries -func generateDefaultRegistryMirrorFile(config providerConfig) ([]cabpkv1.File, error) { +func generateDefaultRegistryMirrorFile(mirror mirrorConfig) ([]cabpkv1.File, error) { t, err := template.New("").Parse(string(defaultRegistryMirrorPatch)) if err != nil { return nil, fmt.Errorf("fail to parse go template for registry mirror: %w", err) @@ -35,11 +100,11 @@ func generateDefaultRegistryMirrorFile(config providerConfig) ([]cabpkv1.File, e URL string CACertPath string }{ - URL: config.URL, + URL: mirror.URL, } // CA cert is optional for mirror registry. // i.e. registry is using signed certificates. Insecure registry will not be allowed. - if config.CACert != "" { + if mirror.CACert != "" { templateInput.CACertPath = mirrorCACertPathOnRemote } @@ -58,7 +123,7 @@ func generateDefaultRegistryMirrorFile(config providerConfig) ([]cabpkv1.File, e } func generateMirrorCACertFile( - config providerConfig, + config mirrorConfig, registry v1alpha1.ImageRegistry, ) []cabpkv1.File { if config.CACert == "" { @@ -70,7 +135,7 @@ func generateMirrorCACertFile( Permissions: "0600", ContentFrom: &cabpkv1.FileSource{ Secret: cabpkv1.SecretFileSource{ - Name: registry.CredentialsSecret.Name, + Name: registry.Mirror.SecretRef.Name, Key: secretKeyForMirrorCACert, }, }, diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go index 0460d4add..03ef70bc8 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go @@ -17,13 +17,13 @@ func Test_generateDefaultRegistryMirrorFile(t *testing.T) { t.Parallel() tests := []struct { name string - config providerConfig + config mirrorConfig want []cabpkv1.File wantErr error }{ { name: "ECR image registry and no CA certificate", - config: providerConfig{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + config: mirrorConfig{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, want: []cabpkv1.File{ { Path: "/etc/containerd/certs.d/_default/hosts.toml", @@ -40,7 +40,7 @@ func Test_generateDefaultRegistryMirrorFile(t *testing.T) { }, { name: "image registry with CA certificates", - config: providerConfig{ + config: mirrorConfig{ URL: "https://myregistry.com", CACert: "mycacert", }, @@ -75,13 +75,13 @@ func Test_generateMirrorCACertFile(t *testing.T) { t.Parallel() tests := []struct { name string - config providerConfig + config mirrorConfig registry v1alpha1.ImageRegistry want []cabpkv1.File }{ { name: "Mirror registry with no CA certificate", - config: providerConfig{ + config: mirrorConfig{ URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }, registry: v1alpha1.ImageRegistry{ @@ -91,14 +91,16 @@ func Test_generateMirrorCACertFile(t *testing.T) { }, { name: "Mirror registry with CA certificate", - config: providerConfig{ + config: mirrorConfig{ URL: "https://myregistry.com", CACert: "mycacert", }, registry: v1alpha1.ImageRegistry{ URL: "https://myregistry.com", - CredentialsSecret: &v1.ObjectReference{ - Name: "my-registry-credentials-secret", + Mirror: &v1alpha1.RegistryMirror{ + SecretRef: &v1.ObjectReference{ + Name: "my-registry-credentials-secret", + }, }, }, want: []cabpkv1.File{ diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go index 46eb674d5..db2984515 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go @@ -22,7 +22,8 @@ import ( ) const ( - validSecretName = "myregistry-credentials" + validSecretName = "myregistry-credentials" + validMirrorSecretName = "myregistry-mirror-cacert" ) func TestGeneratePatches( @@ -40,14 +41,17 @@ func TestGeneratePatches( context.Background(), newRegistryCredentialsSecret(validSecretName, request.Namespace), ) - require.NoError( - t, - fakeClient.Create( - context.Background(), - newEmptySecret( - request.KubeadmControlPlaneTemplateRequestObjectName+"-registry-config", - request.Namespace, - ), + + fakeClient.Create( + context.Background(), + newMirrorSecret(validMirrorSecretName, request.Namespace), + ) + + fakeClient.Create( + context.Background(), + newEmptySecret( + request.KubeadmControlPlaneTemplateRequestObjectName+"-registry-config", + request.Namespace, ), ) require.NoError( @@ -95,9 +99,6 @@ func TestGeneratePatches( gomega.HaveKeyWithValue( "path", "/etc/kubernetes/dynamic-credential-provider-config.yaml", ), - gomega.HaveKeyWithValue( - "path", "/etc/containerd/certs.d/_default/hosts.toml", - ), ), }, { @@ -133,8 +134,10 @@ func TestGeneratePatches( v1alpha1.ImageRegistries{ v1alpha1.ImageRegistry{ URL: "https://my-registry.io", - CredentialsSecret: &corev1.ObjectReference{ - Name: validSecretName, + Credentials: &v1alpha1.ImageCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: validSecretName, + }, }, }, }, @@ -159,12 +162,6 @@ func TestGeneratePatches( gomega.HaveKeyWithValue( "path", "/etc/kubernetes/static-image-credentials.json", ), - gomega.HaveKeyWithValue( - "path", "/etc/containerd/certs.d/_default/hosts.toml", - ), - gomega.HaveKeyWithValue( - "path", "/etc/certs/mirror.pem", - ), ), }, { @@ -192,6 +189,72 @@ func TestGeneratePatches( }, }, }, + capitest.PatchTestDef{ + Name: "files added in KubeadmControlPlaneTemplate for registry with mirror without CA Certificate", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + Mirror: &v1alpha1.RegistryMirror{}, + }, + }, + variablePath..., + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: gomega.ContainElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + ), + }, + }, + }, + capitest.PatchTestDef{ + Name: "files added in KubeadmControlPlaneTemplate for registry with mirror with CA Certificate", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ + URL: "https://mirror-registry.com", + Credentials: &v1alpha1.ImageCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: validSecretName, + }, + }, + Mirror: &v1alpha1.RegistryMirror{ + SecretRef: &corev1.ObjectReference{ + Name: validMirrorSecretName, + }, + }, + }, + }, + variablePath..., + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: gomega.ContainElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/certs/mirror.pem", + ), + ), + }, + }, + }, capitest.PatchTestDef{ Name: "files added in KubeadmConfigTemplate for ECR without a Secret", Vars: []runtimehooksv1.Variable{ @@ -228,9 +291,6 @@ func TestGeneratePatches( gomega.HaveKeyWithValue( "path", "/etc/kubernetes/dynamic-credential-provider-config.yaml", ), - gomega.HaveKeyWithValue( - "path", "/etc/containerd/certs.d/_default/hosts.toml", - ), ), }, { @@ -258,8 +318,10 @@ func TestGeneratePatches( v1alpha1.ImageRegistries{ v1alpha1.ImageRegistry{ URL: "https://my-registry.io", - CredentialsSecret: &corev1.ObjectReference{ - Name: validSecretName, + Credentials: &v1alpha1.ImageCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: validSecretName, + }, }, }, }, @@ -292,12 +354,6 @@ func TestGeneratePatches( gomega.HaveKeyWithValue( "path", "/etc/kubernetes/static-image-credentials.json", ), - gomega.HaveKeyWithValue( - "path", "/etc/containerd/certs.d/_default/hosts.toml", - ), - gomega.HaveKeyWithValue( - "path", "/etc/certs/mirror.pem", - ), ), }, { @@ -317,6 +373,93 @@ func TestGeneratePatches( }, }, }, + capitest.PatchTestDef{ + Name: "files added in KubeadmConfigTemplate for registry mirror with no CA certificate", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ + URL: "https://my-registry.io", + Credentials: &v1alpha1.ImageCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: validSecretName, + }, + }, + Mirror: &v1alpha1.RegistryMirror{}, + }, + }, + variablePath..., + ), + capitest.VariableWithValue( + "builtin", + map[string]any{ + "machineDeployment": map[string]any{ + "class": names.SimpleNameGenerator.GenerateName("worker-"), + }, + }, + ), + }, + RequestItem: request.NewKubeadmConfigTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/files", + ValueMatcher: gomega.ContainElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + ), + }, + }, + }, + capitest.PatchTestDef{ + Name: "files added in KubeadmConfigTemplate for registry mirror with secret for CA certificate", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ + URL: "https://my-registry.io", + Credentials: &v1alpha1.ImageCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: validSecretName, + }, + }, + Mirror: &v1alpha1.RegistryMirror{ + SecretRef: &corev1.ObjectReference{ + Name: validMirrorSecretName, + }, + }, + }, + }, + variablePath..., + ), + capitest.VariableWithValue( + "builtin", + map[string]any{ + "machineDeployment": map[string]any{ + "class": names.SimpleNameGenerator.GenerateName("worker-"), + }, + }, + ), + }, + RequestItem: request.NewKubeadmConfigTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/files", + ValueMatcher: gomega.ContainElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/certs/mirror.pem", + ), + ), + }, + }, + }, capitest.PatchTestDef{ Name: "error for a registry with no credentials", Vars: []runtimehooksv1.Variable{ @@ -355,6 +498,23 @@ func newRegistryCredentialsSecret(name, namespace string) *corev1.Secret { Type: corev1.SecretTypeOpaque, } } +func newMirrorSecret(name, namespace string) *corev1.Secret { + secretData := map[string][]byte{ + "ca.crt": []byte("myCACert"), + } + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: secretData, + Type: corev1.SecretTypeOpaque, + } +} func newEmptySecret(name, namespace string) *corev1.Secret { return &corev1.Secret{ diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go index 7e9c81de6..16562b92f 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go @@ -22,7 +22,7 @@ func TestVariableValidation(t *testing.T) { false, clusterconfig.NewVariable, capitest.VariableTestDef{ - Name: "without a Secret", + Name: "without a credentials secret", Vals: v1alpha1.GenericClusterConfig{ ImageRegistries: []v1alpha1.ImageRegistry{ { @@ -32,13 +32,30 @@ func TestVariableValidation(t *testing.T) { }, }, capitest.VariableTestDef{ - Name: "with a Secret", + Name: "with a credentials secret", Vals: v1alpha1.GenericClusterConfig{ ImageRegistries: []v1alpha1.ImageRegistry{ { URL: "http://a.b.c.example.com", - CredentialsSecret: &corev1.ObjectReference{ - Name: "a.b.c.example.com-creds", + Credentials: &v1alpha1.ImageCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: "a.b.c.example.com-creds", + }, + }, + }, + }, + }, + }, + capitest.VariableTestDef{ + Name: "with a mirror secret", + Vals: v1alpha1.GenericClusterConfig{ + ImageRegistries: []v1alpha1.ImageRegistry{ + { + URL: "http://a.b.c.example.com", + Mirror: &v1alpha1.RegistryMirror{ + SecretRef: &corev1.ObjectReference{ + Name: "a.b.c.example.com-creds", + }, }, }, }, From b93b568d9e03ed6ebd0db1afd78a0175c2dc7533 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Tue, 16 Jan 2024 13:01:26 -0800 Subject: [PATCH 04/12] test: mirror credentials config tests in credential provider config --- .../credential_provider_config_files.go | 6 +- .../credential_provider_config_files_test.go | 70 ++++++++++++++++++- .../imageregistries/credentials/inject.go | 3 +- .../imageregistries/credentials/mirror.go | 23 ++++-- .../credentials/mirror_test.go | 12 ++-- ...mic-credential-provider-config.yaml.gotmpl | 7 ++ .../credentials/tests/generate_patches.go | 33 +++++---- 7 files changed, 127 insertions(+), 27 deletions(-) diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go index fe3d5c6c2..45c9b101d 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go @@ -52,7 +52,7 @@ func (c providerConfig) isCredentialsEmpty() bool { c.Password == "" } -func templateFilesForImageCredentialProviderConfigs(config providerConfig) ([]cabpkv1.File, error) { +func templateFilesForImageCredentialProviderConfigs(config providerConfig, mirror *mirrorConfig) ([]cabpkv1.File, error) { var files []cabpkv1.File kubeletCredentialProviderConfigFile, err := templateKubeletCredentialProviderConfig() @@ -65,6 +65,7 @@ func templateFilesForImageCredentialProviderConfigs(config providerConfig) ([]ca kubeletDynamicCredentialProviderConfigFile, err := templateDynamicCredentialProviderConfig( config, + mirror, ) if err != nil { return nil, err @@ -100,6 +101,7 @@ func templateKubeletCredentialProviderConfig() (*cabpkv1.File, error) { func templateDynamicCredentialProviderConfig( config providerConfig, + mirror *mirrorConfig, ) (*cabpkv1.File, error) { registryURL, err := url.ParseRequestURI(config.URL) if err != nil { @@ -137,11 +139,13 @@ func templateDynamicCredentialProviderConfig( ProviderBinary string ProviderArgs []string ProviderAPIVersion string + Mirror *mirrorConfig }{ RegistryHost: registryHostWithPath, ProviderBinary: providerBinary, ProviderArgs: providerArgs, ProviderAPIVersion: providerAPIVersion, + Mirror: mirror, } return fileFromTemplate(t, templateInput, kubeletDynamicCredentialProviderConfigOnRemote) diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go index e8ce8dd87..b3ce641bc 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go @@ -99,6 +99,7 @@ func Test_templateDynamicCredentialProviderConfig(t *testing.T) { tests := []struct { name string credentials providerConfig + mirror *mirrorConfig want *cabpkv1.File wantErr error }{ @@ -189,6 +190,73 @@ credentialProviders: `, }, }, + + { + name: "ECR image registry used as mirror", + credentials: providerConfig{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + mirror: &mirrorConfig{}, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 +kind: DynamicCredentialProviderConfig +mirror: + endpoint: "123456789.dkr.ecr.us-east-1.amazonaws.com" + credentialsStrategy: "MirrorCredentialsOnly" +credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ +credentialProviders: + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: CredentialProviderConfig + providers: + - name: ecr-credential-provider + args: + - get-credentials + matchImages: + - "123456789.dkr.ecr.us-east-1.amazonaws.com" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 +`, + }, + }, + { + name: "image registry with static credentials used as mirror", + credentials: providerConfig{ + URL: "https://myregistry.com", + Username: "myuser", + Password: "mypassword", + }, + mirror: &mirrorConfig{ + CACert: "my-ca-cert", + }, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 +kind: DynamicCredentialProviderConfig +mirror: + endpoint: "myregistry.com" + credentialsStrategy: "MirrorCredentialsOnly" +credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ +credentialProviders: + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: CredentialProviderConfig + providers: + - name: static-credential-provider + args: + - /etc/kubernetes/static-image-credentials.json + matchImages: + - "myregistry.com" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1beta1 +`, + }, + }, { name: "error for a registry with no credentials", credentials: providerConfig{ @@ -201,7 +269,7 @@ credentialProviders: tt := tests[idx] t.Run(tt.name, func(t *testing.T) { t.Parallel() - file, err := templateDynamicCredentialProviderConfig(tt.credentials) + file, err := templateDynamicCredentialProviderConfig(tt.credentials, tt.mirror) assert.ErrorIs(t, err, tt.wantErr) assert.Equal(t, tt.want, file) }) diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go index 751ac104f..6c8cff9f9 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go @@ -268,7 +268,7 @@ func registryWithOptionalCredentialsFromImageRegistryCredentials( func generateFilesAndCommands( registryWithOptionalCredentials providerConfig, - mirrorConfig mirrorConfig, + mirrorConfig *mirrorConfig, imageRegistry v1alpha1.ImageRegistry, objName string, ) ([]bootstrapv1.File, []string, error) { @@ -281,6 +281,7 @@ func generateFilesAndCommands( } imageCredentialProviderConfigFiles, err := templateFilesForImageCredentialProviderConfigs( registryWithOptionalCredentials, + mirrorConfig, ) if err != nil { return nil, nil, fmt.Errorf( diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go b/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go index 070440986..0fe4c6e62 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go @@ -36,8 +36,16 @@ func mirrorFromImageRegistry( c ctrlclient.Client, imageRegistry v1alpha1.ImageRegistry, obj ctrlclient.Object, -) (mirrorConfig, error) { - mirrorWithOptionalCACert := mirrorConfig{ +) (*mirrorConfig, error) { + // using the registry as a mirror is supported by including empty mirror object or + // mirror with CA certificate to the registry variable. + // ex. + // - url: https://my-registry.com + // mirror: {} + if imageRegistry.Mirror == nil { + return nil, nil + } + mirrorWithOptionalCACert := &mirrorConfig{ URL: imageRegistry.URL, } secret, err := secretForMirrorCACert( @@ -47,7 +55,7 @@ func mirrorFromImageRegistry( obj.GetNamespace(), ) if err != nil { - return mirrorConfig{}, fmt.Errorf( + return &mirrorConfig{}, fmt.Errorf( "error getting secret %s/%s from Image Registry variable: %w", obj.GetNamespace(), imageRegistry.Mirror.SecretRef.Name, @@ -91,7 +99,10 @@ func secretForMirrorCACert( // Default Mirror for all registries. Use a mirror regardless of the intended registry. // The upstream registry will be automatically used after all defined mirrors have been tried. // reference: https://github.com/containerd/containerd/blob/main/docs/hosts.md#setup-default-mirror-for-all-registries -func generateDefaultRegistryMirrorFile(mirror mirrorConfig) ([]cabpkv1.File, error) { +func generateDefaultRegistryMirrorFile(mirror *mirrorConfig) ([]cabpkv1.File, error) { + if mirror == nil { + return nil, nil + } t, err := template.New("").Parse(string(defaultRegistryMirrorPatch)) if err != nil { return nil, fmt.Errorf("fail to parse go template for registry mirror: %w", err) @@ -123,10 +134,10 @@ func generateDefaultRegistryMirrorFile(mirror mirrorConfig) ([]cabpkv1.File, err } func generateMirrorCACertFile( - config mirrorConfig, + config *mirrorConfig, registry v1alpha1.ImageRegistry, ) []cabpkv1.File { - if config.CACert == "" { + if config == nil || config.CACert == "" { return nil } return []cabpkv1.File{ diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go index 03ef70bc8..82440b04d 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go @@ -17,13 +17,13 @@ func Test_generateDefaultRegistryMirrorFile(t *testing.T) { t.Parallel() tests := []struct { name string - config mirrorConfig + config *mirrorConfig want []cabpkv1.File wantErr error }{ { name: "ECR image registry and no CA certificate", - config: mirrorConfig{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + config: &mirrorConfig{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, want: []cabpkv1.File{ { Path: "/etc/containerd/certs.d/_default/hosts.toml", @@ -40,7 +40,7 @@ func Test_generateDefaultRegistryMirrorFile(t *testing.T) { }, { name: "image registry with CA certificates", - config: mirrorConfig{ + config: &mirrorConfig{ URL: "https://myregistry.com", CACert: "mycacert", }, @@ -75,13 +75,13 @@ func Test_generateMirrorCACertFile(t *testing.T) { t.Parallel() tests := []struct { name string - config mirrorConfig + config *mirrorConfig registry v1alpha1.ImageRegistry want []cabpkv1.File }{ { name: "Mirror registry with no CA certificate", - config: mirrorConfig{ + config: &mirrorConfig{ URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }, registry: v1alpha1.ImageRegistry{ @@ -91,7 +91,7 @@ func Test_generateMirrorCACertFile(t *testing.T) { }, { name: "Mirror registry with CA certificate", - config: mirrorConfig{ + config: &mirrorConfig{ URL: "https://myregistry.com", CACert: "mycacert", }, diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl index 80156c219..196203996 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl @@ -1,5 +1,12 @@ apiVersion: credentialprovider.d2iq.com/v1alpha1 kind: DynamicCredentialProviderConfig +{{- if .Mirror }} +mirror: + {{- with .RegistryHost }} + endpoint: {{ printf "%q" . }} + {{- end }} + credentialsStrategy: "MirrorCredentialsOnly" +{{- end }} credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ credentialProviders: apiVersion: kubelet.config.k8s.io/v1 diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go index db2984515..6b4eeb41d 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go @@ -37,21 +37,30 @@ func TestGeneratePatches( // Server side apply does not work with the fake client, hack around it by pre-creating empty Secrets // https://github.com/kubernetes-sigs/controller-runtime/issues/2341 - fakeClient.Create( - context.Background(), - newRegistryCredentialsSecret(validSecretName, request.Namespace), + require.NoError( + t, + fakeClient.Create( + context.Background(), + newRegistryCredentialsSecret(validSecretName, request.Namespace), + ), ) - fakeClient.Create( - context.Background(), - newMirrorSecret(validMirrorSecretName, request.Namespace), + require.NoError( + t, + fakeClient.Create( + context.Background(), + newMirrorSecret(validMirrorSecretName, request.Namespace), + ), ) - fakeClient.Create( - context.Background(), - newEmptySecret( - request.KubeadmControlPlaneTemplateRequestObjectName+"-registry-config", - request.Namespace, + require.NoError( + t, + fakeClient.Create( + context.Background(), + newEmptySecret( + request.KubeadmControlPlaneTemplateRequestObjectName+"-registry-config", + request.Namespace, + ), ), ) require.NoError( @@ -420,7 +429,7 @@ func TestGeneratePatches( variableName, v1alpha1.ImageRegistries{ v1alpha1.ImageRegistry{ - URL: "https://my-registry.io", + URL: "https://mirror-registry.com", Credentials: &v1alpha1.ImageCredentials{ SecretRef: &corev1.ObjectReference{ Name: validSecretName, From 92094dfabb8579327eb7b9da51135872f2f19893 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Tue, 16 Jan 2024 13:01:41 -0800 Subject: [PATCH 05/12] docs: update docs with mirror information --- .../customization/generic/image-registries.md | 64 ++++++++++++++++++- .../credential_provider_config_files.go | 5 +- .../credentials/tests/generate_patches.go | 2 +- 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/docs/content/customization/generic/image-registries.md b/docs/content/customization/generic/image-registries.md index ed7cecd70..1013fb574 100644 --- a/docs/content/customization/generic/image-registries.md +++ b/docs/content/customization/generic/image-registries.md @@ -33,11 +33,71 @@ spec: - name: clusterConfig value: imageRegistries: - credentials: - - url: https://my-registry.io + - url: https://my-registry.io + credentials: secretRef: name: my-registry-credentials ``` Applying this configuration will result in new files and preKubeadmCommands on the `KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate`. + +To use a image registry as mirror with CA certificate, specify the following configuration: + +If your registry mirror requires self signed CA certifate, create a Kubernetes Secret with keys for `ca.crt`: + +```shell +kubectl create secret generic my-mirror-ca-cert-secret \ + --from-file=ca.crt=registry-ca.crt +``` + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: clusterConfig + value: + imageRegistries: + - url: https://my-registry.io + credentials: + secretRef: + name: my-registry-credentials + mirror: + secretRef: + name: my-mirror-ca-cert-secret +``` + +Applying this configuration will result in following new files on the +`KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate` + +- `/etc/containerd/certs.d/_default/hosts.toml` +- `/etc/certs/mirror.pem` + +To use a public hosted image registry (ex. ECR) as mirror, specify the following configuration: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: clusterConfig + value: + imageRegistries: + - url: https://123456789.dkr.ecr.us-east-1.amazonaws.com + credentials: + secretRef: + name: my-registry-credentials + mirror: {} +``` + +Applying this configuration will result in following new files on the +`KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate` + +- `/etc/containerd/certs.d/_default/hosts.toml` diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go index 45c9b101d..9639c37c7 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go @@ -52,7 +52,10 @@ func (c providerConfig) isCredentialsEmpty() bool { c.Password == "" } -func templateFilesForImageCredentialProviderConfigs(config providerConfig, mirror *mirrorConfig) ([]cabpkv1.File, error) { +func templateFilesForImageCredentialProviderConfigs( + config providerConfig, + mirror *mirrorConfig, +) ([]cabpkv1.File, error) { var files []cabpkv1.File kubeletCredentialProviderConfigFile, err := templateKubeletCredentialProviderConfig() diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go index 6b4eeb41d..2dd351ea3 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go @@ -492,7 +492,6 @@ func newRegistryCredentialsSecret(name, namespace string) *corev1.Secret { secretData := map[string][]byte{ "username": []byte("myuser"), "password": []byte("mypassword"), - "ca.crt": []byte("myCACert"), } return &corev1.Secret{ TypeMeta: metav1.TypeMeta{ @@ -507,6 +506,7 @@ func newRegistryCredentialsSecret(name, namespace string) *corev1.Secret { Type: corev1.SecretTypeOpaque, } } + func newMirrorSecret(name, namespace string) *corev1.Secret { secretData := map[string][]byte{ "ca.crt": []byte("myCACert"), From 305d05bb0bb4ff5ef4c239b03593a2c8183e598c Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Thu, 18 Jan 2024 09:11:11 -0800 Subject: [PATCH 06/12] test: update credentials proivider api version to v1 --- api/v1alpha1/clusterconfig_types.go | 2 +- .../pkg/testutils/capitest/request/items.go | 28 +- .../aws/mutation/metapatch_handler_test.go | 8 + .../docker/mutation/metapatch_handler_test.go | 8 + .../credential_provider_config_files_test.go | 8 +- .../tests/generate_mirror_patches.go | 262 ++++++++++++++++++ .../credentials/tests/generate_patches.go | 201 +------------- 7 files changed, 319 insertions(+), 198 deletions(-) create mode 100644 pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_mirror_patches.go diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index c92bb203c..eb118ac56 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -263,7 +263,7 @@ func (ImageCredentials) VariableSchema() clusterv1.VariableSchema { "namespace": { Description: "The namespace of the Secret containing the registry credentials. " + "Defaults to the namespace of the KubeadmControlPlaneTemplate and KubeadmConfigTemplate" + - " that reference this variable.", + "that reference this variable.", Type: "string", }, }, diff --git a/common/pkg/testutils/capitest/request/items.go b/common/pkg/testutils/capitest/request/items.go index a0313cb70..5444bd4b8 100644 --- a/common/pkg/testutils/capitest/request/items.go +++ b/common/pkg/testutils/capitest/request/items.go @@ -21,8 +21,8 @@ import ( const ( ClusterName = "test-cluster" - KubeadmConfigTemplateRequestObjectName = "test-kubeadmconfigtemplate" - KubeadmControlPlaneTemplateRequestObjectName = "test-kubeadmcontrolplanetemplate" + kubeadmConfigTemplateRequestObjectName = "test-kubeadmconfigtemplate" + kubeadmControlPlaneTemplateRequestObjectName = "test-kubeadmcontrolplanetemplate" Namespace = corev1.NamespaceDefault ) @@ -45,7 +45,16 @@ func NewRequestItem( } } -func NewKubeadmConfigTemplateRequestItem(uid types.UID) runtimehooksv1.GeneratePatchesRequestItem { +func NewKubeadmConfigTemplateRequestItem( + uid types.UID, +) runtimehooksv1.GeneratePatchesRequestItem { + return NewKubeadmConfigTemplateRequest(uid, kubeadmConfigTemplateRequestObjectName) +} + +func NewKubeadmConfigTemplateRequest( + uid types.UID, + name string, +) runtimehooksv1.GeneratePatchesRequestItem { return NewRequestItem( &bootstrapv1.KubeadmConfigTemplate{ TypeMeta: metav1.TypeMeta{ @@ -53,7 +62,7 @@ func NewKubeadmConfigTemplateRequestItem(uid types.UID) runtimehooksv1.GenerateP Kind: "KubeadmConfigTemplate", }, ObjectMeta: metav1.ObjectMeta{ - Name: KubeadmConfigTemplateRequestObjectName, + Name: name, Namespace: Namespace, }, Spec: bootstrapv1.KubeadmConfigTemplateSpec{ @@ -75,8 +84,9 @@ func NewKubeadmConfigTemplateRequestItem(uid types.UID) runtimehooksv1.GenerateP ) } -func NewKubeadmControlPlaneTemplateRequestItem( +func NewKubeadmControlPlaneTemplateRequest( uid types.UID, + name string, ) runtimehooksv1.GeneratePatchesRequestItem { return NewRequestItem( &controlplanev1.KubeadmControlPlaneTemplate{ @@ -85,7 +95,7 @@ func NewKubeadmControlPlaneTemplateRequestItem( Kind: "KubeadmControlPlaneTemplate", }, ObjectMeta: metav1.ObjectMeta{ - Name: KubeadmControlPlaneTemplateRequestObjectName, + Name: name, Namespace: Namespace, }, Spec: controlplanev1.KubeadmControlPlaneTemplateSpec{ @@ -113,6 +123,12 @@ func NewKubeadmControlPlaneTemplateRequestItem( ) } +func NewKubeadmControlPlaneTemplateRequestItem( + uid types.UID, +) runtimehooksv1.GeneratePatchesRequestItem { + return NewKubeadmControlPlaneTemplateRequest(uid, kubeadmControlPlaneTemplateRequestObjectName) +} + func NewAWSClusterTemplateRequestItem( uid types.UID, existingSpec ...capav1.AWSClusterTemplateSpec, diff --git a/pkg/handlers/aws/mutation/metapatch_handler_test.go b/pkg/handlers/aws/mutation/metapatch_handler_test.go index a1a2ddc84..ea952363d 100644 --- a/pkg/handlers/aws/mutation/metapatch_handler_test.go +++ b/pkg/handlers/aws/mutation/metapatch_handler_test.go @@ -156,6 +156,14 @@ func TestGeneratePatches(t *testing.T) { imageregistries.VariableName, ) + imageregistrycredentialstests.TestGenerateMirrorPatches( + t, + metaPatchGeneratorFunc(mgr), + mgr.GetClient(), + clusterconfig.MetaVariableName, + imageregistries.VariableName, + ) + amitests.TestControlPlaneGeneratePatches( t, metaPatchGeneratorFunc(mgr), diff --git a/pkg/handlers/docker/mutation/metapatch_handler_test.go b/pkg/handlers/docker/mutation/metapatch_handler_test.go index f6334174d..d3dea97f2 100644 --- a/pkg/handlers/docker/mutation/metapatch_handler_test.go +++ b/pkg/handlers/docker/mutation/metapatch_handler_test.go @@ -112,4 +112,12 @@ func TestGeneratePatches(t *testing.T) { clusterconfig.MetaVariableName, imageregistries.VariableName, ) + + imageregistrycredentialstests.TestGenerateMirrorPatches( + t, + metaPatchGeneratorFunc(mgr), + mgr.GetClient(), + clusterconfig.MetaVariableName, + imageregistries.VariableName, + ) } diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go index b3ce641bc..c73f564e5 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go @@ -208,7 +208,7 @@ mirror: credentialsStrategy: "MirrorCredentialsOnly" credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ credentialProviders: - apiVersion: kubelet.config.k8s.io/v1beta1 + apiVersion: kubelet.config.k8s.io/v1 kind: CredentialProviderConfig providers: - name: ecr-credential-provider @@ -217,7 +217,7 @@ credentialProviders: matchImages: - "123456789.dkr.ecr.us-east-1.amazonaws.com" defaultCacheDuration: "0s" - apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 + apiVersion: credentialprovider.kubelet.k8s.io/v1 `, }, }, @@ -244,7 +244,7 @@ mirror: credentialsStrategy: "MirrorCredentialsOnly" credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ credentialProviders: - apiVersion: kubelet.config.k8s.io/v1beta1 + apiVersion: kubelet.config.k8s.io/v1 kind: CredentialProviderConfig providers: - name: static-credential-provider @@ -253,7 +253,7 @@ credentialProviders: matchImages: - "myregistry.com" defaultCacheDuration: "0s" - apiVersion: credentialprovider.kubelet.k8s.io/v1beta1 + apiVersion: credentialprovider.kubelet.k8s.io/v1 `, }, }, diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_mirror_patches.go b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_mirror_patches.go new file mode 100644 index 000000000..9dd8aa2ad --- /dev/null +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_mirror_patches.go @@ -0,0 +1,262 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "context" + "fmt" + "testing" + + "github.com/onsi/gomega" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/storage/names" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest/request" +) + +const ( + validMirrorCredentialsSecretName = "my-mirror-registry-credentials" + validMirrorCASecretName = "myregistry-mirror-cacert" + //nolint:gosec // Does not contain hard coded credentials. + cpRegistryAsMirrorCreds = "kubeadmControlPlaneRegistryAsMirrorCreds" + //nolint:gosec // Does not contain hard coded credentials. + workerRegistryAsMirrorCreds = "kubeadmConfigTemplateRegistryAsMirrorCreds" + registryStaticCredentialsSecretSuffix = "registry-config" +) + +func TestGenerateMirrorPatches( + t *testing.T, + generatorFunc func() mutation.GeneratePatches, + fakeClient client.Client, + variableName string, + variablePath ...string, +) { + t.Helper() + + require.NoError( + t, + fakeClient.Create( + context.Background(), + newRegistryCredentialsSecret(validMirrorCredentialsSecretName, request.Namespace), + ), + ) + + require.NoError( + t, + fakeClient.Create( + context.Background(), + newMirrorSecret(validMirrorCASecretName, request.Namespace), + ), + ) + + // Server side apply does not work with the fake client, hack around it by pre-creating empty Secrets + // https://github.com/kubernetes-sigs/controller-runtime/issues/2341 + require.NoError( + t, + fakeClient.Create( + context.Background(), + newEmptySecret( + fmt.Sprintf( + "%s-%s", + cpRegistryAsMirrorCreds, + registryStaticCredentialsSecretSuffix, + ), + request.Namespace, + ), + ), + ) + + require.NoError( + t, + fakeClient.Create( + context.Background(), + newEmptySecret( + fmt.Sprintf( + "%s-%s", + workerRegistryAsMirrorCreds, + registryStaticCredentialsSecretSuffix, + ), + request.Namespace, + ), + ), + ) + + capitest.ValidateGeneratePatches( + t, + generatorFunc, + capitest.PatchTestDef{ + Name: "files added in KubeadmControlPlaneTemplate for registry with mirror without CA Certificate", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + Mirror: &v1alpha1.RegistryMirror{}, + }, + }, + variablePath..., + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: gomega.ContainElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + ), + }, + }, + }, + capitest.PatchTestDef{ + Name: "files added in KubeadmControlPlaneTemplate for registry with mirror with CA Certificate", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ + URL: "https://mirror-registry.com", + Credentials: &v1alpha1.ImageCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: validSecretName, + }, + }, + Mirror: &v1alpha1.RegistryMirror{ + SecretRef: &corev1.ObjectReference{ + Name: validMirrorCASecretName, + }, + }, + }, + }, + variablePath..., + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequest("", cpRegistryAsMirrorCreds), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: gomega.ContainElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/certs/mirror.pem", + ), + ), + }, + }, + }, + capitest.PatchTestDef{ + Name: "files added in KubeadmConfigTemplate for registry mirror wihthout CA certificate", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + Mirror: &v1alpha1.RegistryMirror{}, + }, + }, + variablePath..., + ), + capitest.VariableWithValue( + "builtin", + map[string]any{ + "machineDeployment": map[string]any{ + "class": names.SimpleNameGenerator.GenerateName("worker-"), + }, + }, + ), + }, + RequestItem: request.NewKubeadmConfigTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/files", + ValueMatcher: gomega.ContainElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + ), + }, + }, + }, + capitest.PatchTestDef{ + Name: "files added in KubeadmConfigTemplate for registry mirror with secret for CA certificate", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ + URL: "https://mirror-registry.io", + Credentials: &v1alpha1.ImageCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: validSecretName, + }, + }, + Mirror: &v1alpha1.RegistryMirror{ + SecretRef: &corev1.ObjectReference{ + Name: validMirrorCASecretName, + }, + }, + }, + }, + variablePath..., + ), + capitest.VariableWithValue( + "builtin", + map[string]any{ + "machineDeployment": map[string]any{ + "class": names.SimpleNameGenerator.GenerateName("worker-"), + }, + }, + ), + }, + RequestItem: request.NewKubeadmConfigTemplateRequest("", workerRegistryAsMirrorCreds), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/files", + ValueMatcher: gomega.ContainElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/certs/mirror.pem", + ), + ), + }, + }, + }, + ) +} + +func newMirrorSecret(name, namespace string) *corev1.Secret { + secretData := map[string][]byte{ + "ca.crt": []byte("myCACert"), + } + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: secretData, + Type: corev1.SecretTypeOpaque, + } +} diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go index 2dd351ea3..764048397 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go @@ -5,6 +5,7 @@ package tests import ( "context" + "fmt" "testing" "github.com/onsi/gomega" @@ -22,8 +23,11 @@ import ( ) const ( - validSecretName = "myregistry-credentials" - validMirrorSecretName = "myregistry-mirror-cacert" + validSecretName = "myregistry-credentials" + //nolint:gosec // Does not contain hard coded credentials. + cpRegistryCreds = "kubeadmControlPlaneRegistryWithCredentials" + //nolint:gosec // Does not contain hard coded credentials. + workerRegistryCreds = "kubeadmConfigTemplateRegistryWithCreds" ) func TestGeneratePatches( @@ -35,8 +39,6 @@ func TestGeneratePatches( ) { t.Helper() - // Server side apply does not work with the fake client, hack around it by pre-creating empty Secrets - // https://github.com/kubernetes-sigs/controller-runtime/issues/2341 require.NoError( t, fakeClient.Create( @@ -45,30 +47,25 @@ func TestGeneratePatches( ), ) - require.NoError( - t, - fakeClient.Create( - context.Background(), - newMirrorSecret(validMirrorSecretName, request.Namespace), - ), - ) - + // Server side apply does not work with the fake client, hack around it by pre-creating empty Secrets + // https://github.com/kubernetes-sigs/controller-runtime/issues/2341 require.NoError( t, fakeClient.Create( context.Background(), newEmptySecret( - request.KubeadmControlPlaneTemplateRequestObjectName+"-registry-config", + fmt.Sprintf("%s-%s", cpRegistryCreds, registryStaticCredentialsSecretSuffix), request.Namespace, ), ), ) + require.NoError( t, fakeClient.Create( context.Background(), newEmptySecret( - request.KubeadmConfigTemplateRequestObjectName+"-registry-config", + fmt.Sprintf("%s-%s", workerRegistryCreds, registryStaticCredentialsSecretSuffix), request.Namespace, ), ), @@ -153,7 +150,7 @@ func TestGeneratePatches( variablePath..., ), }, - RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + RequestItem: request.NewKubeadmControlPlaneTemplateRequest("", cpRegistryCreds), ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ { Operation: "add", @@ -198,72 +195,6 @@ func TestGeneratePatches( }, }, }, - capitest.PatchTestDef{ - Name: "files added in KubeadmControlPlaneTemplate for registry with mirror without CA Certificate", - Vars: []runtimehooksv1.Variable{ - capitest.VariableWithValue( - variableName, - v1alpha1.ImageRegistries{ - v1alpha1.ImageRegistry{ - URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", - Mirror: &v1alpha1.RegistryMirror{}, - }, - }, - variablePath..., - ), - }, - RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), - ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ - { - Operation: "add", - Path: "/spec/template/spec/kubeadmConfigSpec/files", - ValueMatcher: gomega.ContainElements( - gomega.HaveKeyWithValue( - "path", "/etc/containerd/certs.d/_default/hosts.toml", - ), - ), - }, - }, - }, - capitest.PatchTestDef{ - Name: "files added in KubeadmControlPlaneTemplate for registry with mirror with CA Certificate", - Vars: []runtimehooksv1.Variable{ - capitest.VariableWithValue( - variableName, - v1alpha1.ImageRegistries{ - v1alpha1.ImageRegistry{ - URL: "https://mirror-registry.com", - Credentials: &v1alpha1.ImageCredentials{ - SecretRef: &corev1.ObjectReference{ - Name: validSecretName, - }, - }, - Mirror: &v1alpha1.RegistryMirror{ - SecretRef: &corev1.ObjectReference{ - Name: validMirrorSecretName, - }, - }, - }, - }, - variablePath..., - ), - }, - RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), - ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ - { - Operation: "add", - Path: "/spec/template/spec/kubeadmConfigSpec/files", - ValueMatcher: gomega.ContainElements( - gomega.HaveKeyWithValue( - "path", "/etc/containerd/certs.d/_default/hosts.toml", - ), - gomega.HaveKeyWithValue( - "path", "/etc/certs/mirror.pem", - ), - ), - }, - }, - }, capitest.PatchTestDef{ Name: "files added in KubeadmConfigTemplate for ECR without a Secret", Vars: []runtimehooksv1.Variable{ @@ -345,7 +276,7 @@ func TestGeneratePatches( }, ), }, - RequestItem: request.NewKubeadmConfigTemplateRequestItem(""), + RequestItem: request.NewKubeadmConfigTemplateRequest("", workerRegistryCreds), ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ { Operation: "add", @@ -382,93 +313,6 @@ func TestGeneratePatches( }, }, }, - capitest.PatchTestDef{ - Name: "files added in KubeadmConfigTemplate for registry mirror with no CA certificate", - Vars: []runtimehooksv1.Variable{ - capitest.VariableWithValue( - variableName, - v1alpha1.ImageRegistries{ - v1alpha1.ImageRegistry{ - URL: "https://my-registry.io", - Credentials: &v1alpha1.ImageCredentials{ - SecretRef: &corev1.ObjectReference{ - Name: validSecretName, - }, - }, - Mirror: &v1alpha1.RegistryMirror{}, - }, - }, - variablePath..., - ), - capitest.VariableWithValue( - "builtin", - map[string]any{ - "machineDeployment": map[string]any{ - "class": names.SimpleNameGenerator.GenerateName("worker-"), - }, - }, - ), - }, - RequestItem: request.NewKubeadmConfigTemplateRequestItem(""), - ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ - { - Operation: "add", - Path: "/spec/template/spec/files", - ValueMatcher: gomega.ContainElements( - gomega.HaveKeyWithValue( - "path", "/etc/containerd/certs.d/_default/hosts.toml", - ), - ), - }, - }, - }, - capitest.PatchTestDef{ - Name: "files added in KubeadmConfigTemplate for registry mirror with secret for CA certificate", - Vars: []runtimehooksv1.Variable{ - capitest.VariableWithValue( - variableName, - v1alpha1.ImageRegistries{ - v1alpha1.ImageRegistry{ - URL: "https://mirror-registry.com", - Credentials: &v1alpha1.ImageCredentials{ - SecretRef: &corev1.ObjectReference{ - Name: validSecretName, - }, - }, - Mirror: &v1alpha1.RegistryMirror{ - SecretRef: &corev1.ObjectReference{ - Name: validMirrorSecretName, - }, - }, - }, - }, - variablePath..., - ), - capitest.VariableWithValue( - "builtin", - map[string]any{ - "machineDeployment": map[string]any{ - "class": names.SimpleNameGenerator.GenerateName("worker-"), - }, - }, - ), - }, - RequestItem: request.NewKubeadmConfigTemplateRequestItem(""), - ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ - { - Operation: "add", - Path: "/spec/template/spec/files", - ValueMatcher: gomega.ContainElements( - gomega.HaveKeyWithValue( - "path", "/etc/containerd/certs.d/_default/hosts.toml", - ), - gomega.HaveKeyWithValue( - "path", "/etc/certs/mirror.pem", - ), - ), - }, - }, - }, capitest.PatchTestDef{ Name: "error for a registry with no credentials", Vars: []runtimehooksv1.Variable{ @@ -507,24 +351,7 @@ func newRegistryCredentialsSecret(name, namespace string) *corev1.Secret { } } -func newMirrorSecret(name, namespace string) *corev1.Secret { - secretData := map[string][]byte{ - "ca.crt": []byte("myCACert"), - } - return &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Data: secretData, - Type: corev1.SecretTypeOpaque, - } -} - +//nolint:unparam //namespace can change in future testcases func newEmptySecret(name, namespace string) *corev1.Secret { return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ From 7fe780ec0ee11659c17ce7becfa005a1ed23d1ed Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Mon, 22 Jan 2024 18:48:23 -0800 Subject: [PATCH 07/12] feat: global image registry mirror variable --- .markdownlint.json | 3 +- api/v1alpha1/clusterconfig_types.go | 52 +++--- api/v1alpha1/zz_generated.deepcopy.go | 50 +++--- .../customization/generic/global-mirror.md | 65 +++++++ .../customization/generic/image-registries.md | 60 ------- .../aws/mutation/metapatch_handler_test.go | 6 +- .../docker/mutation/metapatch_handler_test.go | 6 +- pkg/handlers/generic/mutation/handlers.go | 2 + .../credential_provider_config_files.go | 9 +- .../credential_provider_config_files_test.go | 70 +------- .../imageregistries/credentials/inject.go | 37 ---- ...mic-credential-provider-config.yaml.gotmpl | 7 - .../credentials/tests/generate_patches.go | 4 +- .../credentials/variables_test.go | 15 -- .../generic/mutation/mirrors/constants.go | 8 + .../generic/mutation/mirrors/inject.go | 169 ++++++++++++++++++ .../credentials => mirrors}/mirror.go | 39 ++-- .../credentials => mirrors}/mirror_test.go | 13 +- .../templates/hosts.toml.gotmpl | 0 .../tests/generate_patches.go} | 97 ++-------- .../mutation/mirrors/variables_test.go | 46 +++++ 21 files changed, 390 insertions(+), 368 deletions(-) create mode 100644 docs/content/customization/generic/global-mirror.md create mode 100644 pkg/handlers/generic/mutation/mirrors/constants.go create mode 100644 pkg/handlers/generic/mutation/mirrors/inject.go rename pkg/handlers/generic/mutation/{imageregistries/credentials => mirrors}/mirror.go (77%) rename pkg/handlers/generic/mutation/{imageregistries/credentials => mirrors}/mirror_test.go (91%) rename pkg/handlers/generic/mutation/{imageregistries/credentials => mirrors}/templates/hosts.toml.gotmpl (100%) rename pkg/handlers/generic/mutation/{imageregistries/credentials/tests/generate_mirror_patches.go => mirrors/tests/generate_patches.go} (68%) create mode 100644 pkg/handlers/generic/mutation/mirrors/variables_test.go diff --git a/.markdownlint.json b/.markdownlint.json index 4af4c680f..2c20f6728 100644 --- a/.markdownlint.json +++ b/.markdownlint.json @@ -2,5 +2,6 @@ "heading-style": { "style": "atx" }, "ul-style": { "style": "dash" }, "line-length": { "line_length": 120, "stern": true }, - "hr-style": { "style": "---" } + "hr-style": { "style": "---" }, + "MD013": false } diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index eb118ac56..ae02bdb47 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -89,6 +89,9 @@ type GenericClusterConfig struct { // +optional ImageRegistries ImageRegistries `json:"imageRegistries,omitempty"` + // +optional + GlobalImageRegistryMirror *GlobalImageRegistryMirror `json:"globalImageRegistryMirror,omitempty"` + // +optional Addons *Addons `json:"addons,omitempty"` } @@ -107,7 +110,8 @@ func (s GenericClusterConfig) VariableSchema() clusterv1.VariableSchema { //noli "", ).VariableSchema(). OpenAPIV3Schema, - "imageRegistries": ImageRegistries{}.VariableSchema().OpenAPIV3Schema, + "imageRegistries": ImageRegistries{}.VariableSchema().OpenAPIV3Schema, + "globalImageRegistryMirror": GlobalImageRegistryMirror{}.VariableSchema().OpenAPIV3Schema, }, }, } @@ -239,7 +243,7 @@ func (ExtraAPIServerCertSANs) VariableSchema() clusterv1.VariableSchema { type ImageCredentials struct { // The Secret containing the registry credentials and CA certificate - // The Secret should have keys 'username', 'password' and 'caCert' + // The Secret should have keys 'username', 'password' and 'ca.crt' // This credentials Secret is not required for some registries, e.g. ECR. // +optional SecretRef *corev1.ObjectReference `json:"secretRef,omitempty"` @@ -252,7 +256,7 @@ func (ImageCredentials) VariableSchema() clusterv1.VariableSchema { Properties: map[string]clusterv1.JSONSchemaProps{ "secretRef": { Description: "The Secret containing the registry credentials. " + - "The Secret should have keys 'username', 'password'. " + + "The Secret should have keys 'username', 'password' and 'ca.crt' " + "This credentials Secret is not required for some registries, e.g. ECR.", Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ @@ -273,37 +277,28 @@ func (ImageCredentials) VariableSchema() clusterv1.VariableSchema { } } -type RegistryMirror struct { - // The secret containing CA certificate for the registry mirror. - // The secret should have 'ca.crt' key +// GlobalImageRegistryMirror sets default mirror configuration for all the image registries. +type GlobalImageRegistryMirror struct { + // Registry URL. + URL string `json:"url"` + + // Credentials and CA certificate for the image registry mirror // +optional - SecretRef *corev1.ObjectReference `json:"secretRef,omitempty"` + Credentials *ImageCredentials `json:"credentials,omitempty"` } -func (RegistryMirror) VariableSchema() clusterv1.VariableSchema { +func (GlobalImageRegistryMirror) VariableSchema() clusterv1.VariableSchema { return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ - "secretRef": { - Description: "The Secret containing the registry CA certificate. " + - "The Secret should have keys 'ca.crt'. " + - "This credentials Secret is not required for public registries.", - Type: "object", - Properties: map[string]clusterv1.JSONSchemaProps{ - "name": { - Description: "The name of the Secret containing the registry CA certificate.", - Type: "string", - }, - "namespace": { - Description: "The namespace of the Secret containing the registry CA certificate. " + - "Defaults to the namespace of the KubeadmControlPlaneTemplate and KubeadmConfigTemplate" + - " that reference this variable.", - Type: "string", - }, - }, + "url": { + Description: "Registry mirror URL.", + Type: "string", }, + "credentials": ImageCredentials{}.VariableSchema().OpenAPIV3Schema, }, + Required: []string{"url"}, }, } } @@ -312,13 +307,9 @@ type ImageRegistry struct { // Registry URL. URL string `json:"url"` - // Credentials for the image registry + // Credentials and CA certificate for the image registry // +optional Credentials *ImageCredentials `json:"credentials,omitempty"` - - // Use this registry as a mirror - // +optional - Mirror *RegistryMirror `json:"mirror,omitempty"` } func (ImageRegistry) VariableSchema() clusterv1.VariableSchema { @@ -331,7 +322,6 @@ func (ImageRegistry) VariableSchema() clusterv1.VariableSchema { Type: "string", }, "credentials": ImageCredentials{}.VariableSchema().OpenAPIV3Schema, - "mirror": RegistryMirror{}.VariableSchema().OpenAPIV3Schema, }, Required: []string{"url"}, }, diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index c9fa1330a..6f5eafcab 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -442,6 +442,11 @@ func (in *GenericClusterConfig) DeepCopyInto(out *GenericClusterConfig) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.GlobalImageRegistryMirror != nil { + in, out := &in.GlobalImageRegistryMirror, &out.GlobalImageRegistryMirror + *out = new(GlobalImageRegistryMirror) + (*in).DeepCopyInto(*out) + } if in.Addons != nil { in, out := &in.Addons, &out.Addons *out = new(Addons) @@ -474,6 +479,26 @@ func (in *GenericNodeConfig) DeepCopy() *GenericNodeConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalImageRegistryMirror) DeepCopyInto(out *GlobalImageRegistryMirror) { + *out = *in + if in.Credentials != nil { + in, out := &in.Credentials, &out.Credentials + *out = new(ImageCredentials) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalImageRegistryMirror. +func (in *GlobalImageRegistryMirror) DeepCopy() *GlobalImageRegistryMirror { + if in == nil { + return nil + } + out := new(GlobalImageRegistryMirror) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPProxy) DeepCopyInto(out *HTTPProxy) { *out = *in @@ -558,11 +583,6 @@ func (in *ImageRegistry) DeepCopyInto(out *ImageRegistry) { *out = new(ImageCredentials) (*in).DeepCopyInto(*out) } - if in.Mirror != nil { - in, out := &in.Mirror, &out.Mirror - *out = new(RegistryMirror) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistry. @@ -670,26 +690,6 @@ func (in *ObjectMeta) DeepCopy() *ObjectMeta { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RegistryMirror) DeepCopyInto(out *RegistryMirror) { - *out = *in - if in.SecretRef != nil { - in, out := &in.SecretRef, &out.SecretRef - *out = new(v1.ObjectReference) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RegistryMirror. -func (in *RegistryMirror) DeepCopy() *RegistryMirror { - if in == nil { - return nil - } - out := new(RegistryMirror) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecurityGroup) DeepCopyInto(out *SecurityGroup) { *out = *in diff --git a/docs/content/customization/generic/global-mirror.md b/docs/content/customization/generic/global-mirror.md new file mode 100644 index 000000000..543de1129 --- /dev/null +++ b/docs/content/customization/generic/global-mirror.md @@ -0,0 +1,65 @@ ++++ +title = "Global Image Registry Mirror" ++++ + +Add containerd image registry mirror configuration to all Nodes in the cluster. + +When the `globalImageRegistryMirror` variable is set, `files` with configurations for [Containerd default mirror](https://github.com/containerd/containerd/blob/main/docs/hosts.md#setup-default-mirror-for-all-registries) will be added. + +This customization will be available when the +[provider-specific cluster configuration patch]({{< ref "..">}}) is included in the `ClusterClass`. + +## Example + +To provide image registry mirror with CA certificate, specify the following configuration: + +If your registry mirror requires self signed CA certifate, create a Kubernetes Secret with keys for `ca.crt`: + +```shell +kubectl create secret generic my-mirror-ca-cert-secret \ + --from-file=ca.crt=registry-ca.crt +``` + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: clusterConfig + value: + globalImageRegistryMirror: + url: https://my-mirror.io + credentials: + secretRef: + name: my-mirror-ca-cert-secret +``` + +Applying this configuration will result in following new files on the +`KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate` + +- `/etc/containerd/certs.d/_default/hosts.toml` +- `/etc/certs/mirror.pem` + +To use a public hosted image registry (ex. ECR) as mirror, specify the following configuration: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: clusterConfig + value: + globalImageRegistryMirror: + url: https://123456789.dkr.ecr.us-east-1.amazonaws.com +``` + +Applying this configuration will result in following new files on the +`KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate` + +- `/etc/containerd/certs.d/_default/hosts.toml` diff --git a/docs/content/customization/generic/image-registries.md b/docs/content/customization/generic/image-registries.md index 1013fb574..2c1e8b336 100644 --- a/docs/content/customization/generic/image-registries.md +++ b/docs/content/customization/generic/image-registries.md @@ -41,63 +41,3 @@ spec: Applying this configuration will result in new files and preKubeadmCommands on the `KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate`. - -To use a image registry as mirror with CA certificate, specify the following configuration: - -If your registry mirror requires self signed CA certifate, create a Kubernetes Secret with keys for `ca.crt`: - -```shell -kubectl create secret generic my-mirror-ca-cert-secret \ - --from-file=ca.crt=registry-ca.crt -``` - -```yaml -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: -spec: - topology: - variables: - - name: clusterConfig - value: - imageRegistries: - - url: https://my-registry.io - credentials: - secretRef: - name: my-registry-credentials - mirror: - secretRef: - name: my-mirror-ca-cert-secret -``` - -Applying this configuration will result in following new files on the -`KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate` - -- `/etc/containerd/certs.d/_default/hosts.toml` -- `/etc/certs/mirror.pem` - -To use a public hosted image registry (ex. ECR) as mirror, specify the following configuration: - -```yaml -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: -spec: - topology: - variables: - - name: clusterConfig - value: - imageRegistries: - - url: https://123456789.dkr.ecr.us-east-1.amazonaws.com - credentials: - secretRef: - name: my-registry-credentials - mirror: {} -``` - -Applying this configuration will result in following new files on the -`KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate` - -- `/etc/containerd/certs.d/_default/hosts.toml` diff --git a/pkg/handlers/aws/mutation/metapatch_handler_test.go b/pkg/handlers/aws/mutation/metapatch_handler_test.go index ea952363d..2e8e63b5a 100644 --- a/pkg/handlers/aws/mutation/metapatch_handler_test.go +++ b/pkg/handlers/aws/mutation/metapatch_handler_test.go @@ -38,6 +38,8 @@ import ( imageregistrycredentialstests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository" kubernetesimagerepositorytests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository/tests" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors" + globalimageregistrymirrortests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/workerconfig" ) @@ -156,12 +158,12 @@ func TestGeneratePatches(t *testing.T) { imageregistries.VariableName, ) - imageregistrycredentialstests.TestGenerateMirrorPatches( + globalimageregistrymirrortests.TestGeneratePatches( t, metaPatchGeneratorFunc(mgr), mgr.GetClient(), clusterconfig.MetaVariableName, - imageregistries.VariableName, + mirrors.GlobalMirrorVariableName, ) amitests.TestControlPlaneGeneratePatches( diff --git a/pkg/handlers/docker/mutation/metapatch_handler_test.go b/pkg/handlers/docker/mutation/metapatch_handler_test.go index d3dea97f2..6fd017258 100644 --- a/pkg/handlers/docker/mutation/metapatch_handler_test.go +++ b/pkg/handlers/docker/mutation/metapatch_handler_test.go @@ -28,6 +28,8 @@ import ( imageregistrycredentialstests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository" kubernetesimagerepositorytests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository/tests" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors" + globalimageregistrymirrortests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/workerconfig" ) @@ -113,11 +115,11 @@ func TestGeneratePatches(t *testing.T) { imageregistries.VariableName, ) - imageregistrycredentialstests.TestGenerateMirrorPatches( + globalimageregistrymirrortests.TestGeneratePatches( t, metaPatchGeneratorFunc(mgr), mgr.GetClient(), clusterconfig.MetaVariableName, - imageregistries.VariableName, + mirrors.GlobalMirrorVariableName, ) } diff --git a/pkg/handlers/generic/mutation/handlers.go b/pkg/handlers/generic/mutation/handlers.go index aa2160737..7ac6f66f6 100644 --- a/pkg/handlers/generic/mutation/handlers.go +++ b/pkg/handlers/generic/mutation/handlers.go @@ -14,6 +14,7 @@ import ( "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/httpproxy" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors" ) // MetaMutators returns all generic patch handlers. @@ -25,6 +26,7 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator { httpproxy.NewPatch(mgr.GetClient()), kubernetesimagerepository.NewPatch(), credentials.NewPatch(mgr.GetClient()), + mirrors.NewPatch(mgr.GetClient()), calico.NewPatch(), } } diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go index 9639c37c7..fe3d5c6c2 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go @@ -52,10 +52,7 @@ func (c providerConfig) isCredentialsEmpty() bool { c.Password == "" } -func templateFilesForImageCredentialProviderConfigs( - config providerConfig, - mirror *mirrorConfig, -) ([]cabpkv1.File, error) { +func templateFilesForImageCredentialProviderConfigs(config providerConfig) ([]cabpkv1.File, error) { var files []cabpkv1.File kubeletCredentialProviderConfigFile, err := templateKubeletCredentialProviderConfig() @@ -68,7 +65,6 @@ func templateFilesForImageCredentialProviderConfigs( kubeletDynamicCredentialProviderConfigFile, err := templateDynamicCredentialProviderConfig( config, - mirror, ) if err != nil { return nil, err @@ -104,7 +100,6 @@ func templateKubeletCredentialProviderConfig() (*cabpkv1.File, error) { func templateDynamicCredentialProviderConfig( config providerConfig, - mirror *mirrorConfig, ) (*cabpkv1.File, error) { registryURL, err := url.ParseRequestURI(config.URL) if err != nil { @@ -142,13 +137,11 @@ func templateDynamicCredentialProviderConfig( ProviderBinary string ProviderArgs []string ProviderAPIVersion string - Mirror *mirrorConfig }{ RegistryHost: registryHostWithPath, ProviderBinary: providerBinary, ProviderArgs: providerArgs, ProviderAPIVersion: providerAPIVersion, - Mirror: mirror, } return fileFromTemplate(t, templateInput, kubeletDynamicCredentialProviderConfigOnRemote) diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go index c73f564e5..e8ce8dd87 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go @@ -99,7 +99,6 @@ func Test_templateDynamicCredentialProviderConfig(t *testing.T) { tests := []struct { name string credentials providerConfig - mirror *mirrorConfig want *cabpkv1.File wantErr error }{ @@ -190,73 +189,6 @@ credentialProviders: `, }, }, - - { - name: "ECR image registry used as mirror", - credentials: providerConfig{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, - mirror: &mirrorConfig{}, - want: &cabpkv1.File{ - Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", - Owner: "", - Permissions: "0600", - Encoding: "", - Append: false, - Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 -kind: DynamicCredentialProviderConfig -mirror: - endpoint: "123456789.dkr.ecr.us-east-1.amazonaws.com" - credentialsStrategy: "MirrorCredentialsOnly" -credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ -credentialProviders: - apiVersion: kubelet.config.k8s.io/v1 - kind: CredentialProviderConfig - providers: - - name: ecr-credential-provider - args: - - get-credentials - matchImages: - - "123456789.dkr.ecr.us-east-1.amazonaws.com" - defaultCacheDuration: "0s" - apiVersion: credentialprovider.kubelet.k8s.io/v1 -`, - }, - }, - { - name: "image registry with static credentials used as mirror", - credentials: providerConfig{ - URL: "https://myregistry.com", - Username: "myuser", - Password: "mypassword", - }, - mirror: &mirrorConfig{ - CACert: "my-ca-cert", - }, - want: &cabpkv1.File{ - Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", - Owner: "", - Permissions: "0600", - Encoding: "", - Append: false, - Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 -kind: DynamicCredentialProviderConfig -mirror: - endpoint: "myregistry.com" - credentialsStrategy: "MirrorCredentialsOnly" -credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ -credentialProviders: - apiVersion: kubelet.config.k8s.io/v1 - kind: CredentialProviderConfig - providers: - - name: static-credential-provider - args: - - /etc/kubernetes/static-image-credentials.json - matchImages: - - "myregistry.com" - defaultCacheDuration: "0s" - apiVersion: credentialprovider.kubelet.k8s.io/v1 -`, - }, - }, { name: "error for a registry with no credentials", credentials: providerConfig{ @@ -269,7 +201,7 @@ credentialProviders: tt := tests[idx] t.Run(tt.name, func(t *testing.T) { t.Parallel() - file, err := templateDynamicCredentialProviderConfig(tt.credentials, tt.mirror) + file, err := templateDynamicCredentialProviderConfig(tt.credentials) assert.ErrorIs(t, err, tt.wantErr) assert.Equal(t, tt.want, file) }) diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go index 6c8cff9f9..6353c9a60 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go @@ -107,19 +107,8 @@ func (h *imageRegistriesPatchHandler) Mutate( if generateErr != nil { return generateErr } - mirrorConfig, err := mirrorFromImageRegistry( - ctx, - h.client, - imageRegistry, - obj, - ) - if err != nil { - return err - } files, commands, generateErr := generateFilesAndCommands( registryWithOptionalCredentials, - mirrorConfig, - imageRegistry, obj.GetName()) if generateErr != nil { return generateErr @@ -181,19 +170,8 @@ func (h *imageRegistriesPatchHandler) Mutate( if generateErr != nil { return generateErr } - mirrorConfig, err := mirrorFromImageRegistry( - ctx, - h.client, - imageRegistry, - obj, - ) - if err != nil { - return err - } files, commands, generateErr := generateFilesAndCommands( registryWithOptionalCredentials, - mirrorConfig, - imageRegistry, obj.GetName()) if generateErr != nil { return generateErr @@ -268,8 +246,6 @@ func registryWithOptionalCredentialsFromImageRegistryCredentials( func generateFilesAndCommands( registryWithOptionalCredentials providerConfig, - mirrorConfig *mirrorConfig, - imageRegistry v1alpha1.ImageRegistry, objName string, ) ([]bootstrapv1.File, []string, error) { files, commands, err := templateFilesAndCommandsForInstallKubeletCredentialProviders() @@ -281,7 +257,6 @@ func generateFilesAndCommands( } imageCredentialProviderConfigFiles, err := templateFilesForImageCredentialProviderConfigs( registryWithOptionalCredentials, - mirrorConfig, ) if err != nil { return nil, nil, fmt.Errorf( @@ -293,18 +268,6 @@ func generateFilesAndCommands( files = append( files, generateCredentialsSecretFile(registryWithOptionalCredentials, objName)...) - - // Generate default registry mirror file - mirrorHostFiles, err := generateDefaultRegistryMirrorFile(mirrorConfig) - if err != nil { - return nil, nil, err - } - files = append(files, mirrorHostFiles...) - // generate CA certificate file for registry mirror - files = append( - files, - generateMirrorCACertFile(mirrorConfig, imageRegistry)...) - return files, commands, err } diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl index 196203996..80156c219 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl @@ -1,12 +1,5 @@ apiVersion: credentialprovider.d2iq.com/v1alpha1 kind: DynamicCredentialProviderConfig -{{- if .Mirror }} -mirror: - {{- with .RegistryHost }} - endpoint: {{ printf "%q" . }} - {{- end }} - credentialsStrategy: "MirrorCredentialsOnly" -{{- end }} credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ credentialProviders: apiVersion: kubelet.config.k8s.io/v1 diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go index 764048397..40aa0027e 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go @@ -27,7 +27,8 @@ const ( //nolint:gosec // Does not contain hard coded credentials. cpRegistryCreds = "kubeadmControlPlaneRegistryWithCredentials" //nolint:gosec // Does not contain hard coded credentials. - workerRegistryCreds = "kubeadmConfigTemplateRegistryWithCreds" + workerRegistryCreds = "kubeadmConfigTemplateRegistryWithCreds" + registryStaticCredentialsSecretSuffix = "registry-config" ) func TestGeneratePatches( @@ -351,7 +352,6 @@ func newRegistryCredentialsSecret(name, namespace string) *corev1.Secret { } } -//nolint:unparam //namespace can change in future testcases func newEmptySecret(name, namespace string) *corev1.Secret { return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go index 16562b92f..fd687e0af 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go @@ -46,20 +46,5 @@ func TestVariableValidation(t *testing.T) { }, }, }, - capitest.VariableTestDef{ - Name: "with a mirror secret", - Vals: v1alpha1.GenericClusterConfig{ - ImageRegistries: []v1alpha1.ImageRegistry{ - { - URL: "http://a.b.c.example.com", - Mirror: &v1alpha1.RegistryMirror{ - SecretRef: &corev1.ObjectReference{ - Name: "a.b.c.example.com-creds", - }, - }, - }, - }, - }, - }, ) } diff --git a/pkg/handlers/generic/mutation/mirrors/constants.go b/pkg/handlers/generic/mutation/mirrors/constants.go new file mode 100644 index 000000000..13639af8c --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/constants.go @@ -0,0 +1,8 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +const ( + GlobalMirrorVariableName = "globalImageRegistryMirror" +) diff --git a/pkg/handlers/generic/mutation/mirrors/inject.go b/pkg/handlers/generic/mutation/mirrors/inject.go new file mode 100644 index 000000000..e9c42320f --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/inject.go @@ -0,0 +1,169 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +import ( + "context" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/patches" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/patches/selectors" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/variables" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/clusterconfig" +) + +type globalMirrorPatchHandler struct { + client ctrlclient.Client + + variableName string + variableFieldPath []string +} + +func NewPatch( + cl ctrlclient.Client, +) *globalMirrorPatchHandler { + return newGlobalMirrorPatchHandler( + cl, + clusterconfig.MetaVariableName, + GlobalMirrorVariableName, + ) +} + +func newGlobalMirrorPatchHandler( + cl ctrlclient.Client, + variableName string, + variableFieldPath ...string, +) *globalMirrorPatchHandler { + scheme := runtime.NewScheme() + _ = bootstrapv1.AddToScheme(scheme) + _ = controlplanev1.AddToScheme(scheme) + return &globalMirrorPatchHandler{ + client: cl, + variableName: variableName, + variableFieldPath: variableFieldPath, + } +} + +func (h *globalMirrorPatchHandler) Mutate( + ctx context.Context, + obj *unstructured.Unstructured, + vars map[string]apiextensionsv1.JSON, + holderRef runtimehooksv1.HolderReference, + clusterKey ctrlclient.ObjectKey, +) error { + log := ctrl.LoggerFrom(ctx).WithValues( + "holderRef", holderRef, + ) + + globalMirror, found, err := variables.Get[v1alpha1.GlobalImageRegistryMirror]( + vars, + h.variableName, + h.variableFieldPath..., + ) + if err != nil { + return err + } + if !found { + log.V(5).Info("Global registry mirror variable not defined") + return nil + } + + log = log.WithValues( + "variableName", + h.variableName, + "variableFieldPath", + h.variableFieldPath, + "variableValue", + globalMirror, + ) + + if err := patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.ControlPlane(), log, + func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { + mirrorConfig, err := mirrorConfigForGlobalMirror( + ctx, + h.client, + globalMirror, + obj, + ) + if err != nil { + return err + } + files, generateErr := generateFiles( + mirrorConfig, + globalMirror) + if generateErr != nil { + return generateErr + } + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding global registry mirror files to control plane kubeadm config spec") + obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.Files, + files..., + ) + + return nil + }); err != nil { + return err + } + + if err := patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log, + func(obj *bootstrapv1.KubeadmConfigTemplate) error { + mirrorConfig, err := mirrorConfigForGlobalMirror( + ctx, + h.client, + globalMirror, + obj, + ) + if err != nil { + return err + } + files, generateErr := generateFiles( + mirrorConfig, + globalMirror) + if generateErr != nil { + return generateErr + } + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding global registry mirror files to worker node kubeadm config template") + obj.Spec.Template.Spec.Files = append(obj.Spec.Template.Spec.Files, files...) + return nil + }); err != nil { + return err + } + + return nil +} + +func generateFiles( + mirrorConfig *mirrorConfig, + globalMirror v1alpha1.GlobalImageRegistryMirror, +) ([]bootstrapv1.File, error) { + // Generate default registry mirror file + files, err := generateGlobalRegistryMirrorFile(mirrorConfig) + if err != nil { + return nil, err + } + // generate CA certificate file for registry mirror + mirrorCAFile := generateMirrorCACertFile(mirrorConfig, globalMirror) + files = append(files, mirrorCAFile...) + + return files, err +} diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go b/pkg/handlers/generic/mutation/mirrors/mirror.go similarity index 77% rename from pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go rename to pkg/handlers/generic/mutation/mirrors/mirror.go index 0fe4c6e62..409d0fea9 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go +++ b/pkg/handlers/generic/mutation/mirrors/mirror.go @@ -1,7 +1,7 @@ // Copyright 2023 D2iQ, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package credentials +package mirrors import ( "bytes" @@ -31,34 +31,26 @@ type mirrorConfig struct { CACert string } -func mirrorFromImageRegistry( +func mirrorConfigForGlobalMirror( ctx context.Context, c ctrlclient.Client, - imageRegistry v1alpha1.ImageRegistry, + globalMirror v1alpha1.GlobalImageRegistryMirror, obj ctrlclient.Object, ) (*mirrorConfig, error) { - // using the registry as a mirror is supported by including empty mirror object or - // mirror with CA certificate to the registry variable. - // ex. - // - url: https://my-registry.com - // mirror: {} - if imageRegistry.Mirror == nil { - return nil, nil - } mirrorWithOptionalCACert := &mirrorConfig{ - URL: imageRegistry.URL, + URL: globalMirror.URL, } secret, err := secretForMirrorCACert( ctx, c, - imageRegistry, + globalMirror, obj.GetNamespace(), ) if err != nil { return &mirrorConfig{}, fmt.Errorf( "error getting secret %s/%s from Image Registry variable: %w", obj.GetNamespace(), - imageRegistry.Mirror.SecretRef.Name, + globalMirror.Credentials.SecretRef.Name, err, ) } @@ -75,20 +67,20 @@ func mirrorFromImageRegistry( func secretForMirrorCACert( ctx context.Context, c ctrlclient.Reader, - registry v1alpha1.ImageRegistry, + globalMirror v1alpha1.GlobalImageRegistryMirror, objectNamespace string, ) (*corev1.Secret, error) { - if registry.Mirror == nil || registry.Mirror.SecretRef == nil { + if globalMirror.Credentials == nil || globalMirror.Credentials.SecretRef == nil { return nil, nil } namespace := objectNamespace - if registry.Mirror.SecretRef.Namespace != "" { - namespace = registry.Mirror.SecretRef.Namespace + if globalMirror.Credentials.SecretRef.Namespace != "" { + namespace = globalMirror.Credentials.SecretRef.Namespace } key := ctrlclient.ObjectKey{ - Name: registry.Mirror.SecretRef.Name, + Name: globalMirror.Credentials.SecretRef.Name, Namespace: namespace, } secret := &corev1.Secret{} @@ -96,10 +88,11 @@ func secretForMirrorCACert( return secret, err } -// Default Mirror for all registries. Use a mirror regardless of the intended registry. +// Default Mirror for all registries. +// Containerd configuration for global mirror will be created at /etc/containerd/certs.d/_default/hosts.toml // The upstream registry will be automatically used after all defined mirrors have been tried. // reference: https://github.com/containerd/containerd/blob/main/docs/hosts.md#setup-default-mirror-for-all-registries -func generateDefaultRegistryMirrorFile(mirror *mirrorConfig) ([]cabpkv1.File, error) { +func generateGlobalRegistryMirrorFile(mirror *mirrorConfig) ([]cabpkv1.File, error) { if mirror == nil { return nil, nil } @@ -135,7 +128,7 @@ func generateDefaultRegistryMirrorFile(mirror *mirrorConfig) ([]cabpkv1.File, er func generateMirrorCACertFile( config *mirrorConfig, - registry v1alpha1.ImageRegistry, + globalMirror v1alpha1.GlobalImageRegistryMirror, ) []cabpkv1.File { if config == nil || config.CACert == "" { return nil @@ -146,7 +139,7 @@ func generateMirrorCACertFile( Permissions: "0600", ContentFrom: &cabpkv1.FileSource{ Secret: cabpkv1.SecretFileSource{ - Name: registry.Mirror.SecretRef.Name, + Name: globalMirror.Credentials.SecretRef.Name, Key: secretKeyForMirrorCACert, }, }, diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go b/pkg/handlers/generic/mutation/mirrors/mirror_test.go similarity index 91% rename from pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go rename to pkg/handlers/generic/mutation/mirrors/mirror_test.go index 82440b04d..855527a44 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go +++ b/pkg/handlers/generic/mutation/mirrors/mirror_test.go @@ -1,7 +1,7 @@ // Copyright 2023 D2iQ, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package credentials +package mirrors import ( "testing" @@ -64,7 +64,7 @@ func Test_generateDefaultRegistryMirrorFile(t *testing.T) { tt := tests[idx] t.Run(tt.name, func(t *testing.T) { t.Parallel() - file, err := generateDefaultRegistryMirrorFile(tt.config) + file, err := generateGlobalRegistryMirrorFile(tt.config) assert.ErrorIs(t, err, tt.wantErr) assert.Equal(t, tt.want, file) }) @@ -76,7 +76,7 @@ func Test_generateMirrorCACertFile(t *testing.T) { tests := []struct { name string config *mirrorConfig - registry v1alpha1.ImageRegistry + registry v1alpha1.GlobalImageRegistryMirror want []cabpkv1.File }{ { @@ -84,7 +84,7 @@ func Test_generateMirrorCACertFile(t *testing.T) { config: &mirrorConfig{ URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }, - registry: v1alpha1.ImageRegistry{ + registry: v1alpha1.GlobalImageRegistryMirror{ URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }, want: nil, @@ -95,9 +95,10 @@ func Test_generateMirrorCACertFile(t *testing.T) { URL: "https://myregistry.com", CACert: "mycacert", }, - registry: v1alpha1.ImageRegistry{ + registry: v1alpha1.GlobalImageRegistryMirror{ URL: "https://myregistry.com", - Mirror: &v1alpha1.RegistryMirror{ + + Credentials: &v1alpha1.ImageCredentials{ SecretRef: &v1.ObjectReference{ Name: "my-registry-credentials-secret", }, diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/hosts.toml.gotmpl b/pkg/handlers/generic/mutation/mirrors/templates/hosts.toml.gotmpl similarity index 100% rename from pkg/handlers/generic/mutation/imageregistries/credentials/templates/hosts.toml.gotmpl rename to pkg/handlers/generic/mutation/mirrors/templates/hosts.toml.gotmpl diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_mirror_patches.go b/pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go similarity index 68% rename from pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_mirror_patches.go rename to pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go index 9dd8aa2ad..10de0fbd2 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_mirror_patches.go +++ b/pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go @@ -5,7 +5,6 @@ package tests import ( "context" - "fmt" "testing" "github.com/onsi/gomega" @@ -23,16 +22,14 @@ import ( ) const ( - validMirrorCredentialsSecretName = "my-mirror-registry-credentials" - validMirrorCASecretName = "myregistry-mirror-cacert" + validMirrorCASecretName = "myregistry-mirror-cacert" //nolint:gosec // Does not contain hard coded credentials. cpRegistryAsMirrorCreds = "kubeadmControlPlaneRegistryAsMirrorCreds" //nolint:gosec // Does not contain hard coded credentials. - workerRegistryAsMirrorCreds = "kubeadmConfigTemplateRegistryAsMirrorCreds" - registryStaticCredentialsSecretSuffix = "registry-config" + workerRegistryAsMirrorCreds = "kubeadmConfigTemplateRegistryAsMirrorCreds" ) -func TestGenerateMirrorPatches( +func TestGeneratePatches( t *testing.T, generatorFunc func() mutation.GeneratePatches, fakeClient client.Client, @@ -41,14 +38,6 @@ func TestGenerateMirrorPatches( ) { t.Helper() - require.NoError( - t, - fakeClient.Create( - context.Background(), - newRegistryCredentialsSecret(validMirrorCredentialsSecretName, request.Namespace), - ), - ) - require.NoError( t, fakeClient.Create( @@ -57,38 +46,6 @@ func TestGenerateMirrorPatches( ), ) - // Server side apply does not work with the fake client, hack around it by pre-creating empty Secrets - // https://github.com/kubernetes-sigs/controller-runtime/issues/2341 - require.NoError( - t, - fakeClient.Create( - context.Background(), - newEmptySecret( - fmt.Sprintf( - "%s-%s", - cpRegistryAsMirrorCreds, - registryStaticCredentialsSecretSuffix, - ), - request.Namespace, - ), - ), - ) - - require.NoError( - t, - fakeClient.Create( - context.Background(), - newEmptySecret( - fmt.Sprintf( - "%s-%s", - workerRegistryAsMirrorCreds, - registryStaticCredentialsSecretSuffix, - ), - request.Namespace, - ), - ), - ) - capitest.ValidateGeneratePatches( t, generatorFunc, @@ -97,11 +54,8 @@ func TestGenerateMirrorPatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistries{ - v1alpha1.ImageRegistry{ - URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", - Mirror: &v1alpha1.RegistryMirror{}, - }, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }, variablePath..., ), @@ -124,18 +78,11 @@ func TestGenerateMirrorPatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistries{ - v1alpha1.ImageRegistry{ - URL: "https://mirror-registry.com", - Credentials: &v1alpha1.ImageCredentials{ - SecretRef: &corev1.ObjectReference{ - Name: validSecretName, - }, - }, - Mirror: &v1alpha1.RegistryMirror{ - SecretRef: &corev1.ObjectReference{ - Name: validMirrorCASecretName, - }, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://mirror-registry.com", + Credentials: &v1alpha1.ImageCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: validMirrorCASecretName, }, }, }, @@ -163,11 +110,8 @@ func TestGenerateMirrorPatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistries{ - v1alpha1.ImageRegistry{ - URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", - Mirror: &v1alpha1.RegistryMirror{}, - }, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }, variablePath..., ), @@ -198,18 +142,11 @@ func TestGenerateMirrorPatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistries{ - v1alpha1.ImageRegistry{ - URL: "https://mirror-registry.io", - Credentials: &v1alpha1.ImageCredentials{ - SecretRef: &corev1.ObjectReference{ - Name: validSecretName, - }, - }, - Mirror: &v1alpha1.RegistryMirror{ - SecretRef: &corev1.ObjectReference{ - Name: validMirrorCASecretName, - }, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://mirror-registry.io", + Credentials: &v1alpha1.ImageCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: validMirrorCASecretName, }, }, }, diff --git a/pkg/handlers/generic/mutation/mirrors/variables_test.go b/pkg/handlers/generic/mutation/mirrors/variables_test.go new file mode 100644 index 000000000..5533b9906 --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/variables_test.go @@ -0,0 +1,46 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/clusterconfig" +) + +func TestVariableValidation(t *testing.T) { + capitest.ValidateDiscoverVariables( + t, + clusterconfig.MetaVariableName, + ptr.To(v1alpha1.GenericClusterConfig{}.VariableSchema()), + false, + clusterconfig.NewVariable, + capitest.VariableTestDef{ + Name: "without a credentials secret", + Vals: v1alpha1.GenericClusterConfig{ + GlobalImageRegistryMirror: &v1alpha1.GlobalImageRegistryMirror{ + URL: "http://a.b.c.example.com", + }, + }, + }, + capitest.VariableTestDef{ + Name: "with a credentials CA secret", + Vals: v1alpha1.GenericClusterConfig{ + GlobalImageRegistryMirror: &v1alpha1.GlobalImageRegistryMirror{ + URL: "http://a.b.c.example.com", + Credentials: &v1alpha1.ImageCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: "a.b.c.example.com-ca-cert-creds", + }, + }, + }, + }, + }, + ) +} From 2e64b45b23275078bcbd5c0aa73a7c5c6b5bd3c0 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Thu, 25 Jan 2024 13:15:43 -0800 Subject: [PATCH 08/12] docs: fix comments and mirror documentation --- api/v1alpha1/clusterconfig_types.go | 6 +++--- docs/content/customization/generic/global-mirror.md | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index ae02bdb47..384682d89 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -242,8 +242,8 @@ func (ExtraAPIServerCertSANs) VariableSchema() clusterv1.VariableSchema { } type ImageCredentials struct { - // The Secret containing the registry credentials and CA certificate - // The Secret should have keys 'username', 'password' and 'ca.crt' + // The Secret containing the registry credentials and optional CA certificate + // using the keys `username`, `password` and `ca.crt`. // This credentials Secret is not required for some registries, e.g. ECR. // +optional SecretRef *corev1.ObjectReference `json:"secretRef,omitempty"` @@ -256,7 +256,7 @@ func (ImageCredentials) VariableSchema() clusterv1.VariableSchema { Properties: map[string]clusterv1.JSONSchemaProps{ "secretRef": { Description: "The Secret containing the registry credentials. " + - "The Secret should have keys 'username', 'password' and 'ca.crt' " + + "The Secret should have keys 'username', 'password' and optional 'ca.crt' " + "This credentials Secret is not required for some registries, e.g. ECR.", Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ diff --git a/docs/content/customization/generic/global-mirror.md b/docs/content/customization/generic/global-mirror.md index 543de1129..74e959a82 100644 --- a/docs/content/customization/generic/global-mirror.md +++ b/docs/content/customization/generic/global-mirror.md @@ -13,7 +13,8 @@ This customization will be available when the To provide image registry mirror with CA certificate, specify the following configuration: -If your registry mirror requires self signed CA certifate, create a Kubernetes Secret with keys for `ca.crt`: +If your registry mirror requires a private or self-signed CA certificate, +create a Kubernetes Secret with the `ca.crt` key populated with the CA certificate in PEM format: ```shell kubectl create secret generic my-mirror-ca-cert-secret \ From 172e1efb0a60fbaf3ababc4e44e5a8815966ad27 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Thu, 25 Jan 2024 13:40:42 -0800 Subject: [PATCH 09/12] fix: schema validation for registry and mirror URL --- api/v1alpha1/clusterconfig_types.go | 6 ++++- .../customization/generic/global-mirror.md | 4 ++-- .../imageregistries/credentials/inject.go | 3 ++- .../credentials/variables_test.go | 24 ++++++++++++++++++- .../mutation/mirrors/variables_test.go | 18 ++++++++++++++ 5 files changed, 50 insertions(+), 5 deletions(-) diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index 384682d89..94754cbd1 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -266,7 +266,7 @@ func (ImageCredentials) VariableSchema() clusterv1.VariableSchema { }, "namespace": { Description: "The namespace of the Secret containing the registry credentials. " + - "Defaults to the namespace of the KubeadmControlPlaneTemplate and KubeadmConfigTemplate" + + "Defaults to the namespace of the KubeadmControlPlaneTemplate and KubeadmConfigTemplate " + "that reference this variable.", Type: "string", }, @@ -295,6 +295,8 @@ func (GlobalImageRegistryMirror) VariableSchema() clusterv1.VariableSchema { "url": { Description: "Registry mirror URL.", Type: "string", + Format: "uri", + Pattern: "^https?://", }, "credentials": ImageCredentials{}.VariableSchema().OpenAPIV3Schema, }, @@ -320,6 +322,8 @@ func (ImageRegistry) VariableSchema() clusterv1.VariableSchema { "url": { Description: "Registry URL.", Type: "string", + Format: "uri", + Pattern: "^https?://", }, "credentials": ImageCredentials{}.VariableSchema().OpenAPIV3Schema, }, diff --git a/docs/content/customization/generic/global-mirror.md b/docs/content/customization/generic/global-mirror.md index 74e959a82..9df7f1840 100644 --- a/docs/content/customization/generic/global-mirror.md +++ b/docs/content/customization/generic/global-mirror.md @@ -17,7 +17,7 @@ If your registry mirror requires a private or self-signed CA certificate, create a Kubernetes Secret with the `ca.crt` key populated with the CA certificate in PEM format: ```shell -kubectl create secret generic my-mirror-ca-cert-secret \ +kubectl create secret generic my-mirror-ca-cert \ --from-file=ca.crt=registry-ca.crt ``` @@ -35,7 +35,7 @@ spec: url: https://my-mirror.io credentials: secretRef: - name: my-mirror-ca-cert-secret + name: my-mirror-ca-cert ``` Applying this configuration will result in following new files on the diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go index 6353c9a60..b511775e3 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go @@ -84,7 +84,8 @@ func (h *imageRegistriesPatchHandler) Mutate( // TODO: Add support for multiple registries. if len(imageRegistries) > 1 { - return fmt.Errorf("multiple Image Registry are not supported at this time") + return fmt.Errorf("multiple Image Registry are not supported at this time. "+ + "Provide a single registry entry for %s variable", imageregistries.VariableName) } imageRegistry := imageRegistries[0] diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go index fd687e0af..4a5086d98 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go @@ -36,7 +36,7 @@ func TestVariableValidation(t *testing.T) { Vals: v1alpha1.GenericClusterConfig{ ImageRegistries: []v1alpha1.ImageRegistry{ { - URL: "http://a.b.c.example.com", + URL: "https://a.b.c.example.com/a/b/c", Credentials: &v1alpha1.ImageCredentials{ SecretRef: &corev1.ObjectReference{ Name: "a.b.c.example.com-creds", @@ -46,5 +46,27 @@ func TestVariableValidation(t *testing.T) { }, }, }, + capitest.VariableTestDef{ + Name: "invalid registry URL", + Vals: v1alpha1.GenericClusterConfig{ + ImageRegistries: []v1alpha1.ImageRegistry{ + { + URL: "unsupportedformat://a.b.c.example.com", + }, + }, + }, + ExpectError: true, + }, + capitest.VariableTestDef{ + Name: "registry URL without format", + Vals: v1alpha1.GenericClusterConfig{ + ImageRegistries: []v1alpha1.ImageRegistry{ + { + URL: "a.b.c.example.com/a/b/c", + }, + }, + }, + ExpectError: true, + }, ) } diff --git a/pkg/handlers/generic/mutation/mirrors/variables_test.go b/pkg/handlers/generic/mutation/mirrors/variables_test.go index 5533b9906..0bde9dbbc 100644 --- a/pkg/handlers/generic/mutation/mirrors/variables_test.go +++ b/pkg/handlers/generic/mutation/mirrors/variables_test.go @@ -42,5 +42,23 @@ func TestVariableValidation(t *testing.T) { }, }, }, + capitest.VariableTestDef{ + Name: "invalid mirror registry URL", + Vals: v1alpha1.GenericClusterConfig{ + GlobalImageRegistryMirror: &v1alpha1.GlobalImageRegistryMirror{ + URL: "unsupportedformat://a.b.c.example.com", + }, + }, + ExpectError: true, + }, + capitest.VariableTestDef{ + Name: "mirror URL without format", + Vals: v1alpha1.GenericClusterConfig{ + GlobalImageRegistryMirror: &v1alpha1.GlobalImageRegistryMirror{ + URL: "a.b.c.example.com/a/b/c", + }, + }, + ExpectError: true, + }, ) } From 708d42658f3ba1e5073328346152d577d607b0f8 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Wed, 24 Jan 2024 18:38:38 -0800 Subject: [PATCH 10/12] docs: fix linting errors in the markdown docs --- .markdownlint.json | 3 +-- docs/content/customization/generic/global-mirror.md | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.markdownlint.json b/.markdownlint.json index 2c20f6728..4af4c680f 100644 --- a/.markdownlint.json +++ b/.markdownlint.json @@ -2,6 +2,5 @@ "heading-style": { "style": "atx" }, "ul-style": { "style": "dash" }, "line-length": { "line_length": 120, "stern": true }, - "hr-style": { "style": "---" }, - "MD013": false + "hr-style": { "style": "---" } } diff --git a/docs/content/customization/generic/global-mirror.md b/docs/content/customization/generic/global-mirror.md index 9df7f1840..f2e1894f5 100644 --- a/docs/content/customization/generic/global-mirror.md +++ b/docs/content/customization/generic/global-mirror.md @@ -4,7 +4,8 @@ title = "Global Image Registry Mirror" Add containerd image registry mirror configuration to all Nodes in the cluster. -When the `globalImageRegistryMirror` variable is set, `files` with configurations for [Containerd default mirror](https://github.com/containerd/containerd/blob/main/docs/hosts.md#setup-default-mirror-for-all-registries) will be added. +When the `globalImageRegistryMirror` variable is set, `files` with configurations for +[Containerd default mirror](https://github.com/containerd/containerd/blob/main/docs/hosts.md). This customization will be available when the [provider-specific cluster configuration patch]({{< ref "..">}}) is included in the `ClusterClass`. From e4881adc01eb5b14ec3547ad0262d079d1865b48 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Fri, 2 Feb 2024 14:50:36 -0800 Subject: [PATCH 11/12] fix: schema validation to support only one image registry --- api/v1alpha1/clusterconfig_types.go | 3 + .../imageregistries/credentials/inject.go | 245 +++++++++--------- .../credentials/variables_test.go | 14 + 3 files changed, 138 insertions(+), 124 deletions(-) diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index 94754cbd1..bd6860e3d 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -332,6 +332,8 @@ func (ImageRegistry) VariableSchema() clusterv1.VariableSchema { } } +var maxSupportedImageRegistries int64 = 1 + type ImageRegistries []ImageRegistry func (ImageRegistries) VariableSchema() clusterv1.VariableSchema { @@ -341,6 +343,7 @@ func (ImageRegistries) VariableSchema() clusterv1.VariableSchema { Description: "Configuration for image registries.", Type: "array", Items: &resourceSchema, + MaxItems: &maxSupportedImageRegistries, }, } } diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go index b511775e3..b7a8a9f1e 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go @@ -82,132 +82,129 @@ func (h *imageRegistriesPatchHandler) Mutate( return nil } - // TODO: Add support for multiple registries. - if len(imageRegistries) > 1 { - return fmt.Errorf("multiple Image Registry are not supported at this time. "+ - "Provide a single registry entry for %s variable", imageregistries.VariableName) - } - - imageRegistry := imageRegistries[0] - - log = log.WithValues( - "variableName", - h.variableName, - "variableFieldPath", - h.variableFieldPath, - "variableValue", - imageRegistry, - ) + // TODO: Support for multiple registries is constrained with variable schema of ImageRegistries. + // currently only one registry is supported. Implement support for multiple registries in + // DynamicCredentialProviderConfig + for _, imageRegistry := range imageRegistries { + log = log.WithValues( + "variableName", + h.variableName, + "variableFieldPath", + h.variableFieldPath, + "variableValue", + imageRegistry, + ) - if err := patches.MutateIfApplicable( - obj, vars, &holderRef, selectors.ControlPlane(), log, - func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { - registryWithOptionalCredentials, generateErr := registryWithOptionalCredentialsFromImageRegistryCredentials( - ctx, h.client, imageRegistry, obj, - ) - if generateErr != nil { - return generateErr - } - files, commands, generateErr := generateFilesAndCommands( - registryWithOptionalCredentials, - obj.GetName()) - if generateErr != nil { - return generateErr - } - - log.WithValues( - "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), - "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), - ).Info("adding files to control plane kubeadm config spec") - obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( - obj.Spec.Template.Spec.KubeadmConfigSpec.Files, - files..., - ) - - log.WithValues( - "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), - "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), - ).Info("adding PreKubeadmCommands to control plane kubeadm config spec") - obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands = append( - obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands, - commands..., - ) - - generateErr = createSecretIfNeeded(ctx, h.client, registryWithOptionalCredentials, obj, clusterKey) - if generateErr != nil { - return generateErr - } - - initConfiguration := obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration - if initConfiguration == nil { - initConfiguration = &bootstrapv1.InitConfiguration{} - } - obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration = initConfiguration - if initConfiguration.NodeRegistration.KubeletExtraArgs == nil { - initConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} - } - addImageCredentialProviderArgs(initConfiguration.NodeRegistration.KubeletExtraArgs) - - joinConfiguration := obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration - if joinConfiguration == nil { - joinConfiguration = &bootstrapv1.JoinConfiguration{} - } - obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration = joinConfiguration - if joinConfiguration.NodeRegistration.KubeletExtraArgs == nil { - joinConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} - } - addImageCredentialProviderArgs(joinConfiguration.NodeRegistration.KubeletExtraArgs) - return nil - }); err != nil { - return err - } + if err := patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.ControlPlane(), log, + func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { + registryWithOptionalCredentials, generateErr := registryWithOptionalCredentialsFromImageRegistryCredentials( + ctx, h.client, imageRegistry, obj, + ) + if generateErr != nil { + return generateErr + } + files, commands, generateErr := generateFilesAndCommands( + registryWithOptionalCredentials, + obj.GetName()) + if generateErr != nil { + return generateErr + } + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding files to control plane kubeadm config spec") + obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.Files, + files..., + ) + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding PreKubeadmCommands to control plane kubeadm config spec") + obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands, + commands..., + ) + + generateErr = createSecretIfNeeded(ctx, h.client, registryWithOptionalCredentials, obj, clusterKey) + if generateErr != nil { + return generateErr + } + + initConfiguration := obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration + if initConfiguration == nil { + initConfiguration = &bootstrapv1.InitConfiguration{} + } + obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration = initConfiguration + if initConfiguration.NodeRegistration.KubeletExtraArgs == nil { + initConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} + } + addImageCredentialProviderArgs(initConfiguration.NodeRegistration.KubeletExtraArgs) + + joinConfiguration := obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration + if joinConfiguration == nil { + joinConfiguration = &bootstrapv1.JoinConfiguration{} + } + obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration = joinConfiguration + if joinConfiguration.NodeRegistration.KubeletExtraArgs == nil { + joinConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} + } + addImageCredentialProviderArgs(joinConfiguration.NodeRegistration.KubeletExtraArgs) + return nil + }); err != nil { + return err + } - if err := patches.MutateIfApplicable( - obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log, - func(obj *bootstrapv1.KubeadmConfigTemplate) error { - registryWithOptionalCredentials, generateErr := registryWithOptionalCredentialsFromImageRegistryCredentials( - ctx, h.client, imageRegistry, obj, - ) - if generateErr != nil { - return generateErr - } - files, commands, generateErr := generateFilesAndCommands( - registryWithOptionalCredentials, - obj.GetName()) - if generateErr != nil { - return generateErr - } - - log.WithValues( - "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), - "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), - ).Info("adding files to worker node kubeadm config template") - obj.Spec.Template.Spec.Files = append(obj.Spec.Template.Spec.Files, files...) - - log.WithValues( - "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), - "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), - ).Info("adding PreKubeadmCommands to worker node kubeadm config template") - obj.Spec.Template.Spec.PreKubeadmCommands = append(obj.Spec.Template.Spec.PreKubeadmCommands, commands...) - - generateErr = createSecretIfNeeded(ctx, h.client, registryWithOptionalCredentials, obj, clusterKey) - if generateErr != nil { - return generateErr - } - - joinConfiguration := obj.Spec.Template.Spec.JoinConfiguration - if joinConfiguration == nil { - joinConfiguration = &bootstrapv1.JoinConfiguration{} - } - obj.Spec.Template.Spec.JoinConfiguration = joinConfiguration - if joinConfiguration.NodeRegistration.KubeletExtraArgs == nil { - joinConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} - } - addImageCredentialProviderArgs(joinConfiguration.NodeRegistration.KubeletExtraArgs) - - return nil - }); err != nil { - return err + if err := patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log, + func(obj *bootstrapv1.KubeadmConfigTemplate) error { + registryWithOptionalCredentials, generateErr := registryWithOptionalCredentialsFromImageRegistryCredentials( + ctx, h.client, imageRegistry, obj, + ) + if generateErr != nil { + return generateErr + } + files, commands, generateErr := generateFilesAndCommands( + registryWithOptionalCredentials, + obj.GetName()) + if generateErr != nil { + return generateErr + } + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding files to worker node kubeadm config template") + obj.Spec.Template.Spec.Files = append(obj.Spec.Template.Spec.Files, files...) + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding PreKubeadmCommands to worker node kubeadm config template") + obj.Spec.Template.Spec.PreKubeadmCommands = append(obj.Spec.Template.Spec.PreKubeadmCommands, commands...) + + generateErr = createSecretIfNeeded(ctx, h.client, registryWithOptionalCredentials, obj, clusterKey) + if generateErr != nil { + return generateErr + } + + joinConfiguration := obj.Spec.Template.Spec.JoinConfiguration + if joinConfiguration == nil { + joinConfiguration = &bootstrapv1.JoinConfiguration{} + } + obj.Spec.Template.Spec.JoinConfiguration = joinConfiguration + if joinConfiguration.NodeRegistration.KubeletExtraArgs == nil { + joinConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} + } + addImageCredentialProviderArgs(joinConfiguration.NodeRegistration.KubeletExtraArgs) + + return nil + }); err != nil { + return err + } } return nil diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go index 4a5086d98..47b01ae31 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go @@ -46,6 +46,20 @@ func TestVariableValidation(t *testing.T) { }, }, }, + capitest.VariableTestDef{ + Name: "support for only single image registry", + Vals: v1alpha1.GenericClusterConfig{ + ImageRegistries: []v1alpha1.ImageRegistry{ + { + URL: "http://first-image-registry.example.com", + }, + { + URL: "http://second-image-registry.example.com", + }, + }, + }, + ExpectError: true, + }, capitest.VariableTestDef{ Name: "invalid registry URL", Vals: v1alpha1.GenericClusterConfig{ From 894f42ef3b2cd4590cd9dd227d8e36d7881e529d Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Tue, 6 Feb 2024 12:53:43 +0000 Subject: [PATCH 12/12] refactor: Satisfy review comments --- api/v1alpha1/clusterconfig_types.go | 27 ++++++------ api/v1alpha1/zz_generated.deepcopy.go | 44 +++++++++---------- .../customization/generic/global-mirror.md | 12 ++--- .../credential_provider_config_files_test.go | 2 +- .../credentials/tests/generate_patches.go | 10 ++--- .../credentials/variables_test.go | 2 +- .../generic/mutation/mirrors/mirror_test.go | 4 +- .../mirrors/tests/generate_patches.go | 8 ++-- .../mutation/mirrors/variables_test.go | 2 +- 9 files changed, 55 insertions(+), 56 deletions(-) diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index bd6860e3d..0eed0f94c 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -8,6 +8,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/openapi/patterns" @@ -241,7 +242,7 @@ func (ExtraAPIServerCertSANs) VariableSchema() clusterv1.VariableSchema { } } -type ImageCredentials struct { +type RegistryCredentials struct { // The Secret containing the registry credentials and optional CA certificate // using the keys `username`, `password` and `ca.crt`. // This credentials Secret is not required for some registries, e.g. ECR. @@ -249,14 +250,14 @@ type ImageCredentials struct { SecretRef *corev1.ObjectReference `json:"secretRef,omitempty"` } -func (ImageCredentials) VariableSchema() clusterv1.VariableSchema { +func (RegistryCredentials) VariableSchema() clusterv1.VariableSchema { return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ "secretRef": { - Description: "The Secret containing the registry credentials. " + - "The Secret should have keys 'username', 'password' and optional 'ca.crt' " + + Description: "A reference to the Secret containing the registry credentials. " + + "The Secret should have keys 'username', 'password' and optional 'ca.crt'. " + "This credentials Secret is not required for some registries, e.g. ECR.", Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ @@ -266,11 +267,12 @@ func (ImageCredentials) VariableSchema() clusterv1.VariableSchema { }, "namespace": { Description: "The namespace of the Secret containing the registry credentials. " + - "Defaults to the namespace of the KubeadmControlPlaneTemplate and KubeadmConfigTemplate " + + "Defaults to the namespace of the Cluster. " + "that reference this variable.", Type: "string", }, }, + Required: []string{"name"}, }, }, }, @@ -284,7 +286,7 @@ type GlobalImageRegistryMirror struct { // Credentials and CA certificate for the image registry mirror // +optional - Credentials *ImageCredentials `json:"credentials,omitempty"` + Credentials *RegistryCredentials `json:"credentials,omitempty"` } func (GlobalImageRegistryMirror) VariableSchema() clusterv1.VariableSchema { @@ -298,7 +300,7 @@ func (GlobalImageRegistryMirror) VariableSchema() clusterv1.VariableSchema { Format: "uri", Pattern: "^https?://", }, - "credentials": ImageCredentials{}.VariableSchema().OpenAPIV3Schema, + "credentials": RegistryCredentials{}.VariableSchema().OpenAPIV3Schema, }, Required: []string{"url"}, }, @@ -311,7 +313,7 @@ type ImageRegistry struct { // Credentials and CA certificate for the image registry // +optional - Credentials *ImageCredentials `json:"credentials,omitempty"` + Credentials *RegistryCredentials `json:"credentials,omitempty"` } func (ImageRegistry) VariableSchema() clusterv1.VariableSchema { @@ -325,25 +327,22 @@ func (ImageRegistry) VariableSchema() clusterv1.VariableSchema { Format: "uri", Pattern: "^https?://", }, - "credentials": ImageCredentials{}.VariableSchema().OpenAPIV3Schema, + "credentials": RegistryCredentials{}.VariableSchema().OpenAPIV3Schema, }, Required: []string{"url"}, }, } } -var maxSupportedImageRegistries int64 = 1 - type ImageRegistries []ImageRegistry func (ImageRegistries) VariableSchema() clusterv1.VariableSchema { - resourceSchema := ImageRegistry{}.VariableSchema().OpenAPIV3Schema return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ Description: "Configuration for image registries.", Type: "array", - Items: &resourceSchema, - MaxItems: &maxSupportedImageRegistries, + Items: ptr.To(ImageRegistry{}.VariableSchema().OpenAPIV3Schema), + MaxItems: ptr.To[int64](1), }, } } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6f5eafcab..76d67d4bc 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -484,7 +484,7 @@ func (in *GlobalImageRegistryMirror) DeepCopyInto(out *GlobalImageRegistryMirror *out = *in if in.Credentials != nil { in, out := &in.Credentials, &out.Credentials - *out = new(ImageCredentials) + *out = new(RegistryCredentials) (*in).DeepCopyInto(*out) } } @@ -534,26 +534,6 @@ func (in *Image) DeepCopy() *Image { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ImageCredentials) DeepCopyInto(out *ImageCredentials) { - *out = *in - if in.SecretRef != nil { - in, out := &in.SecretRef, &out.SecretRef - *out = new(v1.ObjectReference) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageCredentials. -func (in *ImageCredentials) DeepCopy() *ImageCredentials { - if in == nil { - return nil - } - out := new(ImageCredentials) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in ImageRegistries) DeepCopyInto(out *ImageRegistries) { { @@ -580,7 +560,7 @@ func (in *ImageRegistry) DeepCopyInto(out *ImageRegistry) { *out = *in if in.Credentials != nil { in, out := &in.Credentials, &out.Credentials - *out = new(ImageCredentials) + *out = new(RegistryCredentials) (*in).DeepCopyInto(*out) } } @@ -690,6 +670,26 @@ func (in *ObjectMeta) DeepCopy() *ObjectMeta { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RegistryCredentials) DeepCopyInto(out *RegistryCredentials) { + *out = *in + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(v1.ObjectReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RegistryCredentials. +func (in *RegistryCredentials) DeepCopy() *RegistryCredentials { + if in == nil { + return nil + } + out := new(RegistryCredentials) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecurityGroup) DeepCopyInto(out *SecurityGroup) { *out = *in diff --git a/docs/content/customization/generic/global-mirror.md b/docs/content/customization/generic/global-mirror.md index f2e1894f5..a653b53e2 100644 --- a/docs/content/customization/generic/global-mirror.md +++ b/docs/content/customization/generic/global-mirror.md @@ -12,9 +12,9 @@ This customization will be available when the ## Example -To provide image registry mirror with CA certificate, specify the following configuration: +To provide an image registry mirror with a CA certificate, specify the following configuration: -If your registry mirror requires a private or self-signed CA certificate, +If the registry mirror requires a private or self-signed CA certificate, create a Kubernetes Secret with the `ca.crt` key populated with the CA certificate in PEM format: ```shell @@ -33,19 +33,19 @@ spec: - name: clusterConfig value: globalImageRegistryMirror: - url: https://my-mirror.io + url: https://example.com credentials: secretRef: name: my-mirror-ca-cert ``` Applying this configuration will result in following new files on the -`KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate` +`KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate` resources: - `/etc/containerd/certs.d/_default/hosts.toml` - `/etc/certs/mirror.pem` -To use a public hosted image registry (ex. ECR) as mirror, specify the following configuration: +To use a public hosted image registry (e.g. ECR) as a registry mirror, specify the following configuration: ```yaml apiVersion: cluster.x-k8s.io/v1beta1 @@ -62,6 +62,6 @@ spec: ``` Applying this configuration will result in following new files on the -`KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate` +`KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate` resources: - `/etc/containerd/certs.d/_default/hosts.toml` diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go index e8ce8dd87..09dc598ec 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go @@ -192,7 +192,7 @@ credentialProviders: { name: "error for a registry with no credentials", credentials: providerConfig{ - URL: "https://myregistry.com", + URL: "https://registry.example.com", }, wantErr: ErrCredentialsNotFound, }, diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go index 40aa0027e..13b51e7bf 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go @@ -140,8 +140,8 @@ func TestGeneratePatches( variableName, v1alpha1.ImageRegistries{ v1alpha1.ImageRegistry{ - URL: "https://my-registry.io", - Credentials: &v1alpha1.ImageCredentials{ + URL: "https://registry.example.com", + Credentials: &v1alpha1.RegistryCredentials{ SecretRef: &corev1.ObjectReference{ Name: validSecretName, }, @@ -258,8 +258,8 @@ func TestGeneratePatches( variableName, v1alpha1.ImageRegistries{ v1alpha1.ImageRegistry{ - URL: "https://my-registry.io", - Credentials: &v1alpha1.ImageCredentials{ + URL: "https://registry.example.com", + Credentials: &v1alpha1.RegistryCredentials{ SecretRef: &corev1.ObjectReference{ Name: validSecretName, }, @@ -321,7 +321,7 @@ func TestGeneratePatches( variableName, v1alpha1.ImageRegistries{ v1alpha1.ImageRegistry{ - URL: "https://my-registry.io", + URL: "https://registry.example.com", }, }, variablePath..., diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go index 47b01ae31..2c416bcf7 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go @@ -37,7 +37,7 @@ func TestVariableValidation(t *testing.T) { ImageRegistries: []v1alpha1.ImageRegistry{ { URL: "https://a.b.c.example.com/a/b/c", - Credentials: &v1alpha1.ImageCredentials{ + Credentials: &v1alpha1.RegistryCredentials{ SecretRef: &corev1.ObjectReference{ Name: "a.b.c.example.com-creds", }, diff --git a/pkg/handlers/generic/mutation/mirrors/mirror_test.go b/pkg/handlers/generic/mutation/mirrors/mirror_test.go index 855527a44..f9693579c 100644 --- a/pkg/handlers/generic/mutation/mirrors/mirror_test.go +++ b/pkg/handlers/generic/mutation/mirrors/mirror_test.go @@ -96,9 +96,9 @@ func Test_generateMirrorCACertFile(t *testing.T) { CACert: "mycacert", }, registry: v1alpha1.GlobalImageRegistryMirror{ - URL: "https://myregistry.com", + URL: "https://registry.example.com", - Credentials: &v1alpha1.ImageCredentials{ + Credentials: &v1alpha1.RegistryCredentials{ SecretRef: &v1.ObjectReference{ Name: "my-registry-credentials-secret", }, diff --git a/pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go b/pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go index 10de0fbd2..86c749d11 100644 --- a/pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go +++ b/pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go @@ -79,8 +79,8 @@ func TestGeneratePatches( capitest.VariableWithValue( variableName, v1alpha1.GlobalImageRegistryMirror{ - URL: "https://mirror-registry.com", - Credentials: &v1alpha1.ImageCredentials{ + URL: "https://registry.example.com", + Credentials: &v1alpha1.RegistryCredentials{ SecretRef: &corev1.ObjectReference{ Name: validMirrorCASecretName, }, @@ -143,8 +143,8 @@ func TestGeneratePatches( capitest.VariableWithValue( variableName, v1alpha1.GlobalImageRegistryMirror{ - URL: "https://mirror-registry.io", - Credentials: &v1alpha1.ImageCredentials{ + URL: "https://registry.example.com", + Credentials: &v1alpha1.RegistryCredentials{ SecretRef: &corev1.ObjectReference{ Name: validMirrorCASecretName, }, diff --git a/pkg/handlers/generic/mutation/mirrors/variables_test.go b/pkg/handlers/generic/mutation/mirrors/variables_test.go index 0bde9dbbc..332ddc815 100644 --- a/pkg/handlers/generic/mutation/mirrors/variables_test.go +++ b/pkg/handlers/generic/mutation/mirrors/variables_test.go @@ -34,7 +34,7 @@ func TestVariableValidation(t *testing.T) { Vals: v1alpha1.GenericClusterConfig{ GlobalImageRegistryMirror: &v1alpha1.GlobalImageRegistryMirror{ URL: "http://a.b.c.example.com", - Credentials: &v1alpha1.ImageCredentials{ + Credentials: &v1alpha1.RegistryCredentials{ SecretRef: &corev1.ObjectReference{ Name: "a.b.c.example.com-ca-cert-creds", },