From 4dd5b1ff42bd056c375566cf6c7f079e8ec150eb Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Mon, 22 Jan 2024 18:48:23 -0800 Subject: [PATCH] feat: global image registry mirror variable --- api/v1alpha1/clusterconfig_types.go | 52 +++--- api/v1alpha1/zz_generated.deepcopy.go | 50 +++--- .../customization/generic/global-mirror.md | 66 +++++++ .../aws/mutation/metapatch_handler_test.go | 6 +- .../docker/mutation/metapatch_handler_test.go | 6 +- pkg/handlers/generic/mutation/handlers.go | 2 + .../credential_provider_config_files.go | 9 +- .../credential_provider_config_files_test.go | 70 +------- .../imageregistries/credentials/inject.go | 37 ---- ...mic-credential-provider-config.yaml.gotmpl | 7 - .../credentials/tests/generate_patches.go | 4 +- .../credentials/variables_test.go | 15 -- .../generic/mutation/mirrors/constants.go | 8 + .../generic/mutation/mirrors/inject.go | 169 ++++++++++++++++++ .../credentials => mirrors}/mirror.go | 39 ++-- .../credentials => mirrors}/mirror_test.go | 13 +- .../templates/hosts.toml.gotmpl | 0 .../tests/generate_patches.go} | 97 ++-------- .../mutation/mirrors/variables_test.go | 46 +++++ 19 files changed, 389 insertions(+), 307 deletions(-) create mode 100644 docs/content/customization/generic/global-mirror.md create mode 100644 pkg/handlers/generic/mutation/mirrors/constants.go create mode 100644 pkg/handlers/generic/mutation/mirrors/inject.go rename pkg/handlers/generic/mutation/{imageregistries/credentials => mirrors}/mirror.go (77%) rename pkg/handlers/generic/mutation/{imageregistries/credentials => mirrors}/mirror_test.go (91%) rename pkg/handlers/generic/mutation/{imageregistries/credentials => mirrors}/templates/hosts.toml.gotmpl (100%) rename pkg/handlers/generic/mutation/{imageregistries/credentials/tests/generate_mirror_patches.go => mirrors/tests/generate_patches.go} (68%) create mode 100644 pkg/handlers/generic/mutation/mirrors/variables_test.go diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index 1df74b3bd..aec3a87f4 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -90,6 +90,9 @@ type GenericClusterConfig struct { // +optional ImageRegistries ImageRegistries `json:"imageRegistries,omitempty"` + // +optional + GlobalImageRegistryMirror *GlobalImageRegistryMirror `json:"globalImageRegistryMirror,omitempty"` + // +optional Addons *Addons `json:"addons,omitempty"` } @@ -108,7 +111,8 @@ func (s GenericClusterConfig) VariableSchema() clusterv1.VariableSchema { //noli "", ).VariableSchema(). OpenAPIV3Schema, - "imageRegistries": ImageRegistries{}.VariableSchema().OpenAPIV3Schema, + "imageRegistries": ImageRegistries{}.VariableSchema().OpenAPIV3Schema, + "globalImageRegistryMirror": GlobalImageRegistryMirror{}.VariableSchema().OpenAPIV3Schema, }, }, } @@ -240,7 +244,7 @@ func (ExtraAPIServerCertSANs) VariableSchema() clusterv1.VariableSchema { type ImageCredentials struct { // The Secret containing the registry credentials and CA certificate - // The Secret should have keys 'username', 'password' and 'caCert' + // The Secret should have keys 'username', 'password' and 'ca.crt' // This credentials Secret is not required for some registries, e.g. ECR. // +optional SecretRef *corev1.ObjectReference `json:"secretRef,omitempty"` @@ -253,7 +257,7 @@ func (ImageCredentials) VariableSchema() clusterv1.VariableSchema { Properties: map[string]clusterv1.JSONSchemaProps{ "secretRef": { Description: "The Secret containing the registry credentials. " + - "The Secret should have keys 'username', 'password'. " + + "The Secret should have keys 'username', 'password' and 'ca.crt' " + "This credentials Secret is not required for some registries, e.g. ECR.", Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ @@ -274,37 +278,28 @@ func (ImageCredentials) VariableSchema() clusterv1.VariableSchema { } } -type RegistryMirror struct { - // The secret containing CA certificate for the registry mirror. - // The secret should have 'ca.crt' key +// GlobalImageRegistryMirror sets default mirror configuration for all the image registries. +type GlobalImageRegistryMirror struct { + // Registry URL. + URL string `json:"url"` + + // Credentials and CA certificate for the image registry mirror // +optional - SecretRef *corev1.ObjectReference `json:"secretRef,omitempty"` + Credentials *ImageCredentials `json:"credentials,omitempty"` } -func (RegistryMirror) VariableSchema() clusterv1.VariableSchema { +func (GlobalImageRegistryMirror) VariableSchema() clusterv1.VariableSchema { return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ - "secretRef": { - Description: "The Secret containing the registry CA certificate. " + - "The Secret should have keys 'ca.crt'. " + - "This credentials Secret is not required for public registries.", - Type: "object", - Properties: map[string]clusterv1.JSONSchemaProps{ - "name": { - Description: "The name of the Secret containing the registry CA certificate.", - Type: "string", - }, - "namespace": { - Description: "The namespace of the Secret containing the registry CA certificate. " + - "Defaults to the namespace of the KubeadmControlPlaneTemplate and KubeadmConfigTemplate" + - " that reference this variable.", - Type: "string", - }, - }, + "url": { + Description: "Registry mirror URL.", + Type: "string", }, + "credentials": ImageCredentials{}.VariableSchema().OpenAPIV3Schema, }, + Required: []string{"url"}, }, } } @@ -313,13 +308,9 @@ type ImageRegistry struct { // Registry URL. URL string `json:"url"` - // Credentials for the image registry + // Credentials and CA certificate for the image registry // +optional Credentials *ImageCredentials `json:"credentials,omitempty"` - - // Use this registry as a mirror - // +optional - Mirror *RegistryMirror `json:"mirror,omitempty"` } func (ImageRegistry) VariableSchema() clusterv1.VariableSchema { @@ -332,7 +323,6 @@ func (ImageRegistry) VariableSchema() clusterv1.VariableSchema { Type: "string", }, "credentials": ImageCredentials{}.VariableSchema().OpenAPIV3Schema, - "mirror": RegistryMirror{}.VariableSchema().OpenAPIV3Schema, }, Required: []string{"url"}, }, diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index c9fa1330a..6f5eafcab 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -442,6 +442,11 @@ func (in *GenericClusterConfig) DeepCopyInto(out *GenericClusterConfig) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.GlobalImageRegistryMirror != nil { + in, out := &in.GlobalImageRegistryMirror, &out.GlobalImageRegistryMirror + *out = new(GlobalImageRegistryMirror) + (*in).DeepCopyInto(*out) + } if in.Addons != nil { in, out := &in.Addons, &out.Addons *out = new(Addons) @@ -474,6 +479,26 @@ func (in *GenericNodeConfig) DeepCopy() *GenericNodeConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalImageRegistryMirror) DeepCopyInto(out *GlobalImageRegistryMirror) { + *out = *in + if in.Credentials != nil { + in, out := &in.Credentials, &out.Credentials + *out = new(ImageCredentials) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalImageRegistryMirror. +func (in *GlobalImageRegistryMirror) DeepCopy() *GlobalImageRegistryMirror { + if in == nil { + return nil + } + out := new(GlobalImageRegistryMirror) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPProxy) DeepCopyInto(out *HTTPProxy) { *out = *in @@ -558,11 +583,6 @@ func (in *ImageRegistry) DeepCopyInto(out *ImageRegistry) { *out = new(ImageCredentials) (*in).DeepCopyInto(*out) } - if in.Mirror != nil { - in, out := &in.Mirror, &out.Mirror - *out = new(RegistryMirror) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistry. @@ -670,26 +690,6 @@ func (in *ObjectMeta) DeepCopy() *ObjectMeta { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RegistryMirror) DeepCopyInto(out *RegistryMirror) { - *out = *in - if in.SecretRef != nil { - in, out := &in.SecretRef, &out.SecretRef - *out = new(v1.ObjectReference) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RegistryMirror. -func (in *RegistryMirror) DeepCopy() *RegistryMirror { - if in == nil { - return nil - } - out := new(RegistryMirror) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecurityGroup) DeepCopyInto(out *SecurityGroup) { *out = *in diff --git a/docs/content/customization/generic/global-mirror.md b/docs/content/customization/generic/global-mirror.md new file mode 100644 index 000000000..b54f2f3bb --- /dev/null +++ b/docs/content/customization/generic/global-mirror.md @@ -0,0 +1,66 @@ ++++ +title = "Global Image Registry Mirror" ++++ + +Add containerd image registry mirror configuration to all Nodes in the cluster. + +When the `globalImageRegistryMirror` variable is set, `files` with configurations for +[Containerd default mirror](https://github.com/containerd/containerd/blob/main/docs/hosts.md#setup-default-mirror-for-all-registries) will be added. + +This customization will be available when the +[provider-specific cluster configuration patch]({{< ref "..">}}) is included in the `ClusterClass`. + +## Example + +To provide image registry mirror with CA certificate, specify the following configuration: + +If your registry mirror requires self signed CA certifate, create a Kubernetes Secret with keys for `ca.crt`: + +```shell +kubectl create secret generic my-mirror-ca-cert-secret \ + --from-file=ca.crt=registry-ca.crt +``` + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: clusterConfig + value: + globalImageRegistryMirror: + url: https://my-mirror.io + credentials: + secretRef: + name: my-mirror-ca-cert-secret +``` + +Applying this configuration will result in following new files on the +`KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate` + +- `/etc/containerd/certs.d/_default/hosts.toml` +- `/etc/certs/mirror.pem` + +To use a public hosted image registry (ex. ECR) as mirror, specify the following configuration: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: clusterConfig + value: + globalImageRegistryMirror: + url: https://123456789.dkr.ecr.us-east-1.amazonaws.com +``` + +Applying this configuration will result in following new files on the +`KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate` + +- `/etc/containerd/certs.d/_default/hosts.toml` diff --git a/pkg/handlers/aws/mutation/metapatch_handler_test.go b/pkg/handlers/aws/mutation/metapatch_handler_test.go index ea952363d..2e8e63b5a 100644 --- a/pkg/handlers/aws/mutation/metapatch_handler_test.go +++ b/pkg/handlers/aws/mutation/metapatch_handler_test.go @@ -38,6 +38,8 @@ import ( imageregistrycredentialstests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository" kubernetesimagerepositorytests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository/tests" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors" + globalimageregistrymirrortests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/workerconfig" ) @@ -156,12 +158,12 @@ func TestGeneratePatches(t *testing.T) { imageregistries.VariableName, ) - imageregistrycredentialstests.TestGenerateMirrorPatches( + globalimageregistrymirrortests.TestGeneratePatches( t, metaPatchGeneratorFunc(mgr), mgr.GetClient(), clusterconfig.MetaVariableName, - imageregistries.VariableName, + mirrors.GlobalMirrorVariableName, ) amitests.TestControlPlaneGeneratePatches( diff --git a/pkg/handlers/docker/mutation/metapatch_handler_test.go b/pkg/handlers/docker/mutation/metapatch_handler_test.go index d3dea97f2..6fd017258 100644 --- a/pkg/handlers/docker/mutation/metapatch_handler_test.go +++ b/pkg/handlers/docker/mutation/metapatch_handler_test.go @@ -28,6 +28,8 @@ import ( imageregistrycredentialstests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository" kubernetesimagerepositorytests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository/tests" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors" + globalimageregistrymirrortests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/workerconfig" ) @@ -113,11 +115,11 @@ func TestGeneratePatches(t *testing.T) { imageregistries.VariableName, ) - imageregistrycredentialstests.TestGenerateMirrorPatches( + globalimageregistrymirrortests.TestGeneratePatches( t, metaPatchGeneratorFunc(mgr), mgr.GetClient(), clusterconfig.MetaVariableName, - imageregistries.VariableName, + mirrors.GlobalMirrorVariableName, ) } diff --git a/pkg/handlers/generic/mutation/handlers.go b/pkg/handlers/generic/mutation/handlers.go index aa2160737..7ac6f66f6 100644 --- a/pkg/handlers/generic/mutation/handlers.go +++ b/pkg/handlers/generic/mutation/handlers.go @@ -14,6 +14,7 @@ import ( "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/httpproxy" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors" ) // MetaMutators returns all generic patch handlers. @@ -25,6 +26,7 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator { httpproxy.NewPatch(mgr.GetClient()), kubernetesimagerepository.NewPatch(), credentials.NewPatch(mgr.GetClient()), + mirrors.NewPatch(mgr.GetClient()), calico.NewPatch(), } } diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go index 9639c37c7..fe3d5c6c2 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go @@ -52,10 +52,7 @@ func (c providerConfig) isCredentialsEmpty() bool { c.Password == "" } -func templateFilesForImageCredentialProviderConfigs( - config providerConfig, - mirror *mirrorConfig, -) ([]cabpkv1.File, error) { +func templateFilesForImageCredentialProviderConfigs(config providerConfig) ([]cabpkv1.File, error) { var files []cabpkv1.File kubeletCredentialProviderConfigFile, err := templateKubeletCredentialProviderConfig() @@ -68,7 +65,6 @@ func templateFilesForImageCredentialProviderConfigs( kubeletDynamicCredentialProviderConfigFile, err := templateDynamicCredentialProviderConfig( config, - mirror, ) if err != nil { return nil, err @@ -104,7 +100,6 @@ func templateKubeletCredentialProviderConfig() (*cabpkv1.File, error) { func templateDynamicCredentialProviderConfig( config providerConfig, - mirror *mirrorConfig, ) (*cabpkv1.File, error) { registryURL, err := url.ParseRequestURI(config.URL) if err != nil { @@ -142,13 +137,11 @@ func templateDynamicCredentialProviderConfig( ProviderBinary string ProviderArgs []string ProviderAPIVersion string - Mirror *mirrorConfig }{ RegistryHost: registryHostWithPath, ProviderBinary: providerBinary, ProviderArgs: providerArgs, ProviderAPIVersion: providerAPIVersion, - Mirror: mirror, } return fileFromTemplate(t, templateInput, kubeletDynamicCredentialProviderConfigOnRemote) diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go index c73f564e5..e8ce8dd87 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go @@ -99,7 +99,6 @@ func Test_templateDynamicCredentialProviderConfig(t *testing.T) { tests := []struct { name string credentials providerConfig - mirror *mirrorConfig want *cabpkv1.File wantErr error }{ @@ -190,73 +189,6 @@ credentialProviders: `, }, }, - - { - name: "ECR image registry used as mirror", - credentials: providerConfig{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, - mirror: &mirrorConfig{}, - want: &cabpkv1.File{ - Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", - Owner: "", - Permissions: "0600", - Encoding: "", - Append: false, - Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 -kind: DynamicCredentialProviderConfig -mirror: - endpoint: "123456789.dkr.ecr.us-east-1.amazonaws.com" - credentialsStrategy: "MirrorCredentialsOnly" -credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ -credentialProviders: - apiVersion: kubelet.config.k8s.io/v1 - kind: CredentialProviderConfig - providers: - - name: ecr-credential-provider - args: - - get-credentials - matchImages: - - "123456789.dkr.ecr.us-east-1.amazonaws.com" - defaultCacheDuration: "0s" - apiVersion: credentialprovider.kubelet.k8s.io/v1 -`, - }, - }, - { - name: "image registry with static credentials used as mirror", - credentials: providerConfig{ - URL: "https://myregistry.com", - Username: "myuser", - Password: "mypassword", - }, - mirror: &mirrorConfig{ - CACert: "my-ca-cert", - }, - want: &cabpkv1.File{ - Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", - Owner: "", - Permissions: "0600", - Encoding: "", - Append: false, - Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 -kind: DynamicCredentialProviderConfig -mirror: - endpoint: "myregistry.com" - credentialsStrategy: "MirrorCredentialsOnly" -credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ -credentialProviders: - apiVersion: kubelet.config.k8s.io/v1 - kind: CredentialProviderConfig - providers: - - name: static-credential-provider - args: - - /etc/kubernetes/static-image-credentials.json - matchImages: - - "myregistry.com" - defaultCacheDuration: "0s" - apiVersion: credentialprovider.kubelet.k8s.io/v1 -`, - }, - }, { name: "error for a registry with no credentials", credentials: providerConfig{ @@ -269,7 +201,7 @@ credentialProviders: tt := tests[idx] t.Run(tt.name, func(t *testing.T) { t.Parallel() - file, err := templateDynamicCredentialProviderConfig(tt.credentials, tt.mirror) + file, err := templateDynamicCredentialProviderConfig(tt.credentials) assert.ErrorIs(t, err, tt.wantErr) assert.Equal(t, tt.want, file) }) diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go index 6c8cff9f9..6353c9a60 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go @@ -107,19 +107,8 @@ func (h *imageRegistriesPatchHandler) Mutate( if generateErr != nil { return generateErr } - mirrorConfig, err := mirrorFromImageRegistry( - ctx, - h.client, - imageRegistry, - obj, - ) - if err != nil { - return err - } files, commands, generateErr := generateFilesAndCommands( registryWithOptionalCredentials, - mirrorConfig, - imageRegistry, obj.GetName()) if generateErr != nil { return generateErr @@ -181,19 +170,8 @@ func (h *imageRegistriesPatchHandler) Mutate( if generateErr != nil { return generateErr } - mirrorConfig, err := mirrorFromImageRegistry( - ctx, - h.client, - imageRegistry, - obj, - ) - if err != nil { - return err - } files, commands, generateErr := generateFilesAndCommands( registryWithOptionalCredentials, - mirrorConfig, - imageRegistry, obj.GetName()) if generateErr != nil { return generateErr @@ -268,8 +246,6 @@ func registryWithOptionalCredentialsFromImageRegistryCredentials( func generateFilesAndCommands( registryWithOptionalCredentials providerConfig, - mirrorConfig *mirrorConfig, - imageRegistry v1alpha1.ImageRegistry, objName string, ) ([]bootstrapv1.File, []string, error) { files, commands, err := templateFilesAndCommandsForInstallKubeletCredentialProviders() @@ -281,7 +257,6 @@ func generateFilesAndCommands( } imageCredentialProviderConfigFiles, err := templateFilesForImageCredentialProviderConfigs( registryWithOptionalCredentials, - mirrorConfig, ) if err != nil { return nil, nil, fmt.Errorf( @@ -293,18 +268,6 @@ func generateFilesAndCommands( files = append( files, generateCredentialsSecretFile(registryWithOptionalCredentials, objName)...) - - // Generate default registry mirror file - mirrorHostFiles, err := generateDefaultRegistryMirrorFile(mirrorConfig) - if err != nil { - return nil, nil, err - } - files = append(files, mirrorHostFiles...) - // generate CA certificate file for registry mirror - files = append( - files, - generateMirrorCACertFile(mirrorConfig, imageRegistry)...) - return files, commands, err } diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl index 196203996..80156c219 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl @@ -1,12 +1,5 @@ apiVersion: credentialprovider.d2iq.com/v1alpha1 kind: DynamicCredentialProviderConfig -{{- if .Mirror }} -mirror: - {{- with .RegistryHost }} - endpoint: {{ printf "%q" . }} - {{- end }} - credentialsStrategy: "MirrorCredentialsOnly" -{{- end }} credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ credentialProviders: apiVersion: kubelet.config.k8s.io/v1 diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go index 764048397..40aa0027e 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go @@ -27,7 +27,8 @@ const ( //nolint:gosec // Does not contain hard coded credentials. cpRegistryCreds = "kubeadmControlPlaneRegistryWithCredentials" //nolint:gosec // Does not contain hard coded credentials. - workerRegistryCreds = "kubeadmConfigTemplateRegistryWithCreds" + workerRegistryCreds = "kubeadmConfigTemplateRegistryWithCreds" + registryStaticCredentialsSecretSuffix = "registry-config" ) func TestGeneratePatches( @@ -351,7 +352,6 @@ func newRegistryCredentialsSecret(name, namespace string) *corev1.Secret { } } -//nolint:unparam //namespace can change in future testcases func newEmptySecret(name, namespace string) *corev1.Secret { return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go index 16562b92f..fd687e0af 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go @@ -46,20 +46,5 @@ func TestVariableValidation(t *testing.T) { }, }, }, - capitest.VariableTestDef{ - Name: "with a mirror secret", - Vals: v1alpha1.GenericClusterConfig{ - ImageRegistries: []v1alpha1.ImageRegistry{ - { - URL: "http://a.b.c.example.com", - Mirror: &v1alpha1.RegistryMirror{ - SecretRef: &corev1.ObjectReference{ - Name: "a.b.c.example.com-creds", - }, - }, - }, - }, - }, - }, ) } diff --git a/pkg/handlers/generic/mutation/mirrors/constants.go b/pkg/handlers/generic/mutation/mirrors/constants.go new file mode 100644 index 000000000..13639af8c --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/constants.go @@ -0,0 +1,8 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +const ( + GlobalMirrorVariableName = "globalImageRegistryMirror" +) diff --git a/pkg/handlers/generic/mutation/mirrors/inject.go b/pkg/handlers/generic/mutation/mirrors/inject.go new file mode 100644 index 000000000..e9c42320f --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/inject.go @@ -0,0 +1,169 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +import ( + "context" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/patches" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/patches/selectors" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/variables" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/clusterconfig" +) + +type globalMirrorPatchHandler struct { + client ctrlclient.Client + + variableName string + variableFieldPath []string +} + +func NewPatch( + cl ctrlclient.Client, +) *globalMirrorPatchHandler { + return newGlobalMirrorPatchHandler( + cl, + clusterconfig.MetaVariableName, + GlobalMirrorVariableName, + ) +} + +func newGlobalMirrorPatchHandler( + cl ctrlclient.Client, + variableName string, + variableFieldPath ...string, +) *globalMirrorPatchHandler { + scheme := runtime.NewScheme() + _ = bootstrapv1.AddToScheme(scheme) + _ = controlplanev1.AddToScheme(scheme) + return &globalMirrorPatchHandler{ + client: cl, + variableName: variableName, + variableFieldPath: variableFieldPath, + } +} + +func (h *globalMirrorPatchHandler) Mutate( + ctx context.Context, + obj *unstructured.Unstructured, + vars map[string]apiextensionsv1.JSON, + holderRef runtimehooksv1.HolderReference, + clusterKey ctrlclient.ObjectKey, +) error { + log := ctrl.LoggerFrom(ctx).WithValues( + "holderRef", holderRef, + ) + + globalMirror, found, err := variables.Get[v1alpha1.GlobalImageRegistryMirror]( + vars, + h.variableName, + h.variableFieldPath..., + ) + if err != nil { + return err + } + if !found { + log.V(5).Info("Global registry mirror variable not defined") + return nil + } + + log = log.WithValues( + "variableName", + h.variableName, + "variableFieldPath", + h.variableFieldPath, + "variableValue", + globalMirror, + ) + + if err := patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.ControlPlane(), log, + func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { + mirrorConfig, err := mirrorConfigForGlobalMirror( + ctx, + h.client, + globalMirror, + obj, + ) + if err != nil { + return err + } + files, generateErr := generateFiles( + mirrorConfig, + globalMirror) + if generateErr != nil { + return generateErr + } + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding global registry mirror files to control plane kubeadm config spec") + obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.Files, + files..., + ) + + return nil + }); err != nil { + return err + } + + if err := patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log, + func(obj *bootstrapv1.KubeadmConfigTemplate) error { + mirrorConfig, err := mirrorConfigForGlobalMirror( + ctx, + h.client, + globalMirror, + obj, + ) + if err != nil { + return err + } + files, generateErr := generateFiles( + mirrorConfig, + globalMirror) + if generateErr != nil { + return generateErr + } + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding global registry mirror files to worker node kubeadm config template") + obj.Spec.Template.Spec.Files = append(obj.Spec.Template.Spec.Files, files...) + return nil + }); err != nil { + return err + } + + return nil +} + +func generateFiles( + mirrorConfig *mirrorConfig, + globalMirror v1alpha1.GlobalImageRegistryMirror, +) ([]bootstrapv1.File, error) { + // Generate default registry mirror file + files, err := generateGlobalRegistryMirrorFile(mirrorConfig) + if err != nil { + return nil, err + } + // generate CA certificate file for registry mirror + mirrorCAFile := generateMirrorCACertFile(mirrorConfig, globalMirror) + files = append(files, mirrorCAFile...) + + return files, err +} diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go b/pkg/handlers/generic/mutation/mirrors/mirror.go similarity index 77% rename from pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go rename to pkg/handlers/generic/mutation/mirrors/mirror.go index 0fe4c6e62..409d0fea9 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror.go +++ b/pkg/handlers/generic/mutation/mirrors/mirror.go @@ -1,7 +1,7 @@ // Copyright 2023 D2iQ, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package credentials +package mirrors import ( "bytes" @@ -31,34 +31,26 @@ type mirrorConfig struct { CACert string } -func mirrorFromImageRegistry( +func mirrorConfigForGlobalMirror( ctx context.Context, c ctrlclient.Client, - imageRegistry v1alpha1.ImageRegistry, + globalMirror v1alpha1.GlobalImageRegistryMirror, obj ctrlclient.Object, ) (*mirrorConfig, error) { - // using the registry as a mirror is supported by including empty mirror object or - // mirror with CA certificate to the registry variable. - // ex. - // - url: https://my-registry.com - // mirror: {} - if imageRegistry.Mirror == nil { - return nil, nil - } mirrorWithOptionalCACert := &mirrorConfig{ - URL: imageRegistry.URL, + URL: globalMirror.URL, } secret, err := secretForMirrorCACert( ctx, c, - imageRegistry, + globalMirror, obj.GetNamespace(), ) if err != nil { return &mirrorConfig{}, fmt.Errorf( "error getting secret %s/%s from Image Registry variable: %w", obj.GetNamespace(), - imageRegistry.Mirror.SecretRef.Name, + globalMirror.Credentials.SecretRef.Name, err, ) } @@ -75,20 +67,20 @@ func mirrorFromImageRegistry( func secretForMirrorCACert( ctx context.Context, c ctrlclient.Reader, - registry v1alpha1.ImageRegistry, + globalMirror v1alpha1.GlobalImageRegistryMirror, objectNamespace string, ) (*corev1.Secret, error) { - if registry.Mirror == nil || registry.Mirror.SecretRef == nil { + if globalMirror.Credentials == nil || globalMirror.Credentials.SecretRef == nil { return nil, nil } namespace := objectNamespace - if registry.Mirror.SecretRef.Namespace != "" { - namespace = registry.Mirror.SecretRef.Namespace + if globalMirror.Credentials.SecretRef.Namespace != "" { + namespace = globalMirror.Credentials.SecretRef.Namespace } key := ctrlclient.ObjectKey{ - Name: registry.Mirror.SecretRef.Name, + Name: globalMirror.Credentials.SecretRef.Name, Namespace: namespace, } secret := &corev1.Secret{} @@ -96,10 +88,11 @@ func secretForMirrorCACert( return secret, err } -// Default Mirror for all registries. Use a mirror regardless of the intended registry. +// Default Mirror for all registries. +// Containerd configuration for global mirror will be created at /etc/containerd/certs.d/_default/hosts.toml // The upstream registry will be automatically used after all defined mirrors have been tried. // reference: https://github.com/containerd/containerd/blob/main/docs/hosts.md#setup-default-mirror-for-all-registries -func generateDefaultRegistryMirrorFile(mirror *mirrorConfig) ([]cabpkv1.File, error) { +func generateGlobalRegistryMirrorFile(mirror *mirrorConfig) ([]cabpkv1.File, error) { if mirror == nil { return nil, nil } @@ -135,7 +128,7 @@ func generateDefaultRegistryMirrorFile(mirror *mirrorConfig) ([]cabpkv1.File, er func generateMirrorCACertFile( config *mirrorConfig, - registry v1alpha1.ImageRegistry, + globalMirror v1alpha1.GlobalImageRegistryMirror, ) []cabpkv1.File { if config == nil || config.CACert == "" { return nil @@ -146,7 +139,7 @@ func generateMirrorCACertFile( Permissions: "0600", ContentFrom: &cabpkv1.FileSource{ Secret: cabpkv1.SecretFileSource{ - Name: registry.Mirror.SecretRef.Name, + Name: globalMirror.Credentials.SecretRef.Name, Key: secretKeyForMirrorCACert, }, }, diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go b/pkg/handlers/generic/mutation/mirrors/mirror_test.go similarity index 91% rename from pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go rename to pkg/handlers/generic/mutation/mirrors/mirror_test.go index 82440b04d..855527a44 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/mirror_test.go +++ b/pkg/handlers/generic/mutation/mirrors/mirror_test.go @@ -1,7 +1,7 @@ // Copyright 2023 D2iQ, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package credentials +package mirrors import ( "testing" @@ -64,7 +64,7 @@ func Test_generateDefaultRegistryMirrorFile(t *testing.T) { tt := tests[idx] t.Run(tt.name, func(t *testing.T) { t.Parallel() - file, err := generateDefaultRegistryMirrorFile(tt.config) + file, err := generateGlobalRegistryMirrorFile(tt.config) assert.ErrorIs(t, err, tt.wantErr) assert.Equal(t, tt.want, file) }) @@ -76,7 +76,7 @@ func Test_generateMirrorCACertFile(t *testing.T) { tests := []struct { name string config *mirrorConfig - registry v1alpha1.ImageRegistry + registry v1alpha1.GlobalImageRegistryMirror want []cabpkv1.File }{ { @@ -84,7 +84,7 @@ func Test_generateMirrorCACertFile(t *testing.T) { config: &mirrorConfig{ URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }, - registry: v1alpha1.ImageRegistry{ + registry: v1alpha1.GlobalImageRegistryMirror{ URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }, want: nil, @@ -95,9 +95,10 @@ func Test_generateMirrorCACertFile(t *testing.T) { URL: "https://myregistry.com", CACert: "mycacert", }, - registry: v1alpha1.ImageRegistry{ + registry: v1alpha1.GlobalImageRegistryMirror{ URL: "https://myregistry.com", - Mirror: &v1alpha1.RegistryMirror{ + + Credentials: &v1alpha1.ImageCredentials{ SecretRef: &v1.ObjectReference{ Name: "my-registry-credentials-secret", }, diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/hosts.toml.gotmpl b/pkg/handlers/generic/mutation/mirrors/templates/hosts.toml.gotmpl similarity index 100% rename from pkg/handlers/generic/mutation/imageregistries/credentials/templates/hosts.toml.gotmpl rename to pkg/handlers/generic/mutation/mirrors/templates/hosts.toml.gotmpl diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_mirror_patches.go b/pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go similarity index 68% rename from pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_mirror_patches.go rename to pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go index 9dd8aa2ad..10de0fbd2 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_mirror_patches.go +++ b/pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go @@ -5,7 +5,6 @@ package tests import ( "context" - "fmt" "testing" "github.com/onsi/gomega" @@ -23,16 +22,14 @@ import ( ) const ( - validMirrorCredentialsSecretName = "my-mirror-registry-credentials" - validMirrorCASecretName = "myregistry-mirror-cacert" + validMirrorCASecretName = "myregistry-mirror-cacert" //nolint:gosec // Does not contain hard coded credentials. cpRegistryAsMirrorCreds = "kubeadmControlPlaneRegistryAsMirrorCreds" //nolint:gosec // Does not contain hard coded credentials. - workerRegistryAsMirrorCreds = "kubeadmConfigTemplateRegistryAsMirrorCreds" - registryStaticCredentialsSecretSuffix = "registry-config" + workerRegistryAsMirrorCreds = "kubeadmConfigTemplateRegistryAsMirrorCreds" ) -func TestGenerateMirrorPatches( +func TestGeneratePatches( t *testing.T, generatorFunc func() mutation.GeneratePatches, fakeClient client.Client, @@ -41,14 +38,6 @@ func TestGenerateMirrorPatches( ) { t.Helper() - require.NoError( - t, - fakeClient.Create( - context.Background(), - newRegistryCredentialsSecret(validMirrorCredentialsSecretName, request.Namespace), - ), - ) - require.NoError( t, fakeClient.Create( @@ -57,38 +46,6 @@ func TestGenerateMirrorPatches( ), ) - // Server side apply does not work with the fake client, hack around it by pre-creating empty Secrets - // https://github.com/kubernetes-sigs/controller-runtime/issues/2341 - require.NoError( - t, - fakeClient.Create( - context.Background(), - newEmptySecret( - fmt.Sprintf( - "%s-%s", - cpRegistryAsMirrorCreds, - registryStaticCredentialsSecretSuffix, - ), - request.Namespace, - ), - ), - ) - - require.NoError( - t, - fakeClient.Create( - context.Background(), - newEmptySecret( - fmt.Sprintf( - "%s-%s", - workerRegistryAsMirrorCreds, - registryStaticCredentialsSecretSuffix, - ), - request.Namespace, - ), - ), - ) - capitest.ValidateGeneratePatches( t, generatorFunc, @@ -97,11 +54,8 @@ func TestGenerateMirrorPatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistries{ - v1alpha1.ImageRegistry{ - URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", - Mirror: &v1alpha1.RegistryMirror{}, - }, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }, variablePath..., ), @@ -124,18 +78,11 @@ func TestGenerateMirrorPatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistries{ - v1alpha1.ImageRegistry{ - URL: "https://mirror-registry.com", - Credentials: &v1alpha1.ImageCredentials{ - SecretRef: &corev1.ObjectReference{ - Name: validSecretName, - }, - }, - Mirror: &v1alpha1.RegistryMirror{ - SecretRef: &corev1.ObjectReference{ - Name: validMirrorCASecretName, - }, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://mirror-registry.com", + Credentials: &v1alpha1.ImageCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: validMirrorCASecretName, }, }, }, @@ -163,11 +110,8 @@ func TestGenerateMirrorPatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistries{ - v1alpha1.ImageRegistry{ - URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", - Mirror: &v1alpha1.RegistryMirror{}, - }, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }, variablePath..., ), @@ -198,18 +142,11 @@ func TestGenerateMirrorPatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistries{ - v1alpha1.ImageRegistry{ - URL: "https://mirror-registry.io", - Credentials: &v1alpha1.ImageCredentials{ - SecretRef: &corev1.ObjectReference{ - Name: validSecretName, - }, - }, - Mirror: &v1alpha1.RegistryMirror{ - SecretRef: &corev1.ObjectReference{ - Name: validMirrorCASecretName, - }, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://mirror-registry.io", + Credentials: &v1alpha1.ImageCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: validMirrorCASecretName, }, }, }, diff --git a/pkg/handlers/generic/mutation/mirrors/variables_test.go b/pkg/handlers/generic/mutation/mirrors/variables_test.go new file mode 100644 index 000000000..5533b9906 --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/variables_test.go @@ -0,0 +1,46 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/clusterconfig" +) + +func TestVariableValidation(t *testing.T) { + capitest.ValidateDiscoverVariables( + t, + clusterconfig.MetaVariableName, + ptr.To(v1alpha1.GenericClusterConfig{}.VariableSchema()), + false, + clusterconfig.NewVariable, + capitest.VariableTestDef{ + Name: "without a credentials secret", + Vals: v1alpha1.GenericClusterConfig{ + GlobalImageRegistryMirror: &v1alpha1.GlobalImageRegistryMirror{ + URL: "http://a.b.c.example.com", + }, + }, + }, + capitest.VariableTestDef{ + Name: "with a credentials CA secret", + Vals: v1alpha1.GenericClusterConfig{ + GlobalImageRegistryMirror: &v1alpha1.GlobalImageRegistryMirror{ + URL: "http://a.b.c.example.com", + Credentials: &v1alpha1.ImageCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: "a.b.c.example.com-ca-cert-creds", + }, + }, + }, + }, + }, + ) +}