From ee44666774d57a74bd4774afec019944e42a926a Mon Sep 17 00:00:00 2001 From: Zack Zlotnik Date: Wed, 3 Apr 2024 10:35:37 -0400 Subject: [PATCH] adds Red Hat entitlement functionality for on-cluster layering This adds the capability for BuildController to use the RHEL entitlement secrets to allow cluster admins to inject RHEL content into their builds that they are entitled to receive. This also allows the injection / consumption of content into /etc/yum.repos.d as well as /etc/pki/rpm-gpg. There are a few notes about the implementation that I would like to have at a higher level: - Because we run rootless Buildah, we're more prone to running into SELinux complications. This makes it more difficult to directly mount the contents of /etc/yum.repos.d, /etc/pki/entitlement, and /etc/pki/rpm-gpg directly into the build context. With that in mind, we copy everything into a series of temp directories first, and then mount those temp directories into the build context as a volume. - We also create an emptyDir which is mounted into the build pod at /home/build/.local/share/containers. It is unclear why this is necessary, but as mentioned before, I suspect that this is due to SELinux issues. - The e2e test suite now has the capability to stream the container logs from the build pod to the filesystem as there is useful information contained within those logs if the e2e test fails. In OpenShift CI, this location will be determined by the ARTIFACT_DIR env var. If this env var is not present, it will default the current directory. --- pkg/controller/build/assets/buildah-build.sh | 62 +++- pkg/controller/build/build_controller.go | 103 ++++++- pkg/controller/build/image_build_request.go | 288 ++++++++++++----- test/e2e-techpreview/Containerfile.cowsay | 9 + test/e2e-techpreview/Containerfile.entitled | 6 + .../e2e-techpreview/Containerfile.yum-repos-d | 3 + test/e2e-techpreview/helpers_test.go | 291 +++++++++++++++++- test/e2e-techpreview/onclusterbuild_test.go | 145 +++++++-- 8 files changed, 785 insertions(+), 122 deletions(-) create mode 100644 test/e2e-techpreview/Containerfile.cowsay create mode 100644 test/e2e-techpreview/Containerfile.entitled create mode 100644 test/e2e-techpreview/Containerfile.yum-repos-d diff --git a/pkg/controller/build/assets/buildah-build.sh b/pkg/controller/build/assets/buildah-build.sh index e57bdcdad9..0c77101ee9 100644 --- a/pkg/controller/build/assets/buildah-build.sh +++ b/pkg/controller/build/assets/buildah-build.sh @@ -5,6 +5,10 @@ # custom build pod. set -xeuo +ETC_PKI_ENTITLEMENT_MOUNTPOINT="${ETC_PKI_ENTITLEMENT_MOUNTPOINT:-}" +ETC_PKI_RPM_GPG_MOUNTPOINT="${ETC_PKI_RPM_GPG_MOUNTPOINT:-}" +ETC_YUM_REPOS_D_MOUNTPOINT="${ETC_YUM_REPOS_D_MOUNTPOINT:-}" + build_context="$HOME/context" # Create a directory to hold our build context. @@ -14,12 +18,58 @@ mkdir -p "$build_context/machineconfig" cp /tmp/dockerfile/Dockerfile "$build_context" cp /tmp/machineconfig/machineconfig.json.gz "$build_context/machineconfig/" -# Build our image using Buildah. -buildah bud \ - --storage-driver vfs \ - --authfile="$BASE_IMAGE_PULL_CREDS" \ - --tag "$TAG" \ - --file="$build_context/Dockerfile" "$build_context" +build_args=( + --log-level=DEBUG + --storage-driver vfs + --authfile="$BASE_IMAGE_PULL_CREDS" + --tag "$TAG" + --file="$build_context/Dockerfile" +) + +mount_opts="z,rw" + +# If we have RHSM certs, copy them into a tempdir to avoid SELinux issues, and +# tell Buildah about them. +rhsm_path="/var/run/secrets/rhsm" +if [[ -d "$rhsm_path" ]]; then + rhsm_certs="$(mktemp -d)" + cp -r -v "$rhsm_path/." "$rhsm_certs" + chmod -R 0755 "$rhsm_certs" + build_args+=("--volume=$rhsm_certs:/run/secrets/rhsm:$mount_opts") +fi + +# If we have /etc/pki/entitlement certificates, commonly used with RHEL +# entitlements, copy them into a tempdir to avoid SELinux issues, and tell +# Buildah about them. +if [[ -n "$ETC_PKI_ENTITLEMENT_MOUNTPOINT" ]] && [[ -d "$ETC_PKI_ENTITLEMENT_MOUNTPOINT" ]]; then + configs="$(mktemp -d)" + cp -r -v "$ETC_PKI_ENTITLEMENT_MOUNTPOINT/." "$configs" + chmod -R 0755 "$configs" + build_args+=("--volume=$configs:$ETC_PKI_ENTITLEMENT_MOUNTPOINT:$mount_opts") +fi + +# If we have /etc/yum.repos.d configs, commonly used with Red Hat Satellite +# subscriptions, copy them into a tempdir to avoid SELinux issues, and tell +# Buildah about them. +if [[ -n "$ETC_YUM_REPOS_D_MOUNTPOINT" ]] && [[ -d "$ETC_YUM_REPOS_D_MOUNTPOINT" ]]; then + configs="$(mktemp -d)" + cp -r -v "$ETC_YUM_REPOS_D_MOUNTPOINT/." "$configs" + chmod -R 0755 "$configs" + build_args+=("--volume=$configs:$ETC_YUM_REPOS_D_MOUNTPOINT:$mount_opts") +fi + +# If we have /etc/pki/rpm-gpg configs, commonly used with Red Hat Satellite +# subscriptions, copy them into a tempdir to avoid SELinux issues, and tell +# Buildah about them. +if [[ -n "$ETC_PKI_RPM_GPG_MOUNTPOINT" ]] && [[ -d "$ETC_PKI_RPM_GPG_MOUNTPOINT" ]]; then + configs="$(mktemp -d)" + cp -r -v "$ETC_PKI_RPM_GPG_MOUNTPOINT/." "$configs" + chmod -R 0755 "$configs" + build_args+=("--volume=$configs:$ETC_PKI_RPM_GPG_MOUNTPOINT:$mount_opts") +fi + +# Build our image. +buildah bud "${build_args[@]}" "$build_context" # Push our built image. buildah push \ diff --git a/pkg/controller/build/build_controller.go b/pkg/controller/build/build_controller.go index be416aefa6..ad701e7517 100644 --- a/pkg/controller/build/build_controller.go +++ b/pkg/controller/build/build_controller.go @@ -52,6 +52,17 @@ import ( "github.com/openshift/machine-config-operator/internal/clients" ) +const ( + // Name of the etc-pki-entitlement secret from the openshift-config-managed namespace. + etcPkiEntitlementSecretName = "etc-pki-entitlement" + + // Name of the etc-pki-rpm-gpg secret. + etcPkiRpmGpgSecretName = "etc-pki-rpm-gpg" + + // Name of the etc-yum-repos-d ConfigMap. + etcYumReposDConfigMapName = "etc-yum-repos-d" +) + const ( targetMachineConfigPoolLabel = "machineconfiguration.openshift.io/targetMachineConfigPool" // TODO(zzlotnik): Is there a constant for this someplace else? @@ -472,6 +483,20 @@ func (ctrl *Controller) customBuildPodUpdater(pod *corev1.Pod) error { ps := newPoolState(pool) + // We cannot solely rely upon the pod phase to determine whether the build + // pod is in an error state. This is because it is possible for the build + // container to enter an error state while the wait-for-done container is + // still running. The pod phase in this state will still be "Running" as + // opposed to error. + if isBuildPodError(pod) { + if err := ctrl.markBuildFailed(ps); err != nil { + return err + } + + ctrl.enqueueMachineConfigPool(pool) + return nil + } + switch pod.Status.Phase { case corev1.PodPending: if !ps.IsBuildPending() { @@ -503,6 +528,22 @@ func (ctrl *Controller) customBuildPodUpdater(pod *corev1.Pod) error { return nil } +// Determines if the build pod is in an error state by examining the individual +// container statuses. Returns true if a single container is in an error state. +func isBuildPodError(pod *corev1.Pod) bool { + for _, container := range pod.Status.ContainerStatuses { + if container.State.Waiting != nil && container.State.Waiting.Reason == "ErrImagePull" { + return true + } + + if container.State.Terminated != nil && container.State.Terminated.ExitCode != 0 { + return true + } + } + + return false +} + func (ctrl *Controller) handleConfigMapError(pools []*mcfgv1.MachineConfigPool, err error, key interface{}) { klog.V(2).Infof("Error syncing configmap %v: %v", key, err) utilruntime.HandleError(err) @@ -950,17 +991,69 @@ func (ctrl *Controller) getBuildInputs(ps *poolState) (*buildInputs, error) { return nil, fmt.Errorf("could not get MachineConfig %s: %w", currentMC, err) } + etcPkiEntitlements, err := ctrl.getOptionalSecret(etcPkiEntitlementSecretName) + if err != nil { + return nil, err + } + + etcPkiRpmGpgKeys, err := ctrl.getOptionalSecret(etcPkiRpmGpgSecretName) + if err != nil { + return nil, err + } + + etcYumReposDConfigs, err := ctrl.getOptionalConfigMap(etcYumReposDConfigMapName) + if err != nil { + return nil, err + } + inputs := &buildInputs{ - onClusterBuildConfig: onClusterBuildConfig, - osImageURL: osImageURL, - customDockerfiles: customDockerfiles, - pool: ps.MachineConfigPool(), - machineConfig: mc, + onClusterBuildConfig: onClusterBuildConfig, + osImageURL: osImageURL, + customDockerfiles: customDockerfiles, + pool: ps.MachineConfigPool(), + machineConfig: mc, + etcPkiEntitlementKeys: etcPkiEntitlements, + etcYumReposDConfigs: etcYumReposDConfigs, + etcPkiRpmGpgKeys: etcPkiRpmGpgKeys, } return inputs, nil } +// Fetches an optional secret to inject into the build. Returns a nil error if +// the secret is not found. +func (ctrl *Controller) getOptionalSecret(secretName string) (*corev1.Secret, error) { + optionalSecret, err := ctrl.kubeclient.CoreV1().Secrets(ctrlcommon.MCONamespace).Get(context.TODO(), secretName, metav1.GetOptions{}) + if err == nil { + klog.Infof("Optional build secret %q found, will include in build", secretName) + return optionalSecret, nil + } + + if k8serrors.IsNotFound(err) { + klog.Infof("Could not find optional secret %q, will not include in build", secretName) + return nil, nil + } + + return nil, fmt.Errorf("could not retrieve optional secret: %s: %w", secretName, err) +} + +// Fetches an optional ConfigMap to inject into the build. Returns a nil error if +// the ConfigMap is not found. +func (ctrl *Controller) getOptionalConfigMap(configmapName string) (*corev1.ConfigMap, error) { + optionalConfigMap, err := ctrl.kubeclient.CoreV1().ConfigMaps(ctrlcommon.MCONamespace).Get(context.TODO(), configmapName, metav1.GetOptions{}) + if err == nil { + klog.Infof("Optional build ConfigMap %q found, will include in build", configmapName) + return optionalConfigMap, nil + } + + if k8serrors.IsNotFound(err) { + klog.Infof("Could not find ConfigMap %q, will not include in build", configmapName) + return nil, nil + } + + return nil, fmt.Errorf("could not retrieve optional ConfigMap: %s: %w", configmapName, err) +} + // Prepares all of the objects needed to perform an image build. func (ctrl *Controller) prepareForBuild(inputs *buildInputs) (ImageBuildRequest, error) { ibr := newImageBuildRequestFromBuildInputs(inputs) diff --git a/pkg/controller/build/image_build_request.go b/pkg/controller/build/image_build_request.go index 7bd2e00a58..f8ce3316e4 100644 --- a/pkg/controller/build/image_build_request.go +++ b/pkg/controller/build/image_build_request.go @@ -19,6 +19,7 @@ const ( mcPoolAnnotation string = "machineconfiguration.openshift.io/pool" machineConfigJSONFilename string = "machineconfig.json.gz" buildahImagePullspec string = "quay.io/buildah/stable:latest" + rhelEntitlementSecret string = "etc-pki-entitlement" ) //go:embed assets/Dockerfile.on-cluster-build-template @@ -55,14 +56,23 @@ type ImageBuildRequest struct { ReleaseVersion string // An optional user-supplied Dockerfile that gets injected into the build. CustomDockerfile string + // Has /etc/pki/entitlement + HasEtcPkiEntitlementKeys bool + // Has /etc/yum.repos.d configs + HasEtcYumReposDConfigs bool + // Has /etc/pki/rpm-gpg configs + HasEtcPkiRpmGpgKeys bool } type buildInputs struct { - onClusterBuildConfig *corev1.ConfigMap - osImageURL *corev1.ConfigMap - customDockerfiles *corev1.ConfigMap - pool *mcfgv1.MachineConfigPool - machineConfig *mcfgv1.MachineConfig + onClusterBuildConfig *corev1.ConfigMap + osImageURL *corev1.ConfigMap + customDockerfiles *corev1.ConfigMap + pool *mcfgv1.MachineConfigPool + machineConfig *mcfgv1.MachineConfig + etcPkiEntitlementKeys *corev1.Secret + etcPkiRpmGpgKeys *corev1.Secret + etcYumReposDConfigs *corev1.ConfigMap } // Constructs a simple ImageBuildRequest. @@ -112,12 +122,15 @@ func newImageBuildRequestFromBuildInputs(inputs *buildInputs) ImageBuildRequest } return ImageBuildRequest{ - Pool: inputs.pool.DeepCopy(), - BaseImage: newBaseImageInfo(inputs), - FinalImage: newFinalImageInfo(inputs), - ExtensionsImage: newExtensionsImageInfo(inputs), - ReleaseVersion: inputs.osImageURL.Data[releaseVersionConfigKey], - CustomDockerfile: customDockerfile, + Pool: inputs.pool.DeepCopy(), + BaseImage: newBaseImageInfo(inputs), + FinalImage: newFinalImageInfo(inputs), + ExtensionsImage: newExtensionsImageInfo(inputs), + ReleaseVersion: inputs.osImageURL.Data[releaseVersionConfigKey], + CustomDockerfile: customDockerfile, + HasEtcPkiEntitlementKeys: inputs.etcPkiEntitlementKeys != nil, + HasEtcYumReposDConfigs: inputs.etcYumReposDConfigs != nil, + HasEtcPkiRpmGpgKeys: inputs.etcPkiRpmGpgKeys != nil, } } @@ -465,8 +478,6 @@ func (i ImageBuildRequest) toBuildahPod() *corev1.Pod { RunAsGroup: &gid, } - command := []string{"/bin/bash", "-c"} - volumeMounts := []corev1.VolumeMount{ { Name: "machineconfig", @@ -488,6 +499,168 @@ func (i ImageBuildRequest) toBuildahPod() *corev1.Pod { Name: "done", MountPath: "/tmp/done", }, + { + Name: "buildah-cache", + MountPath: "/home/build/.local/share/containers", + }, + } + + // Octal: 0755. + var mountMode int32 = 493 + + volumes := []corev1.Volume{ + { + // Provides the rendered Dockerfile. + Name: "dockerfile", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: i.getDockerfileConfigMapName(), + }, + }, + }, + }, + { + // Provides the rendered MachineConfig in a gzipped / base64-encoded + // format. + Name: "machineconfig", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: i.getMCConfigMapName(), + }, + }, + }, + }, + { + // Provides the credentials needed to pull the base OS image. + Name: "base-image-pull-creds", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: i.BaseImage.PullSecret.Name, + Items: []corev1.KeyToPath{ + { + Key: corev1.DockerConfigJsonKey, + Path: "config.json", + }, + }, + }, + }, + }, + { + // Provides the credentials needed to push the final OS image. + Name: "final-image-push-creds", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: i.FinalImage.PullSecret.Name, + Items: []corev1.KeyToPath{ + { + Key: corev1.DockerConfigJsonKey, + Path: "config.json", + }, + }, + }, + }, + }, + { + // Provides a way for the "image-build" container to signal that it + // finished so that the "wait-for-done" container can retrieve the + // iamge SHA. + Name: "done", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + }, + }, + }, + { + // This provides a dedicated place for Buildah to store / cache its + // images during the build. This seems to be required for the build-time + // volume mounts to work correctly, most likely due to an issue with + // SELinux that I have yet to figure out. Despite being called a cache + // directory, it gets removed whenever the build pod exits + Name: "buildah-cache", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } + + // If the etc-pki-entitlement secret is found, mount it into the build pod. + if i.HasEtcPkiEntitlementKeys { + mountPoint := "/etc/pki/entitlement" + + env = append(env, corev1.EnvVar{ + Name: "ETC_PKI_ENTITLEMENT_MOUNTPOINT", + Value: mountPoint, + }) + + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: etcPkiEntitlementSecretName, + MountPath: mountPoint, + }) + + volumes = append(volumes, corev1.Volume{ + Name: etcPkiEntitlementSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &mountMode, + SecretName: etcPkiEntitlementSecretName, + }, + }, + }) + } + + // If the etc-yum-repos-d ConfigMap is found, mount it into the build pod. + if i.HasEtcYumReposDConfigs { + mountPoint := "/etc/yum.repos.d" + + env = append(env, corev1.EnvVar{ + Name: "ETC_YUM_REPOS_D_MOUNTPOINT", + Value: mountPoint, + }) + + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: etcYumReposDConfigMapName, + MountPath: mountPoint, + }) + + volumes = append(volumes, corev1.Volume{ + Name: etcYumReposDConfigMapName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + DefaultMode: &mountMode, + LocalObjectReference: corev1.LocalObjectReference{ + Name: etcYumReposDConfigMapName, + }, + }, + }, + }) + } + + // If the etc-pki-rpm-gpg secret is found, mount it into the build pod. + if i.HasEtcPkiRpmGpgKeys { + mountPoint := "/etc/pki/rpm-gpg" + + env = append(env, corev1.EnvVar{ + Name: "ETC_PKI_RPM_GPG_MOUNTPOINT", + Value: mountPoint, + }) + + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: etcPkiRpmGpgSecretName, + MountPath: mountPoint, + }) + + volumes = append(volumes, corev1.Volume{ + Name: etcPkiRpmGpgSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &mountMode, + SecretName: etcPkiRpmGpgSecretName, + }, + }, + }) } // TODO: We need pull creds with permissions to pull the base image. By @@ -510,7 +683,7 @@ func (i ImageBuildRequest) toBuildahPod() *corev1.Pod { // TODO: Figure out how to not hard-code this here. Image: buildahImagePullspec, Env: env, - Command: append(command, buildahBuildScript), + Command: []string{"/bin/bash", "-c", buildahBuildScript}, ImagePullPolicy: corev1.PullAlways, SecurityContext: securityContext, VolumeMounts: volumeMounts, @@ -523,7 +696,7 @@ func (i ImageBuildRequest) toBuildahPod() *corev1.Pod { // us to avoid parsing log files. Name: "wait-for-done", Env: env, - Command: append(command, waitScript), + Command: []string{"/bin/bash", "-c", waitScript}, Image: i.BaseImage.Pullspec, ImagePullPolicy: corev1.PullAlways, SecurityContext: securityContext, @@ -531,79 +704,14 @@ func (i ImageBuildRequest) toBuildahPod() *corev1.Pod { }, }, ServiceAccountName: "machine-os-builder", - Volumes: []corev1.Volume{ - { - // Provides the rendered Dockerfile. - Name: "dockerfile", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: i.getDockerfileConfigMapName(), - }, - }, - }, - }, - { - // Provides the rendered MachineConfig in a gzipped / base64-encoded - // format. - Name: "machineconfig", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: i.getMCConfigMapName(), - }, - }, - }, - }, - { - // Provides the credentials needed to pull the base OS image. - Name: "base-image-pull-creds", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: i.BaseImage.PullSecret.Name, - Items: []corev1.KeyToPath{ - { - Key: corev1.DockerConfigJsonKey, - Path: "config.json", - }, - }, - }, - }, - }, - { - // Provides the credentials needed to push the final OS image. - Name: "final-image-push-creds", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: i.FinalImage.PullSecret.Name, - Items: []corev1.KeyToPath{ - { - Key: corev1.DockerConfigJsonKey, - Path: "config.json", - }, - }, - }, - }, - }, - { - // Provides a way for the "image-build" container to signal that it - // finished so that the "wait-for-done" container can retrieve the - // iamge SHA. - Name: "done", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMediumMemory, - }, - }, - }, - }, + Volumes: volumes, }, } } // Constructs a common metav1.ObjectMeta object with the namespace, labels, and annotations set. func (i ImageBuildRequest) getObjectMeta(name string) metav1.ObjectMeta { - return metav1.ObjectMeta{ + objectMeta := metav1.ObjectMeta{ Name: name, Namespace: ctrlcommon.MCONamespace, Labels: map[string]string{ @@ -615,6 +723,22 @@ func (i ImageBuildRequest) getObjectMeta(name string) metav1.ObjectMeta { mcPoolAnnotation: "", }, } + + hasOptionalBuildInputTemplate := "machineconfiguration.openshift.io/has-%s" + + if i.HasEtcPkiEntitlementKeys { + objectMeta.Annotations[fmt.Sprintf(hasOptionalBuildInputTemplate, etcPkiEntitlementSecretName)] = "" + } + + if i.HasEtcYumReposDConfigs { + objectMeta.Annotations[fmt.Sprintf(hasOptionalBuildInputTemplate, etcYumReposDConfigMapName)] = "" + } + + if i.HasEtcPkiRpmGpgKeys { + objectMeta.Annotations[fmt.Sprintf(hasOptionalBuildInputTemplate, etcPkiRpmGpgSecretName)] = "" + } + + return objectMeta } // Computes the Dockerfile ConfigMap name based upon the MachineConfigPool name. diff --git a/test/e2e-techpreview/Containerfile.cowsay b/test/e2e-techpreview/Containerfile.cowsay new file mode 100644 index 0000000000..9bd23fa342 --- /dev/null +++ b/test/e2e-techpreview/Containerfile.cowsay @@ -0,0 +1,9 @@ +FROM quay.io/centos/centos:stream9 AS centos +RUN dnf install -y epel-release + +FROM configs AS final +COPY --from=centos /etc/yum.repos.d /etc/yum.repos.d +COPY --from=centos /etc/pki/rpm-gpg/RPM-GPG-KEY-* /etc/pki/rpm-gpg/ +RUN sed -i 's/\$stream/9-stream/g' /etc/yum.repos.d/centos*.repo && \ + rpm-ostree install cowsay && \ + ostree container commit diff --git a/test/e2e-techpreview/Containerfile.entitled b/test/e2e-techpreview/Containerfile.entitled new file mode 100644 index 0000000000..18e925ecc6 --- /dev/null +++ b/test/e2e-techpreview/Containerfile.entitled @@ -0,0 +1,6 @@ +FROM configs AS final + +RUN rm -rf /etc/rhsm-host && \ + rpm-ostree install buildah && \ + ln -s /run/secrets/rhsm /etc/rhsm-host && \ + ostree container commit diff --git a/test/e2e-techpreview/Containerfile.yum-repos-d b/test/e2e-techpreview/Containerfile.yum-repos-d new file mode 100644 index 0000000000..f6b9fd4d42 --- /dev/null +++ b/test/e2e-techpreview/Containerfile.yum-repos-d @@ -0,0 +1,3 @@ +FROM configs AS final +RUN rpm-ostree install buildah && \ + ostree container commit diff --git a/test/e2e-techpreview/helpers_test.go b/test/e2e-techpreview/helpers_test.go index 37afe84c64..f9eb5a2746 100644 --- a/test/e2e-techpreview/helpers_test.go +++ b/test/e2e-techpreview/helpers_test.go @@ -1,8 +1,14 @@ package e2e_techpreview_test import ( + "bytes" "context" "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" "strings" "testing" "time" @@ -13,10 +19,13 @@ import ( "github.com/openshift/machine-config-operator/test/framework" "github.com/openshift/machine-config-operator/test/helpers" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" corev1 "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" + "sigs.k8s.io/yaml" ) // Identifies a secret in the MCO namespace that has permissions to push to the ImageStream used for the test. @@ -106,24 +115,96 @@ func createSecret(t *testing.T, cs *framework.ClientSet, secret *corev1.Secret) // Copies the global pull secret from openshift-config/pull-secret into the MCO // namespace so that it can be used by the build processes. func copyGlobalPullSecret(t *testing.T, cs *framework.ClientSet) func() { - globalPullSecret, err := cs.CoreV1Interface.Secrets("openshift-config").Get(context.TODO(), "pull-secret", metav1.GetOptions{}) + return cloneSecret(t, cs, "pull-secret", "openshift-config", globalPullSecretCloneName, ctrlcommon.MCONamespace) +} + +// Copy the entitlement certificates into the MCO namespace. If the secrets +// cannot be found, calls t.Skip() to skip the test. +// +// Registers and returns a cleanup function to remove the certificate(s) after test completion. +func copyEntitlementCerts(t *testing.T, cs *framework.ClientSet) func() { + namespace := "openshift-config-managed" + name := "etc-pki-entitlement" + + _, err := cs.CoreV1Interface.Secrets(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err == nil { + return cloneSecret(t, cs, name, namespace, name, ctrlcommon.MCONamespace) + } + + if apierrs.IsNotFound(err) { + t.Logf("Secret %q not found in %q, skipping test", name, namespace) + t.Skip() + return func() {} + } + + t.Fatalf("could not get %q from %q: %s", name, namespace, err) + return func() {} +} + +// Uses the centos stream 9 container and extracts the contents of both the +// /etc/yum.repos.d and /etc/pki/rpm-gpg directories and injects those into a +// ConfigMap and Secret, respectively. This is so that the build process will +// consume those objects as part of the build process, injecting them into the +// build context. +func injectYumRepos(t *testing.T, cs *framework.ClientSet) func() { + tempDir := t.TempDir() + + yumReposPath := filepath.Join(tempDir, "yum-repos-d") + require.NoError(t, os.MkdirAll(yumReposPath, 0o755)) + + centosPullspec := "quay.io/centos/centos:stream9" + yumReposContents := convertFilesFromContainerImageToBytesMap(t, centosPullspec, "/etc/yum.repos.d/") + rpmGpgContents := convertFilesFromContainerImageToBytesMap(t, centosPullspec, "/etc/pki/rpm-gpg/") + + configMapCleanupFunc := createConfigMap(t, cs, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "etc-yum-repos-d", + Namespace: ctrlcommon.MCONamespace, + }, + // Note: Even though the BuildController retrieves this ConfigMap, it only + // does so to determine whether or not it is present. It does not look at + // its contents. For that reason, we can use the BinaryData field here + // because the Build Pod will use its contents the same regardless of + // whether its string data or binary data. + BinaryData: yumReposContents, + }) + + secretCleanupFunc := createSecret(t, cs, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "etc-pki-rpm-gpg", + Namespace: ctrlcommon.MCONamespace, + }, + Data: rpmGpgContents, + }) + + return makeIdempotentAndRegister(t, func() { + configMapCleanupFunc() + secretCleanupFunc() + }) +} + +// Clones a given secret from a given namespace into the MCO namespace. +// Registers and returns a cleanup function to delete the secret upon test +// completion. +func cloneSecret(t *testing.T, cs *framework.ClientSet, srcName, srcNamespace, dstName, dstNamespace string) func() { + secret, err := cs.CoreV1Interface.Secrets(srcNamespace).Get(context.TODO(), srcName, metav1.GetOptions{}) require.NoError(t, err) secretCopy := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: globalPullSecretCloneName, - Namespace: ctrlcommon.MCONamespace, + Name: dstName, + Namespace: dstNamespace, }, - Data: globalPullSecret.Data, - Type: globalPullSecret.Type, + Data: secret.Data, + Type: secret.Type, } cleanup := createSecret(t, cs, secretCopy) - t.Logf("Cloned global pull secret %q into namespace %q as %q", "pull-secret", ctrlcommon.MCONamespace, secretCopy.Name) + t.Logf("Cloned \"%s/%s\" to \"%s/%s\"", srcNamespace, srcName, dstNamespace, dstName) return makeIdempotentAndRegister(t, func() { cleanup() - t.Logf("Deleted global pull secret copy %q", secretCopy.Name) + t.Logf("Deleted cloned secret \"%s/%s\"", dstNamespace, dstName) }) } @@ -138,7 +219,9 @@ func waitForPoolToReachState(t *testing.T, cs *framework.ClientSet, poolName str return condFunc(mcp), nil }) - require.NoError(t, err, "MachineConfigPool %q did not reach desired state", poolName) + if err != nil { + t.Fatalf("MachineConfigPool %q did not reach desired state", poolName) + } } // Registers a cleanup function, making it idempotent, and wiring up the @@ -152,3 +235,195 @@ func makeIdempotentAndRegister(t *testing.T, cleanupFunc func()) func() { t.Cleanup(out) return out } + +// Determines where to write the build logs in the event of a failure. +// ARTIFACT_DIR is a well-known env var provided by the OpenShift CI system. +// Writing to the path in this env var will ensure that any files written to +// that path end up in the OpenShift CI GCP bucket for later viewing. +// +// If this env var is not set, these files will be written to the current +// working directory. +func getBuildArtifactDir(t *testing.T) string { + artifactDir := os.Getenv("ARTIFACT_DIR") + if artifactDir != "" { + return artifactDir + } + + cwd, err := os.Getwd() + require.NoError(t, err) + return cwd +} + +// Writes any ephemeral ConfigMaps that got created as part of the build +// process to a file. Also writes the build pod spec. +func writeBuildArtifactsToFiles(t *testing.T, cs *framework.ClientSet, pool *mcfgv1.MachineConfigPool) error { + dirPath := getBuildArtifactDir(t) + + configmaps := []string{ + "on-cluster-build-config", + "on-cluster-build-custom-dockerfile", + fmt.Sprintf("dockerfile-%s", pool.Spec.Configuration.Name), + fmt.Sprintf("mc-%s", pool.Spec.Configuration.Name), + } + + for _, configmap := range configmaps { + if err := writeConfigMapToFile(t, cs, configmap, dirPath); err != nil { + return err + } + } + + return writePodSpecToFile(t, cs, pool, dirPath) +} + +// Writes a given ConfigMap to a file. +func writeConfigMapToFile(t *testing.T, cs *framework.ClientSet, configmapName, dirPath string) error { + cm, err := cs.CoreV1Interface.ConfigMaps(ctrlcommon.MCONamespace).Get(context.TODO(), configmapName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("could not get configmap %s: %w", configmapName, err) + } + + out, err := yaml.Marshal(cm) + if err != nil { + return fmt.Errorf("could not marshal configmap %s to YAML: %w", configmapName, err) + } + + filename := filepath.Join(dirPath, fmt.Sprintf("%s-%s-configmap.yaml", t.Name(), configmapName)) + t.Logf("Writing configmap (%s) contents to %s", configmapName, filename) + return os.WriteFile(filename, out, 0o755) +} + +// Wrttes a build pod spec to a file. +func writePodSpecToFile(t *testing.T, cs *framework.ClientSet, pool *mcfgv1.MachineConfigPool, dirPath string) error { + podName := fmt.Sprintf("build-%s", pool.Spec.Configuration.Name) + + pod, err := cs.CoreV1Interface.Pods(ctrlcommon.MCONamespace).Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("could not get pod %s: %w", podName, err) + } + + out, err := yaml.Marshal(pod) + if err != nil { + return err + } + + podFilename := filepath.Join(dirPath, fmt.Sprintf("%s-%s-pod.yaml", t.Name(), pod.Name)) + t.Logf("Writing spec for pod %s to %s", pod.Name, podFilename) + return os.WriteFile(podFilename, out, 0o755) +} + +// Streams the logs for all of the containers running in the build pod. The pod +// logs can provide a valuable window into how / why a given build failed. +func streamBuildPodLogsToFile(ctx context.Context, t *testing.T, cs *framework.ClientSet, pool *mcfgv1.MachineConfigPool) error { + podName := fmt.Sprintf("build-%s", pool.Spec.Configuration.Name) + + pod, err := cs.CoreV1Interface.Pods(ctrlcommon.MCONamespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("could not get pod %s: %w", podName, err) + } + + errGroup, egCtx := errgroup.WithContext(ctx) + + for _, container := range pod.Spec.Containers { + container := container + pod := pod.DeepCopy() + + // Because we follow the logs for each container in a build pod, this + // blocks the current Goroutine. So we run each log stream operation in a + // separate Goroutine to avoid blocking the main Goroutine. + errGroup.Go(func() error { + return streamContainerLogToFile(egCtx, t, cs, pod, container) + }) + } + + return errGroup.Wait() +} + +// Streams the logs for a given container to a file. +func streamContainerLogToFile(ctx context.Context, t *testing.T, cs *framework.ClientSet, pod *corev1.Pod, container corev1.Container) error { + dirPath := getBuildArtifactDir(t) + + logger, err := cs.CoreV1Interface.Pods(ctrlcommon.MCONamespace).GetLogs(pod.Name, &corev1.PodLogOptions{ + Container: container.Name, + Follow: true, + }).Stream(ctx) + + defer logger.Close() + + if err != nil { + return fmt.Errorf("could not get logs for container %s in pod %s: %w", container.Name, pod.Name, err) + } + + filename := filepath.Join(dirPath, fmt.Sprintf("%s-%s-%s.log", t.Name(), pod.Name, container.Name)) + file, err := os.Create(filename) + if err != nil { + return err + } + + defer file.Close() + + t.Logf("Streaming pod (%s) container (%s) logs to %s", pod.Name, container.Name, filename) + if _, err := io.Copy(file, logger); err != nil { + return fmt.Errorf("could not write pod logs to %s: %w", filename, err) + } + + return nil +} + +// Skips a given test if it is detected that the cluster is running OKD. We +// skip these tests because they're either irrelevant for OKD or would fail. +func skipOnOKD(t *testing.T) { + cs := framework.NewClientSet("") + + isOKD, err := helpers.IsOKDCluster(cs) + require.NoError(t, err) + + if isOKD { + t.Logf("OKD detected, skipping test %s", t.Name()) + t.Skip() + } +} + +// Extracts the contents of a directory within a given container to a temporary +// directory. Next, it loads them into a bytes map keyed by filename. It does +// not handle nested directories, so use with caution. +func convertFilesFromContainerImageToBytesMap(t *testing.T, pullspec, containerFilepath string) map[string][]byte { + tempDir := t.TempDir() + + path := fmt.Sprintf("%s:%s", containerFilepath, tempDir) + cmd := exec.Command("oc", "image", "extract", pullspec, "--path", path) + t.Logf("Extracting files under %q from %q to %q; running %s", containerFilepath, pullspec, tempDir, cmd.String()) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + require.NoError(t, cmd.Run()) + + out := map[string][]byte{} + + isCentosImage := strings.Contains(pullspec, "centos") + + err := filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + contents, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + if isCentosImage { + contents = bytes.ReplaceAll(contents, []byte("$stream"), []byte("9-stream")) + } + + // Replace $stream with 9-stream in any of the Centos repo content we pulled. + out[filepath.Base(path)] = contents + return nil + }) + + require.NoError(t, err) + + return out +} diff --git a/test/e2e-techpreview/onclusterbuild_test.go b/test/e2e-techpreview/onclusterbuild_test.go index 3f508dea01..8709de4c4f 100644 --- a/test/e2e-techpreview/onclusterbuild_test.go +++ b/test/e2e-techpreview/onclusterbuild_test.go @@ -2,6 +2,7 @@ package e2e_techpreview_test import ( "context" + _ "embed" "flag" "strings" "testing" @@ -31,19 +32,29 @@ const ( // The name of the global pull secret copy to use for the tests. globalPullSecretCloneName string = "global-pull-secret-copy" - - // The custom Dockerfile content to build for the tests. - cowsayDockerfile string = `FROM quay.io/centos/centos:stream9 AS centos -RUN dnf install -y epel-release -FROM configs AS final -COPY --from=centos /etc/yum.repos.d /etc/yum.repos.d -COPY --from=centos /etc/pki/rpm-gpg/RPM-GPG-KEY-* /etc/pki/rpm-gpg/ -RUN sed -i 's/\$stream/9-stream/g' /etc/yum.repos.d/centos*.repo && \ - rpm-ostree install cowsay` ) var skipCleanup bool +var ( + // Provides a Containerfile that installs cowsayusing the Centos Stream 9 + // EPEL repository to do so without requiring any entitlements. + //go:embed Containerfile.cowsay + cowsayDockerfile string + + // Provides a Containerfile that installs Buildah from the default RHCOS RPM + // repositories. If the installation succeeds, the entitlement certificate is + // working. + //go:embed Containerfile.entitled + entitledDockerfile string + + // Provides a Containerfile that works similarly to the cowsay Dockerfile + // with the exception that the /etc/yum.repos.d and /etc/pki/rpm-gpg key + // content is mounted into the build context by the BuildController. + //go:embed Containerfile.yum-repos-d + yumReposDockerfile string +) + func init() { // Skips running the cleanup functions. Useful for debugging tests. flag.BoolVar(&skipCleanup, "skip-cleanup", false, "Skips running the cleanup functions") @@ -62,36 +73,84 @@ type onClusterBuildTestOpts struct { // What MachineConfigPool name to use for the test. poolName string + + // Use RHEL entitlements + useEtcPkiEntitlement bool + + // Inject YUM repo information from a Centos 9 stream container + useYumRepos bool } -// Tests that an on-cluster build can be performed with the OpenShift Image Builder. -func TestOnClusterBuildsOpenshiftImageBuilder(t *testing.T) { +// Tests tha an on-cluster build can be performed with the Custom Pod Builder. +func TestOnClusterBuildsCustomPodBuilder(t *testing.T) { runOnClusterBuildTest(t, onClusterBuildTestOpts{ - imageBuilderType: build.OpenshiftImageBuilder, - poolName: layeredMCPName, + poolName: layeredMCPName, customDockerfiles: map[string]string{ layeredMCPName: cowsayDockerfile, }, }) } -// Tests tha an on-cluster build can be performed with the Custom Pod Builder. -func TestOnClusterBuildsCustomPodBuilder(t *testing.T) { +// This test extracts the /etc/yum.repos.d and /etc/pki/rpm-gpg content from a +// Centos Stream 9 image and injects them into the MCO namespace. It then +// performs a build with the expectation that these artifacts will be used, +// simulating a build where someone has added this content; usually a Red Hat +// Satellite user. +func TestYumReposBuilds(t *testing.T) { runOnClusterBuildTest(t, onClusterBuildTestOpts{ - imageBuilderType: build.CustomPodImageBuilder, - poolName: layeredMCPName, + poolName: layeredMCPName, customDockerfiles: map[string]string{ - layeredMCPName: cowsayDockerfile, + layeredMCPName: yumReposDockerfile, + }, + useYumRepos: true, + }) +} + +// Clones the etc-pki-entitlement certificate from the openshift-config-managed +// namespace into the MCO namespace. Then performs an on-cluster layering build +// which should consume the entitlement certificates. +func TestEntitledBuilds(t *testing.T) { + skipOnOKD(t) + + runOnClusterBuildTest(t, onClusterBuildTestOpts{ + poolName: layeredMCPName, + customDockerfiles: map[string]string{ + layeredMCPName: entitledDockerfile, + }, + useEtcPkiEntitlement: true, + }) +} + +// Performs the same build as above, but deploys the built image to a node on +// that cluster and attempts to run the binary installed in the process (in +// this case, buildah). +func TestEntitledBuildsRollsOutImage(t *testing.T) { + skipOnOKD(t) + + imagePullspec := runOnClusterBuildTest(t, onClusterBuildTestOpts{ + poolName: layeredMCPName, + customDockerfiles: map[string]string{ + layeredMCPName: entitledDockerfile, }, + useEtcPkiEntitlement: true, }) + + cs := framework.NewClientSet("") + node := helpers.GetRandomNode(t, cs, "worker") + t.Cleanup(makeIdempotentAndRegister(t, func() { + helpers.DeleteNodeAndMachine(t, cs, node) + })) + helpers.LabelNode(t, cs, node, helpers.MCPNameToRole(layeredMCPName)) + helpers.WaitForNodeImageChange(t, cs, node, imagePullspec) + + t.Log(helpers.ExecCmdOnNode(t, cs, node, "chroot", "/rootfs", "buildah", "--help")) } // Tests that an on-cluster build can be performed and that the resulting image // is rolled out to an opted-in node. func TestOnClusterBuildRollsOutImage(t *testing.T) { imagePullspec := runOnClusterBuildTest(t, onClusterBuildTestOpts{ - imageBuilderType: build.OpenshiftImageBuilder, - poolName: layeredMCPName, + poolName: layeredMCPName, customDockerfiles: map[string]string{ layeredMCPName: cowsayDockerfile, }, @@ -111,20 +170,35 @@ func TestOnClusterBuildRollsOutImage(t *testing.T) { // Sets up and performs an on-cluster build for a given set of parameters. // Returns the built image pullspec for later consumption. func runOnClusterBuildTest(t *testing.T, testOpts onClusterBuildTestOpts) string { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + cs := framework.NewClientSet("") t.Logf("Running with ImageBuilder type: %s", testOpts.imageBuilderType) + // Create all of the objects needed to set up our test. prepareForTest(t, cs, testOpts) + // Opt the test MachineConfigPool into layering to signal the build to begin. optPoolIntoLayering(t, cs, testOpts.poolName) t.Logf("Wait for build to start") + var pool *mcfgv1.MachineConfigPool waitForPoolToReachState(t, cs, testOpts.poolName, func(mcp *mcfgv1.MachineConfigPool) bool { + pool = mcp return ctrlcommon.NewLayeredPoolState(mcp).IsBuilding() }) t.Logf("Build started! Waiting for completion...") + + // The pod log collection blocks the main Goroutine since we follow the logs + // for each container in the build pod. So they must run in a separate + // Goroutine so that the rest of the test can continue. + go func() { + require.NoError(t, streamBuildPodLogsToFile(ctx, t, cs, pool)) + }() + imagePullspec := "" waitForPoolToReachState(t, cs, testOpts.poolName, func(mcp *mcfgv1.MachineConfigPool) bool { lps := ctrlcommon.NewLayeredPoolState(mcp) @@ -134,6 +208,7 @@ func runOnClusterBuildTest(t *testing.T, testOpts onClusterBuildTestOpts) string } if lps.IsBuildFailure() { + require.NoError(t, writeBuildArtifactsToFiles(t, cs, pool)) t.Fatalf("Build unexpectedly failed.") } @@ -189,6 +264,7 @@ func optPoolIntoLayering(t *testing.T, cs *framework.ClientSet, pool string) fun // - Gets the Docker Builder secret name from the MCO namespace. // - Creates the imagestream to use for the test. // - Clones the global pull secret into the MCO namespace. +// - If requrested, clones the RHEL entitlement secret into the MCO namespace. // - Creates the on-cluster-build-config ConfigMap. // - Creates the target MachineConfigPool and waits for it to get a rendered config. // - Creates the on-cluster-build-custom-dockerfile ConfigMap. @@ -199,14 +275,39 @@ func prepareForTest(t *testing.T, cs *framework.ClientSet, testOpts onClusterBui pushSecretName, err := getBuilderPushSecretName(cs) require.NoError(t, err) + // If the test requires RHEL entitlements, clone them from + // "etc-pki-entitlement" in the "openshift-config-managed" namespace. + // If we want to use RHEL entit + if testOpts.useEtcPkiEntitlement { + t.Cleanup(copyEntitlementCerts(t, cs)) + } + + // If the test requires /etc/yum.repos.d and /etc/pki/rpm-gpg, pull a Centos + // Stream 9 container image and populate them from there. This is intended to + // emulate the Red Hat Satellite enablement process, but does not actually + // require any Red Hat Satellite creds to work. + if testOpts.useYumRepos { + t.Cleanup(injectYumRepos(t, cs)) + } + + // Creates an imagestream to push the built OS image to. This is so that the + // test may be self-contained within the test cluster. imagestreamName := "os-image" t.Cleanup(createImagestream(t, cs, imagestreamName)) + // Default to the custom pod builder image builder type. + if testOpts.imageBuilderType == "" { + testOpts.imageBuilderType = build.CustomPodImageBuilder + } + + // Copy the global pull secret into the MCO namespace. t.Cleanup(copyGlobalPullSecret(t, cs)) + // Get the final image pullspec from the imagestream that we just created. finalPullspec, err := getImagestreamPullspec(cs, imagestreamName) require.NoError(t, err) + // Set up the on-cluster-build-config ConfigMap. cmCleanup := createConfigMap(t, cs, &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: build.OnClusterBuildConfigMapName, @@ -222,10 +323,13 @@ func prepareForTest(t *testing.T, cs *framework.ClientSet, testOpts onClusterBui t.Cleanup(cmCleanup) + // Create the MachineConfigPool that we intend to target for the test. t.Cleanup(makeIdempotentAndRegister(t, helpers.CreateMCP(t, cs, testOpts.poolName))) + // Create the on-cluster-build-custom-dockerfile ConfigMap. t.Cleanup(createCustomDockerfileConfigMap(t, cs, testOpts.customDockerfiles)) + // Wait for our targeted MachineConfigPool to get a base MachineConfig. _, err = helpers.WaitForRenderedConfig(t, cs, testOpts.poolName, "00-worker") require.NoError(t, err) } @@ -239,7 +343,6 @@ func TestSSHKeyAndPasswordForOSBuilder(t *testing.T) { // prepare for on cluster build test prepareForTest(t, cs, onClusterBuildTestOpts{ - imageBuilderType: build.OpenshiftImageBuilder, poolName: layeredMCPName, customDockerfiles: map[string]string{}, })