From 89d09dcfe262abc9685769c7994307e1a99f6986 Mon Sep 17 00:00:00 2001 From: Laurent Goderre Date: Thu, 9 Oct 2025 09:17:57 -0400 Subject: [PATCH 1/2] fix(k3s): add better error handling to LoadImages --- modules/k3s/k3s.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/k3s/k3s.go b/modules/k3s/k3s.go index e69fbc8655..21c8dc9088 100644 --- a/modules/k3s/k3s.go +++ b/modules/k3s/k3s.go @@ -209,10 +209,14 @@ func (c *K3sContainer) LoadImagesWithOpts(ctx context.Context, images []string, return fmt.Errorf("copying image to container %w", err) } - _, _, err = c.Exec(ctx, []string{"ctr", "-n=k8s.io", "images", "import", "--all-platforms", containerPath}) + exit, reader, err := c.Exec(ctx, []string{"ctr", "-n=k8s.io", "images", "import", "--all-platforms", containerPath}) if err != nil { return fmt.Errorf("importing image %w", err) } + if exit != 0 { + b, _ := io.ReadAll(reader) + return fmt.Errorf("importing image %s", string(b)) + } return nil } From 67f7ac8575f0d6b215cf83f5cd17596e364c3def Mon Sep 17 00:00:00 2001 From: Laurent Goderre Date: Thu, 9 Oct 2025 09:25:06 -0400 Subject: [PATCH 2/2] fix(k3s): don't attempt to load all platforms when using LoadImages deprecate LoadImagesWithOpts with a new LoadImagesWithPlatform --- docker.go | 21 +++++++ image.go | 7 +++ modules/k3s/go.mod | 4 +- modules/k3s/k3s.go | 58 ++++++++++++++++++- modules/k3s/k3s_test.go | 122 +++++++++++++++++++++++++++++++++++++++- 5 files changed, 207 insertions(+), 5 deletions(-) diff --git a/docker.go b/docker.go index f8a9e49081..8c7e776eb7 100644 --- a/docker.go +++ b/docker.go @@ -1841,6 +1841,27 @@ func (p *DockerProvider) PullImage(ctx context.Context, img string) error { return p.attemptToPullImage(ctx, img, image.PullOptions{}) } +// PullImage pulls image from registry, passing options to the provider +func (p *DockerProvider) PullImageWithOpts(ctx context.Context, img string, opts ...PullImageOption) error { + pullOpts := pullImageOptions{} + + for _, opt := range opts { + if err := opt(&pullOpts); err != nil { + return fmt.Errorf("applying pull image option: %w", err) + } + } + + return p.attemptToPullImage(ctx, img, pullOpts.dockerPullOpts) +} + +func PullDockerImageWithPlatform(platform specs.Platform) PullImageOption { + return func(opts *pullImageOptions) error { + opts.dockerPullOpts.Platform = platforms.Format(platform) + + return nil + } +} + var permanentClientErrors = []func(error) bool{ errdefs.IsNotFound, errdefs.IsInvalidArgument, diff --git a/image.go b/image.go index 11154d8625..81e042dca0 100644 --- a/image.go +++ b/image.go @@ -3,6 +3,7 @@ package testcontainers import ( "context" + "github.com/docker/docker/api/types/image" "github.com/docker/docker/client" ) @@ -18,6 +19,12 @@ type saveImageOptions struct { type SaveImageOption func(*saveImageOptions) error +type pullImageOptions struct { + dockerPullOpts image.PullOptions +} + +type PullImageOption func(*pullImageOptions) error + // ImageProvider allows manipulating images type ImageProvider interface { ListImages(context.Context) ([]ImageInfo, error) diff --git a/modules/k3s/go.mod b/modules/k3s/go.mod index 7e97a63cbd..1b6826c33c 100644 --- a/modules/k3s/go.mod +++ b/modules/k3s/go.mod @@ -5,8 +5,10 @@ go 1.24.0 toolchain go1.24.7 require ( + github.com/containerd/platforms v0.2.1 github.com/docker/docker v28.3.3+incompatible github.com/docker/go-connections v0.6.0 + github.com/opencontainers/image-spec v1.1.1 github.com/stretchr/testify v1.11.1 github.com/testcontainers/testcontainers-go v0.39.0 gopkg.in/yaml.v3 v3.0.1 @@ -23,7 +25,6 @@ require ( github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/containerd/platforms v0.2.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect @@ -61,7 +62,6 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect diff --git a/modules/k3s/k3s.go b/modules/k3s/k3s.go index 21c8dc9088..2894872956 100644 --- a/modules/k3s/k3s.go +++ b/modules/k3s/k3s.go @@ -7,9 +7,11 @@ import ( "os" "path/filepath" + "github.com/containerd/platforms" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" "github.com/docker/go-connections/nat" + v1 "github.com/opencontainers/image-spec/specs-go/v1" "gopkg.in/yaml.v3" "github.com/testcontainers/testcontainers-go" @@ -179,10 +181,12 @@ func unmarshal(bytes []byte) (*KubeConfigValue, error) { return &kubeConfig, nil } +// LoadImages imports local images into the cluster using containerd func (c *K3sContainer) LoadImages(ctx context.Context, images ...string) error { - return c.LoadImagesWithOpts(ctx, images) + return c.LoadImagesWithPlatform(ctx, images, nil) } +// Deprecated: use LoadImagesWithPlatform instead func (c *K3sContainer) LoadImagesWithOpts(ctx context.Context, images []string, opts ...testcontainers.SaveImageOption) error { provider, err := testcontainers.ProviderDocker.GetProvider() if err != nil { @@ -220,3 +224,55 @@ func (c *K3sContainer) LoadImagesWithOpts(ctx context.Context, images []string, return nil } + +// LoadImagesWithPlatform imports local images into the cluster using containerd for a specific platform +func (c *K3sContainer) LoadImagesWithPlatform(ctx context.Context, images []string, platform *v1.Platform) error { + provider, err := testcontainers.ProviderDocker.GetProvider() + if err != nil { + return fmt.Errorf("getting docker provider %w", err) + } + + opts := []testcontainers.SaveImageOption{} + if platform != nil { + opts = append(opts, testcontainers.SaveDockerImageWithPlatforms(*platform)) + } + + // save image + imagesTar, err := os.CreateTemp(os.TempDir(), "images*.tar") + if err != nil { + return fmt.Errorf("creating temporary images file %w", err) + } + defer func() { + _ = os.Remove(imagesTar.Name()) + }() + + err = provider.SaveImagesWithOpts(context.Background(), imagesTar.Name(), images, opts...) + if err != nil { + return fmt.Errorf("saving images %w", err) + } + + containerPath := "/tmp/" + filepath.Base(imagesTar.Name()) + err = c.CopyFileToContainer(ctx, imagesTar.Name(), containerPath, 0x644) + if err != nil { + return fmt.Errorf("copying image to container %w", err) + } + + cmd := []string{"ctr", "-n=k8s.io", "images", "import"} + + if platform != nil { + cmd = append(cmd, "--platform", platforms.Format(*platform)) + } + + cmd = append(cmd, containerPath) + + exit, reader, err := c.Exec(ctx, cmd) + if err != nil { + return fmt.Errorf("importing image %w", err) + } + if exit != 0 { + b, _ := io.ReadAll(reader) + return fmt.Errorf("importing image %s", string(b)) + } + + return nil +} diff --git a/modules/k3s/k3s_test.go b/modules/k3s/k3s_test.go index 3328a56d3d..8f46505f61 100644 --- a/modules/k3s/k3s_test.go +++ b/modules/k3s/k3s_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/containerd/platforms" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -39,8 +40,18 @@ func Test_LoadImages(t *testing.T) { provider, err := testcontainers.ProviderDocker.GetProvider() require.NoError(t, err) + dockerProvider, _ := provider.(*testcontainers.DockerProvider) + + // This function only works for single architecture images + // Forces the test to use a single-arch version of the image + arch := platforms.DefaultSpec().Architecture + if platforms.DefaultSpec().Variant != "" { + arch += platforms.DefaultSpec().Variant + } + nginxImg := arch + "/nginx" + // ensure nginx image is available locally - err = provider.PullImage(ctx, "nginx") + err = provider.PullImage(ctx, nginxImg) require.NoError(t, err) t.Run("Test load image not available", func(t *testing.T) { @@ -48,8 +59,115 @@ func Test_LoadImages(t *testing.T) { require.Error(t, err) }) + t.Run("Test load image with wrong architecture", func(t *testing.T) { + p, _ := platforms.Parse("linux/s390x") + img := "nginx:mainline" + err = dockerProvider.PullImageWithOpts(ctx, img, testcontainers.PullDockerImageWithPlatform(p)) + require.NoError(t, err) + + err := k3sContainer.LoadImages(ctx, img) + require.Error(t, err) + require.Regexp(t, "content digest .* not found", err) + }) + + t.Run("Test load image in cluster", func(t *testing.T) { + err := k3sContainer.LoadImages(ctx, nginxImg) + require.NoError(t, err) + + pod := &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: nginxImg, + ImagePullPolicy: corev1.PullNever, // use image only if already present + }, + }, + }, + } + + _, err = k8s.CoreV1().Pods("default").Create(ctx, pod, metav1.CreateOptions{}) + require.NoError(t, err) + + err = kwait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { + state, err := getTestPodState(ctx, k8s) + if err != nil { + return false, err + } + if state.Terminated != nil { + return false, fmt.Errorf("pod terminated: %v", state.Terminated) + } + return state.Running != nil, nil + }) + require.NoError(t, err) + + state, err := getTestPodState(ctx, k8s) + require.NoError(t, err) + require.NotNil(t, state.Running) + }) +} + +func Test_LoadImagesWithPlatform(t *testing.T) { + // Give up to three minutes to run this test + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(3*time.Minute)) + defer cancel() + + k3sContainer, err := k3s.Run(ctx, "rancher/k3s:v1.27.1-k3s1") + testcontainers.CleanupContainer(t, k3sContainer) + require.NoError(t, err) + + kubeConfigYaml, err := k3sContainer.GetKubeConfig(ctx) + require.NoError(t, err) + + restcfg, err := clientcmd.RESTConfigFromKubeConfig(kubeConfigYaml) + require.NoError(t, err) + + k8s, err := kubernetes.NewForConfig(restcfg) + require.NoError(t, err) + + provider, err := testcontainers.ProviderDocker.GetProvider() + require.NoError(t, err) + + dockerProvider, _ := provider.(*testcontainers.DockerProvider) + + // ensure nginx image is available locally + err = provider.PullImage(ctx, "nginx") + require.NoError(t, err) + + t.Run("Test load image not available", func(t *testing.T) { + p, _ := platforms.Parse("linux/amd64") + err := k3sContainer.LoadImagesWithPlatform(ctx, []string{"fake.registry/fake:non-existing"}, &p) + require.Error(t, err) + }) + + t.Run("Test load image with wrong architecture", func(t *testing.T) { + pullPlatform, _ := platforms.Parse("linux/s390x") + img := "nginx:mainline" + err = dockerProvider.PullImageWithOpts(ctx, img, testcontainers.PullDockerImageWithPlatform(pullPlatform)) + require.NoError(t, err) + + loadPlatform, _ := platforms.Parse("linux/amd64") + err := k3sContainer.LoadImagesWithPlatform(ctx, []string{img}, &loadPlatform) + require.Error(t, err) + expected := fmt.Sprintf( + "image with reference %s was found but does not provide the specified platform (%s)", + img, + platforms.Format(loadPlatform), + ) + require.Contains(t, err.Error(), expected) + }) + t.Run("Test load image in cluster", func(t *testing.T) { - err := k3sContainer.LoadImages(ctx, "nginx") + p := platforms.DefaultSpec() + p.OS = "linux" + err := k3sContainer.LoadImagesWithPlatform(ctx, []string{"nginx"}, &p) require.NoError(t, err) pod := &corev1.Pod{