diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d66091726..f7c5f821b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ For older changes see the [archived Singularity change log](https://github.com/a - Label process for starter binary of interactive containers with image filename, for example: `Apptainer runtime parent: example.sif`. +- The `registry login` and `registry logout` commands now support a `--authfile + ` flag, which causes the OCI credentials to be written to / removed from + a custom file located at `` instead of the default location + (`$HOME/.apptainer/docker-config.json`). The commands `pull`, `push`, `run`, + `exec`, `shell`, and `instance start` can now also be passed a `--authfile + ` option, to read OCI registry credentials from this custom file. ## Changes for v1.3.x diff --git a/cmd/internal/cli/action_flags.go b/cmd/internal/cli/action_flags.go index 1616a57223..f187911dda 100644 --- a/cmd/internal/cli/action_flags.go +++ b/cmd/internal/cli/action_flags.go @@ -940,5 +940,6 @@ func init() { cmdManager.RegisterFlagForCmd(&actionUnderlayFlag, actionsInstanceCmd...) cmdManager.RegisterFlagForCmd(&actionShareNSFlag, actionsCmd...) cmdManager.RegisterFlagForCmd(&actionRunscriptTimeoutFlag, actionsRunscriptCmd...) + cmdManager.RegisterFlagForCmd(&commonAuthFileFlag, actionsInstanceCmd...) }) } diff --git a/cmd/internal/cli/actions.go b/cmd/internal/cli/actions.go index a9cee24f1a..795f730194 100644 --- a/cmd/internal/cli/actions.go +++ b/cmd/internal/cli/actions.go @@ -85,10 +85,11 @@ func handleOCI(ctx context.Context, imgCache *cache.Handle, cmd *cobra.Command, } pullOpts := oci.PullOptions{ - TmpDir: tmpDir, - OciAuth: ociAuth, - DockerHost: dockerHost, - NoHTTPS: noHTTPS, + TmpDir: tmpDir, + OciAuth: ociAuth, + DockerHost: dockerHost, + NoHTTPS: noHTTPS, + ReqAuthFile: reqAuthFile, } return oci.Pull(ctx, imgCache, pullFrom, pullOpts) @@ -99,7 +100,7 @@ func handleOras(ctx context.Context, imgCache *cache.Handle, cmd *cobra.Command, if err != nil { return "", fmt.Errorf("while creating docker credentials: %v", err) } - return oras.Pull(ctx, imgCache, pullFrom, tmpDir, ociAuth, noHTTPS) + return oras.Pull(ctx, imgCache, pullFrom, tmpDir, ociAuth, noHTTPS, reqAuthFile) } func handleLibrary(ctx context.Context, imgCache *cache.Handle, pullFrom string) (string, error) { diff --git a/cmd/internal/cli/apptainer.go b/cmd/internal/cli/apptainer.go index 8816177d22..4b4998b3cb 100644 --- a/cmd/internal/cli/apptainer.go +++ b/cmd/internal/cli/apptainer.go @@ -66,6 +66,8 @@ var ( noHTTPS bool useBuildConfig bool tmpDir string + // Optional user requested authentication file for writing/reading OCI registry credentials + reqAuthFile string ) // apptainer command flags @@ -258,6 +260,16 @@ var singBuildConfigFlag = cmdline.Flag{ Usage: "use configuration needed for building containers", } +// --authfile +var commonAuthFileFlag = cmdline.Flag{ + ID: "commonAuthFileFlag", + Value: &reqAuthFile, + DefaultValue: "", + Name: "authfile", + Usage: "Docker-style authentication file to use for writing/reading OCI registry credentials", + EnvKeys: []string{"AUTHFILE"}, +} + func getCurrentUser() *user.User { usr, err := user.Current() if err != nil { diff --git a/cmd/internal/cli/pull.go b/cmd/internal/cli/pull.go index c12562b462..23bb915f5d 100644 --- a/cmd/internal/cli/pull.go +++ b/cmd/internal/cli/pull.go @@ -166,6 +166,7 @@ func init() { cmdManager.RegisterFlagForCmd(&pullAllowUnauthenticatedFlag, PullCmd) cmdManager.RegisterFlagForCmd(&pullArchFlag, PullCmd) cmdManager.RegisterFlagForCmd(&pullArchVariantFlag, PullCmd) + cmdManager.RegisterFlagForCmd(&commonAuthFileFlag, PullCmd) }) } @@ -267,7 +268,7 @@ func pullRun(cmd *cobra.Command, args []string) { sylog.Fatalf("Unable to make docker oci credentials: %s", err) } - _, err = oras.PullToFile(ctx, imgCache, pullTo, pullFrom, tmpDir, ociAuth, noHTTPS) + _, err = oras.PullToFile(ctx, imgCache, pullTo, pullFrom, tmpDir, ociAuth, noHTTPS, reqAuthFile) if err != nil { sylog.Fatalf("While pulling image from oci registry: %v", err) } @@ -288,12 +289,13 @@ func pullRun(cmd *cobra.Command, args []string) { return } pullOpts := oci.PullOptions{ - TmpDir: tmpDir, - OciAuth: ociAuth, - DockerHost: dockerHost, - NoHTTPS: noHTTPS, - NoCleanUp: buildArgs.noCleanUp, - Pullarch: arch, + TmpDir: tmpDir, + OciAuth: ociAuth, + DockerHost: dockerHost, + NoHTTPS: noHTTPS, + NoCleanUp: buildArgs.noCleanUp, + Pullarch: arch, + ReqAuthFile: reqAuthFile, } _, err = oci.PullToFile(ctx, imgCache, pullTo, pullFrom, pullOpts) diff --git a/cmd/internal/cli/push.go b/cmd/internal/cli/push.go index 6b80455a8c..e170200316 100644 --- a/cmd/internal/cli/push.go +++ b/cmd/internal/cli/push.go @@ -80,6 +80,7 @@ func init() { cmdManager.RegisterFlagForCmd(&dockerHostFlag, PushCmd) cmdManager.RegisterFlagForCmd(&dockerUsernameFlag, PushCmd) cmdManager.RegisterFlagForCmd(&dockerPasswordFlag, PushCmd) + cmdManager.RegisterFlagForCmd(&commonAuthFileFlag, PushCmd) }) } @@ -167,7 +168,7 @@ var PushCmd = &cobra.Command{ sylog.Fatalf("Unable to make docker oci credentials: %s", err) } - if err := oras.UploadImage(cmd.Context(), file, ref, ociAuth, noHTTPS); err != nil { + if err := oras.UploadImage(cmd.Context(), file, ref, ociAuth, noHTTPS, reqAuthFile); err != nil { sylog.Fatalf("Unable to push image to oci registry: %v", err) } sylog.Infof("Upload complete") diff --git a/cmd/internal/cli/registry.go b/cmd/internal/cli/registry.go index 479b2d9f05..8feab86e30 100644 --- a/cmd/internal/cli/registry.go +++ b/cmd/internal/cli/registry.go @@ -72,6 +72,8 @@ func init() { cmdManager.RegisterFlagForCmd(®istryLoginUsernameFlag, RegistryLoginCmd) cmdManager.RegisterFlagForCmd(®istryLoginPasswordFlag, RegistryLoginCmd) cmdManager.RegisterFlagForCmd(®istryLoginPasswordStdinFlag, RegistryLoginCmd) + cmdManager.RegisterFlagForCmd(&commonAuthFileFlag, RegistryLoginCmd) + cmdManager.RegisterFlagForCmd(&commonAuthFileFlag, RegistryLogoutCmd) }) } @@ -90,8 +92,8 @@ var RegistryCmd = &cobra.Command{ // RegistryLoginCmd apptainer registry login [option] var RegistryLoginCmd = &cobra.Command{ Args: cobra.ExactArgs(1), - Run: func(_ *cobra.Command, args []string) { - if err := apptainer.RegistryLogin(remoteConfig, ObtainLoginArgs(args[0])); err != nil { + Run: func(cmd *cobra.Command, args []string) { + if err := apptainer.RegistryLogin(remoteConfig, ObtainLoginArgs(args[0]), reqAuthFile); err != nil { sylog.Fatalf("%s", err) } }, @@ -114,7 +116,7 @@ var RegistryLogoutCmd = &cobra.Command{ name = args[0] } - if err := apptainer.RegistryLogout(remoteConfig, name); err != nil { + if err := apptainer.RegistryLogout(remoteConfig, name, reqAuthFile); err != nil { sylog.Fatalf("%s", err) } sylog.Infof("Logout succeeded") diff --git a/cmd/internal/cli/remote.go b/cmd/internal/cli/remote.go index ac0c07d977..67cdb9dd4b 100644 --- a/cmd/internal/cli/remote.go +++ b/cmd/internal/cli/remote.go @@ -215,6 +215,9 @@ func init() { cmdManager.RegisterFlagForCmd(&remoteKeyserverOrderFlag, RemoteAddKeyserverCmd) cmdManager.RegisterFlagForCmd(&remoteKeyserverInsecureFlag, RemoteAddKeyserverCmd) + + cmdManager.RegisterFlagForCmd(&commonAuthFileFlag, RemoteLoginCmd) + cmdManager.RegisterFlagForCmd(&commonAuthFileFlag, RemoteLogoutCmd) }) } @@ -306,7 +309,7 @@ var RemoteAddCmd = &cobra.Command{ Name: name, Tokenfile: loginTokenFile, } - if err := apptainer.RemoteLogin(remoteConfig, loginArgs); err != nil { + if err := apptainer.RemoteLogin(remoteConfig, loginArgs, reqAuthFile); err != nil { sylog.Fatalf("%s", err) } } @@ -402,7 +405,7 @@ var RemoteLoginCmd = &cobra.Command{ loginArgs.Password = strings.TrimSuffix(loginArgs.Password, "\r") } - if err := apptainer.RemoteLogin(remoteConfig, loginArgs); err != nil { + if err := apptainer.RemoteLogin(remoteConfig, loginArgs, reqAuthFile); err != nil { sylog.Fatalf("%s", err) } }, @@ -425,7 +428,7 @@ var RemoteLogoutCmd = &cobra.Command{ name = args[0] } - if err := apptainer.RemoteLogout(remoteConfig, name); err != nil { + if err := apptainer.RemoteLogout(remoteConfig, name, reqAuthFile); err != nil { sylog.Fatalf("%s", err) } sylog.Infof("Logout succeeded") diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 20cf3d6230..5199af5f61 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -3012,6 +3012,133 @@ func (c actionTests) relWorkdirScratch(t *testing.T) { } } +// actionAuth tests run/exec/shell flows that involve authenticated pulls from +// OCI registries. +func (c actionTests) actionAuth(t *testing.T) { + profiles := []e2e.Profile{ + e2e.UserProfile, + e2e.RootProfile, + e2e.FakerootProfile, + } + + for _, p := range profiles { + t.Run(p.String(), func(t *testing.T) { + t.Run("default", func(t *testing.T) { + c.actionAuthTester(t, false, p) + }) + t.Run("custom", func(t *testing.T) { + c.actionAuthTester(t, true, p) + }) + }) + } +} + +func (c actionTests) actionAuthTester(t *testing.T, withCustomAuthFile bool, profile e2e.Profile) { + e2e.EnsureImage(t, c.env) + + tmpdir, tmpdirCleanup := e2e.MakeTempDir(t, c.env.TestDir, "action-auth", "") + t.Cleanup(func() { + if !t.Failed() { + tmpdirCleanup(t) + } + }) + + prevCwd, err := os.Getwd() + if err != nil { + t.Fatalf("could not get current working directory: %s", err) + } + defer os.Chdir(prevCwd) + if err = os.Chdir(tmpdir); err != nil { + t.Fatalf("could not change cwd to %q: %s", tmpdir, err) + } + + localAuthFileName := "" + if withCustomAuthFile { + localAuthFileName = "./my_local_authfile" + } + + authFileArgs := []string{} + if withCustomAuthFile { + authFileArgs = []string{"--authfile", localAuthFileName} + } + + t.Cleanup(func() { + e2e.PrivateRepoLogout(t, c.env, e2e.UserProfile, localAuthFileName) + }) + + orasCustomPushTarget := fmt.Sprintf( + "oras://%s/authfile-%s-oras-alpine:latest", + c.env.TestRegistryPrivPath, strings.ToLower(profile.String()), + ) + + tests := []struct { + name string + cmd string + args []string + whileLoggedIn bool + expectExit int + }{ + { + name: "docker before auth", + cmd: "exec", + args: []string{"--disable-cache", "--no-https", c.env.TestRegistryPrivImage, "true"}, + whileLoggedIn: false, + expectExit: 255, + }, + { + name: "docker", + cmd: "exec", + args: []string{"--disable-cache", "--no-https", c.env.TestRegistryPrivImage, "true"}, + whileLoggedIn: true, + expectExit: 0, + }, + { + name: "noauth docker", + cmd: "exec", + args: []string{"--disable-cache", "--no-https", c.env.TestRegistryPrivImage, "true"}, + whileLoggedIn: false, + expectExit: 255, + }, + { + name: "oras push", + cmd: "push", + args: []string{c.env.ImagePath, orasCustomPushTarget}, + whileLoggedIn: true, + expectExit: 0, + }, + { + name: "noauth oras", + cmd: "exec", + args: []string{"--disable-cache", "--no-https", orasCustomPushTarget, "true"}, + whileLoggedIn: false, + expectExit: 255, + }, + { + name: "oras", + cmd: "exec", + args: []string{"--disable-cache", "--no-https", orasCustomPushTarget, "true"}, + whileLoggedIn: true, + expectExit: 0, + }, + } + + for _, tt := range tests { + if tt.whileLoggedIn { + e2e.PrivateRepoLogin(t, c.env, profile, localAuthFileName) + } else { + e2e.PrivateRepoLogout(t, c.env, profile, localAuthFileName) + } + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(profile), + e2e.WithCommand(tt.cmd), + e2e.WithArgs(append(authFileArgs, tt.args...)...), + e2e.ExpectExit(tt.expectExit), + ) + } +} + // E2ETests is the main func to trigger the test suite func E2ETests(env e2e.TestEnv) testhelper.Tests { c := actionTests{ @@ -3064,5 +3191,6 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "fakeroot home": c.actionFakerootHome, // test home dir in fakeroot "relWorkdirScratch": np(c.relWorkdirScratch), // test relative --workdir with --scratch "issue 1868": c.issue1868, // https://github.com/apptainer/apptainer/issues/1868 + "auth": np(c.actionAuth), // tests action cmds w/authenticated pulls from OCI registries } } diff --git a/e2e/instance/instance.go b/e2e/instance/instance.go index af1ced068c..596dce807d 100644 --- a/e2e/instance/instance.go +++ b/e2e/instance/instance.go @@ -288,6 +288,92 @@ func (c *ctx) testInstanceFromURI(t *testing.T) { } } +// Test that custom auth file authentication works with instance start +func (c *ctx) testInstanceAuthFile(t *testing.T) { + e2e.EnsureORASImage(t, c.env) + instanceName := "actionAuthTesterInstance" + localAuthFileName := "./my_local_authfile" + authFileArgs := []string{"--authfile", localAuthFileName} + + tmpdir, tmpdirCleanup := e2e.MakeTempDir(t, c.env.TestDir, "action-auth", "") + t.Cleanup(func() { + if !t.Failed() { + tmpdirCleanup(t) + } + }) + + prevCwd, err := os.Getwd() + if err != nil { + t.Fatalf("could not get current working directory: %s", err) + } + defer os.Chdir(prevCwd) + if err = os.Chdir(tmpdir); err != nil { + t.Fatalf("could not change cwd to %q: %s", tmpdir, err) + } + + tests := []struct { + name string + subCmd string + args []string + whileLoggedIn bool + expectExit int + }{ + { + name: "start before auth", + subCmd: "start", + args: append(authFileArgs, "--disable-cache", "--no-https", c.env.TestRegistryPrivImage, instanceName), + whileLoggedIn: false, + expectExit: 255, + }, + { + name: "start", + subCmd: "start", + args: append(authFileArgs, "--disable-cache", "--no-https", c.env.TestRegistryPrivImage, instanceName), + whileLoggedIn: true, + expectExit: 0, + }, + { + name: "stop", + subCmd: "stop", + args: []string{instanceName}, + whileLoggedIn: true, + expectExit: 0, + }, + { + name: "start noauth", + subCmd: "start", + args: append(authFileArgs, "--disable-cache", "--no-https", c.env.TestRegistryPrivImage, instanceName), + whileLoggedIn: false, + expectExit: 255, + }, + } + + profiles := []e2e.Profile{ + e2e.UserProfile, + e2e.RootProfile, + } + + for _, p := range profiles { + t.Run(p.String(), func(t *testing.T) { + for _, tt := range tests { + if tt.whileLoggedIn { + e2e.PrivateRepoLogin(t, c.env, e2e.UserProfile, localAuthFileName) + } else { + e2e.PrivateRepoLogout(t, c.env, e2e.UserProfile, localAuthFileName) + } + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.UserProfile), + e2e.WithCommand("instance "+tt.subCmd), + e2e.WithArgs(tt.args...), + e2e.ExpectExit(tt.expectExit), + ) + } + }) + } +} + // Execute an instance process, kill master process // and try to start another instance with same name func (c *ctx) testGhostInstance(t *testing.T) { @@ -455,6 +541,8 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { env: env, } + np := testhelper.NoParallel + return testhelper.Tests{ "ordered": func(t *testing.T) { c := &ctx{ @@ -497,6 +585,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { }) } }, - "issue 5033": c.issue5033, // https://github.com/apptainer/singularity/issues/4836 + "issue 5033": c.issue5033, // https://github.com/apptainer/singularity/issues/4836 + "auth": np(c.testInstanceAuthFile), // custom --authfile with instance start command } } diff --git a/e2e/internal/e2e/dockerhub_auth.go b/e2e/internal/e2e/dockerhub_auth.go index 3dfe327a63..40da72a41e 100644 --- a/e2e/internal/e2e/dockerhub_auth.go +++ b/e2e/internal/e2e/dockerhub_auth.go @@ -10,15 +10,13 @@ package e2e import ( - "encoding/json" "os" "path/filepath" "testing" + "github.com/apptainer/apptainer/internal/pkg/remote/credential/ociauth" "github.com/apptainer/apptainer/internal/pkg/util/user" "github.com/apptainer/apptainer/pkg/syfs" - "github.com/docker/cli/cli/config/configfile" - "github.com/docker/cli/cli/config/types" ) const dockerHub = "docker.io" @@ -45,19 +43,7 @@ func SetupDockerHubCredentials(t *testing.T) { func writeDockerHubCredentials(t *testing.T, dir, username, pass string) { configPath := filepath.Join(dir, ".apptainer", syfs.DockerConfFile) - cf := configfile.ConfigFile{ - AuthConfigs: map[string]types.AuthConfig{ - dockerHub: { - Username: username, - Password: pass, - }, - }, - } - - configData, err := json.Marshal(cf) - if err != nil { + if err := ociauth.LoginAndStore(dockerHub, username, pass, false, configPath); err != nil { t.Error(err) } - - os.WriteFile(configPath, configData, 0o600) } diff --git a/e2e/internal/e2e/env.go b/e2e/internal/e2e/env.go index da4fec7d6f..55152b1e99 100644 --- a/e2e/internal/e2e/env.go +++ b/e2e/internal/e2e/env.go @@ -13,19 +13,23 @@ package e2e // from specifying which Apptainer binary to use to controlling how Apptainer // environment variables will be set. type TestEnv struct { - CmdPath string // Path to the Apptainer binary to use for the execution of an Apptainer command - ImagePath string // Path to the image that has to be used for the execution of an Apptainer command - SingularityImagePath string // Path to a Singularity image for legacy tests - DebianImagePath string // Path to an image containing a Debian distribution with libc compatible to the host libc - OrasTestImage string // URI to SIF image pushed into local registry with ORAS - TestDir string // Path to the directory from which an Apptainer command needs to be executed - TestRegistry string // Host:Port of local registry - TestRegistryImage string // URI to OCI image pushed into local registry - HomeDir string // HomeDir sets the home directory that will be used for the execution of a command - KeyringDir string // KeyringDir sets the directory where the keyring will be created for the execution of a command (instead of using APPTAINER_KEYSDIR which should be avoided when running e2e tests) - PrivCacheDir string // PrivCacheDir sets the location of the image cache to be used by the Apptainer command to be executed as root (instead of using APPTAINER_CACHE_DIR which should be avoided when running e2e tests) - UnprivCacheDir string // UnprivCacheDir sets the location of the image cache to be used by the Apptainer command to be executed as the unpriv user (instead of using APPTAINER_CACHE_DIR which should be avoided when running e2e tests) - RunDisabled bool - DisableCache bool // DisableCache can be set to disable the cache during the execution of a e2e command - InsecureRegistry string // Insecure registry replaced with nip.io + CmdPath string // Path to the Apptainer binary to use for the execution of an Apptainer command + ImagePath string // Path to the image that has to be used for the execution of an Apptainer command + SingularityImagePath string // Path to a Singularity image for legacy tests + DebianImagePath string // Path to an image containing a Debian distribution with libc compatible to the host libc + OrasTestImage string // URI to SIF image pushed into local registry with ORAS + TestDir string // Path to the directory from which an Apptainer command needs to be executed + TestRegistry string // Host:Port of local registry + TestRegistryImage string // URI to OCI image pushed into local registry + HomeDir string // HomeDir sets the home directory that will be used for the execution of a command + KeyringDir string // KeyringDir sets the directory where the keyring will be created for the execution of a command (instead of using APPTAINER_KEYSDIR which should be avoided when running e2e tests) + PrivCacheDir string // PrivCacheDir sets the location of the image cache to be used by the Apptainer command to be executed as root (instead of using APPTAINER_CACHE_DIR which should be avoided when running e2e tests) + UnprivCacheDir string // UnprivCacheDir sets the location of the image cache to be used by the Apptainer command to be executed as the unpriv user (instead of using APPTAINER_CACHE_DIR which should be avoided when running e2e tests) + RunDisabled bool + DisableCache bool // DisableCache can be set to disable the cache during the execution of a e2e command + InsecureRegistry string // Insecure registry replaced with nip.io + TestRegistryPrivURI string // Transport (docker://) + Host:Port of local registry + path to private location + TestRegistryPrivPath string // Host:Port of local registry + path to private location + TestRegistryPrivImage string // URI to OCI image pushed into private location in local registry + OrasTestPrivImage string // URI to SIF image pushed into local registry with ORAS } diff --git a/e2e/internal/e2e/image.go b/e2e/internal/e2e/image.go index eb171c55af..13bc43cec7 100644 --- a/e2e/internal/e2e/image.go +++ b/e2e/internal/e2e/image.go @@ -13,6 +13,7 @@ import ( "context" "fmt" "io" + "net/url" "os" "os/exec" "path/filepath" @@ -193,6 +194,29 @@ func EnsureORASImage(t *testing.T, env TestEnv) { }) } +var orasPrivImageOnce sync.Once + +func EnsureORASPrivImage(t *testing.T, env TestEnv) { + EnsureImage(t, env) + + ensureMutex.Lock() + defer ensureMutex.Unlock() + + orasPrivImageOnce.Do(func() { + t.Logf("Pushing %s to %s", env.ImagePath, env.OrasTestPrivImage) + env.RunApptainer( + t, + WithProfile(UserProfile), + WithCommand("push"), + WithArgs(env.ImagePath, env.OrasTestPrivImage), + ExpectExit(0), + ) + if t.Failed() { + t.Fatalf("failed to push ORAS image to local private registry") + } + }) +} + // PullImage will pull a test image. func PullImage(t *testing.T, env TestEnv, imageURL string, arch string, path string) { pullMutex.Lock() @@ -246,10 +270,10 @@ func CopyImage(t *testing.T, source, dest string, insecureSource, insecureDest b // We don't want to inadvertently send out credentials over http (!) u := CurrentUser(t) configPath := filepath.Join(u.Dir, ".apptainer", syfs.DockerConfFile) - if !insecureSource { + if !insecureSource || isLocalHost(source) { srcCtx.AuthFilePath = configPath } - if !insecureDest { + if !insecureDest || isLocalHost(dest) { dstCtx.AuthFilePath = configPath } @@ -272,6 +296,23 @@ func CopyImage(t *testing.T, source, dest string, insecureSource, insecureDest b } } +// isLocalHost checks if the host component of a given URI points to the +// localhost. Note that this function returns a boolean: a malformed URI is +// considered a URI whose host does not point to localhost. +func isLocalHost(uri string) bool { + u, err := url.Parse(uri) + if err != nil { + return false + } + + switch u.Hostname() { + case "localhost", "127.0.0.1": + return true + } + + return false +} + // BusyboxImage will provide the path to a local busybox SIF image for the current architecture func BusyboxSIF(t *testing.T) string { busyboxSIF := "testdata/busybox_" + runtime.GOARCH + ".sif" diff --git a/e2e/internal/e2e/private_repo.go b/e2e/internal/e2e/private_repo.go new file mode 100644 index 0000000000..98c6e15c9a --- /dev/null +++ b/e2e/internal/e2e/private_repo.go @@ -0,0 +1,49 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2019-2022 Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Copyright (c) 2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package e2e + +import ( + "testing" +) + +func PrivateRepoLogin(t *testing.T, env TestEnv, profile Profile, reqAuthFile string) { + args := []string{} + if reqAuthFile != "" { + args = append(args, "--authfile", reqAuthFile) + } + args = append(args, "-u", DefaultUsername, "-p", DefaultPassword, env.TestRegistryPrivURI) + env.RunApptainer( + t, + WithProfile(profile), + WithCommand("registry login"), + WithArgs(args...), + ExpectExit(0), + ) +} + +func PrivateRepoLogout(t *testing.T, env TestEnv, profile Profile, reqAuthFile string) { + args := []string{} + if reqAuthFile != "" { + args = append(args, "--authfile", reqAuthFile) + } + args = append(args, env.TestRegistryPrivURI) + env.RunApptainer( + t, + WithProfile(profile), + WithCommand("registry logout"), + WithArgs(args...), + ExpectExit(0), + ) +} diff --git a/e2e/registry/registry.go b/e2e/registry/registry.go index cf1a46ea47..79315b50da 100644 --- a/e2e/registry/registry.go +++ b/e2e/registry/registry.go @@ -11,23 +11,14 @@ package registry import ( - "context" - "errors" "fmt" "io" "os" - "path/filepath" "strings" "testing" "github.com/apptainer/apptainer/e2e/internal/e2e" "github.com/apptainer/apptainer/e2e/internal/testhelper" - "github.com/apptainer/apptainer/pkg/syfs" - useragent "github.com/apptainer/apptainer/pkg/util/user-agent" - "github.com/containers/image/v5/copy" - "github.com/containers/image/v5/docker" - "github.com/containers/image/v5/signature" - "github.com/containers/image/v5/types" ) type ctx struct { @@ -157,10 +148,12 @@ func (c ctx) registryLogin(t *testing.T) { expectExit: 0, }, { - name: "logout KO", + // We've defined asking for logout when the user is already logged + // out as a NOOP - so this should succeed. + name: "already logged-out", command: "registry logout", args: []string{badRegistry}, - expectExit: 255, + expectExit: 0, }, { name: "logout OK", @@ -288,16 +281,19 @@ func (c ctx) registryLoginRepeated(t *testing.T) { } } -// JSON files created by our `remote login` flow should be usable in execution -// flows that use containers/image APIs. -// See https://github.com/sylabs/singularity/issues/2226 -func (c ctx) registryIssue2226(t *testing.T) { - testRegistry := c.env.TestRegistry - testRegistryURI := fmt.Sprintf("docker://%s", testRegistry) - privRepo := fmt.Sprintf("%s/private/e2eprivrepo", testRegistry) - privRepoURI := fmt.Sprintf("docker://%s", privRepo) +// Tests authentication with registries, incl. +// https://github.com/sylabs/singularity/issues/2226, among many other flows. +func (c ctx) registryAuth(t *testing.T) { + t.Run("default", func(t *testing.T) { + c.registryAuthTester(t, false) + }) + t.Run("custom", func(t *testing.T) { + c.registryAuthTester(t, true) + }) +} - tmpdir, tmpdirCleanup := e2e.MakeTempDir(t, "", "issue2226", "") +func (c ctx) registryAuthTester(t *testing.T, withCustomAuthFile bool) { + tmpdir, tmpdirCleanup := e2e.MakeTempDir(t, c.env.TestDir, "registry-auth", "") t.Cleanup(func() { if !t.Failed() { tmpdirCleanup(t) @@ -313,103 +309,95 @@ func (c ctx) registryIssue2226(t *testing.T) { t.Fatalf("could not change cwd to %q: %s", tmpdir, err) } - areWeLoggedIn := false - - privRepoLogin := func() { - c.env.RunApptainer( - t, - e2e.WithProfile(e2e.UserProfile), - e2e.WithCommand("registry login"), - e2e.WithArgs("-u", e2e.DefaultUsername, "-p", e2e.DefaultPassword, testRegistryURI), - e2e.ExpectExit(0), - ) - areWeLoggedIn = true + localAuthFileName := "" + if withCustomAuthFile { + localAuthFileName = "./my_local_authfile" } - privRepoLogout := func() { - c.env.RunApptainer( - t, - e2e.WithProfile(e2e.UserProfile), - e2e.WithCommand("registry logout"), - e2e.WithArgs(testRegistryURI), - e2e.ExpectExit(0), - ) - areWeLoggedIn = false + + authFileArgs := []string{} + if withCustomAuthFile { + authFileArgs = []string{"--authfile", localAuthFileName} } - privRepoLogin() t.Cleanup(func() { - if areWeLoggedIn { - privRepoLogout() - } + e2e.PrivateRepoLogout(t, c.env, e2e.UserProfile, localAuthFileName) }) - policy := &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}} - policyCtx, err := signature.NewPolicyContext(policy) - if err != nil { - t.Fatalf("failed to create new policy context: %v", err) - } - - sourceCtx := &types.SystemContext{ - OCIInsecureSkipTLSVerify: false, - DockerInsecureSkipTLSVerify: types.NewOptionalBool(false), - DockerRegistryUserAgent: useragent.Value(), - } - destCtx := &types.SystemContext{ - OCIInsecureSkipTLSVerify: true, - DockerInsecureSkipTLSVerify: types.NewOptionalBool(true), - DockerRegistryUserAgent: useragent.Value(), - } - - u := e2e.CurrentUser(t) - configPath := filepath.Join(u.Dir, ".apptainer", syfs.DockerConfFile) - sourceCtx.AuthFilePath = configPath - destCtx.AuthFilePath = configPath + orasCustomPushTarget := fmt.Sprintf("oras://%s/authfile-pushtest-oras-alpine:latest", c.env.TestRegistryPrivPath) - source := "docker://alpine:latest" - dest := fmt.Sprintf("%s/my-alpine:latest", privRepoURI) - sourceRef, err := docker.ParseReference(strings.TrimPrefix(source, "docker:")) - if err != nil { - t.Fatalf("failed to parse %s reference: %s", source, err) - } - destRef, err := docker.ParseReference(strings.TrimPrefix(dest, "docker:")) - if err != nil { - t.Fatalf("failed to parse %s reference: %s", dest, err) + tests := []struct { + name string + cmd string + args []string + whileLoggedIn bool + expectExit int + }{ + { + name: "docker pull before auth", + cmd: "pull", + args: []string{"-F", "--disable-cache", "--no-https", c.env.TestRegistryPrivImage}, + whileLoggedIn: false, + expectExit: 255, + }, + { + name: "docker pull", + cmd: "pull", + args: []string{"-F", "--disable-cache", "--no-https", c.env.TestRegistryPrivImage}, + whileLoggedIn: true, + expectExit: 0, + }, + { + name: "noauth docker pull", + cmd: "pull", + args: []string{"-F", "--disable-cache", "--no-https", c.env.TestRegistryPrivImage}, + whileLoggedIn: false, + expectExit: 255, + }, + { + name: "noauth oras push", + cmd: "push", + args: []string{"my-alpine_latest.sif", orasCustomPushTarget}, + whileLoggedIn: false, + expectExit: 255, + }, + { + name: "oras push", + cmd: "push", + args: []string{"my-alpine_latest.sif", orasCustomPushTarget}, + whileLoggedIn: true, + expectExit: 0, + }, + { + name: "noauth oras pull", + cmd: "pull", + args: []string{"-F", "--disable-cache", "--no-https", orasCustomPushTarget}, + whileLoggedIn: false, + expectExit: 255, + }, + { + name: "oras pull", + cmd: "pull", + args: []string{"-F", "--disable-cache", "--no-https", orasCustomPushTarget}, + whileLoggedIn: true, + expectExit: 0, + }, } - _, err = copy.Image(context.Background(), policyCtx, destRef, sourceRef, ©.Options{ - ReportWriter: io.Discard, - SourceCtx: sourceCtx, - DestinationCtx: destCtx, - }) - if err != nil { - var e docker.ErrUnauthorizedForCredentials - if errors.As(err, &e) { - t.Fatalf("Authentication info written by 'registry login' did not work when trying to copy OCI image to private repo (%v)", e) + for _, tt := range tests { + if tt.whileLoggedIn { + e2e.PrivateRepoLogin(t, c.env, e2e.UserProfile, localAuthFileName) + } else { + e2e.PrivateRepoLogout(t, c.env, e2e.UserProfile, localAuthFileName) } - t.Fatalf("Failed to copy %s to %s: %s", source, dest, err) + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.UserProfile), + e2e.WithCommand(tt.cmd), + e2e.WithArgs(append(authFileArgs, tt.args...)...), + e2e.ExpectExit(tt.expectExit), + ) } - - privRepoLogout() - - c.env.RunApptainer( - t, - e2e.AsSubtest("noauth"), - e2e.WithProfile(e2e.UserProfile), - e2e.WithCommand("pull --no-https"), - e2e.WithArgs(dest), - e2e.ExpectExit(255), - ) - - privRepoLogin() - - c.env.RunApptainer( - t, - e2e.AsSubtest("auth"), - e2e.WithProfile(e2e.UserProfile), - e2e.WithCommand("pull --no-https"), - e2e.WithArgs(dest), - e2e.ExpectExit(0), - ) } // E2ETests is the main func to trigger the test suite @@ -426,6 +414,6 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "registry login push private": np(c.registryLoginPushPrivate), "registry login repeated": np(c.registryLoginRepeated), "registry list": np(c.registryList), - "registry issue 2226": np(c.registryIssue2226), + "auth": np(c.registryAuth), } } diff --git a/e2e/suite.go b/e2e/suite.go index c724c19f62..33a298a6bf 100644 --- a/e2e/suite.go +++ b/e2e/suite.go @@ -173,6 +173,9 @@ func Run(t *testing.T) { // Provision local registry testenv.TestRegistryImage = fmt.Sprintf("docker://%s/my-busybox:latest", testenv.TestRegistry) + testenv.TestRegistryPrivURI = fmt.Sprintf("docker://%s", testenv.TestRegistry) + testenv.TestRegistryPrivPath = fmt.Sprintf("%s/private/e2eprivrepo", testenv.TestRegistry) + testenv.TestRegistryPrivImage = fmt.Sprintf("docker://%s/my-alpine:latest", testenv.TestRegistryPrivPath) // Copy small test image (busybox:latest) into local registry from DockerHub insecureSource := false @@ -185,6 +188,11 @@ func Run(t *testing.T) { } e2e.CopyImage(t, "docker://busybox:latest", testenv.TestRegistryImage, insecureSource, true) + // Copy same test image into private location in test registry + e2e.PrivateRepoLogin(t, testenv, e2e.UserProfile, "") + e2e.CopyImage(t, "docker://alpine:latest", testenv.TestRegistryPrivImage, insecureSource, true) + e2e.PrivateRepoLogout(t, testenv, e2e.UserProfile, "") + // SIF base test path, built on demand by e2e.EnsureImage imagePath := path.Join(name, "test.sif") t.Log("Path to test image:", imagePath) @@ -193,6 +201,9 @@ func Run(t *testing.T) { // Local registry ORAS SIF image, built on demand by e2e.EnsureORASImage testenv.OrasTestImage = fmt.Sprintf("oras://%s/oras_test_sif:latest", testenv.TestRegistry) + // Local private registry ORAS SIF image, built on demand by e2e.EnsureORASPrivImage + testenv.OrasTestPrivImage = fmt.Sprintf("oras://%s/oras-alpine:latest", testenv.TestRegistryPrivPath) + t.Cleanup(func() { os.Remove(imagePath) }) diff --git a/internal/app/apptainer/keyserver_login.go b/internal/app/apptainer/keyserver_login.go index 174d91624f..147da78de3 100644 --- a/internal/app/apptainer/keyserver_login.go +++ b/internal/app/apptainer/keyserver_login.go @@ -38,7 +38,7 @@ func KeyserverLogin(usrConfigFile string, args *LoginArgs) (err error) { return err } - if err := c.Login(args.Name, args.Username, args.Password, args.Insecure); err != nil { + if err := c.Login(args.Name, args.Username, args.Password, args.Insecure, ""); err != nil { return fmt.Errorf("while login to %s: %s", args.Name, err) } diff --git a/internal/app/apptainer/keyserver_logout.go b/internal/app/apptainer/keyserver_logout.go index bfdadf6d23..974be856c6 100644 --- a/internal/app/apptainer/keyserver_logout.go +++ b/internal/app/apptainer/keyserver_logout.go @@ -38,7 +38,7 @@ func KeyserverLogout(usrConfigFile, name string) (err error) { } // services - if err := c.Logout(name); err != nil { + if err := c.Logout(name, ""); err != nil { return fmt.Errorf("while verifying token: %v", err) } diff --git a/internal/app/apptainer/registry_login.go b/internal/app/apptainer/registry_login.go index 4f38eeaf63..7cf681f6ac 100644 --- a/internal/app/apptainer/registry_login.go +++ b/internal/app/apptainer/registry_login.go @@ -20,7 +20,7 @@ import ( ) // RegistryLogin logs in to an OCI/Docker registry. -func RegistryLogin(usrConfigFile string, args *LoginArgs) (err error) { +func RegistryLogin(usrConfigFile string, args *LoginArgs, reqAuthFile string) (err error) { // opening config file file, err := os.OpenFile(usrConfigFile, os.O_RDWR|os.O_CREATE, 0o600) if err != nil { @@ -38,7 +38,7 @@ func RegistryLogin(usrConfigFile string, args *LoginArgs) (err error) { return err } - if err := c.Login(args.Name, args.Username, args.Password, args.Insecure); err != nil { + if err := c.Login(args.Name, args.Username, args.Password, args.Insecure, reqAuthFile); err != nil { return fmt.Errorf("while login to %s: %s", args.Name, err) } diff --git a/internal/app/apptainer/registry_logout.go b/internal/app/apptainer/registry_logout.go index 3f5d019d58..b2cba7762a 100644 --- a/internal/app/apptainer/registry_logout.go +++ b/internal/app/apptainer/registry_logout.go @@ -19,7 +19,7 @@ import ( ) // RegistryLogout logs out from an OCI/Docker registry. -func RegistryLogout(usrConfigFile, name string) (err error) { +func RegistryLogout(usrConfigFile, name string, reqAuthFile string) (err error) { // opening config file file, err := os.OpenFile(usrConfigFile, os.O_RDWR|os.O_CREATE, 0o600) if err != nil { @@ -38,7 +38,7 @@ func RegistryLogout(usrConfigFile, name string) (err error) { } // services - if err := c.Logout(name); err != nil { + if err := c.Logout(name, reqAuthFile); err != nil { return fmt.Errorf("while verifying token: %v", err) } diff --git a/internal/app/apptainer/remote_login.go b/internal/app/apptainer/remote_login.go index 5a5b881748..02cc41801d 100644 --- a/internal/app/apptainer/remote_login.go +++ b/internal/app/apptainer/remote_login.go @@ -37,7 +37,7 @@ var ErrLoginAborted = errors.New("user aborted login") // RemoteLogin logs in remote by setting API token // If the supplied remote name is an empty string, it will attempt // to use the default remote. -func RemoteLogin(usrConfigFile string, args *LoginArgs) (err error) { +func RemoteLogin(usrConfigFile string, args *LoginArgs, reqAuthFile string) (err error) { // opening config file file, err := os.OpenFile(usrConfigFile, os.O_RDWR|os.O_CREATE, 0o600) if err != nil { @@ -77,7 +77,7 @@ func RemoteLogin(usrConfigFile string, args *LoginArgs) (err error) { return fmt.Errorf("--tokenfile is only supported for login to a remote endpoint, not OCI (docker/oras) or keyservers") } sylog.Warningf("'remote login' is deprecated for registries or keyservers and will be removed in a future release; running 'registry login'") - return RegistryLogin(usrConfigFile, args) + return RegistryLogin(usrConfigFile, args, reqAuthFile) } // truncating file before writing new contents and syncing to commit file diff --git a/internal/app/apptainer/remote_logout.go b/internal/app/apptainer/remote_logout.go index 443ca94b3f..93e793c781 100644 --- a/internal/app/apptainer/remote_logout.go +++ b/internal/app/apptainer/remote_logout.go @@ -21,7 +21,7 @@ import ( ) // RemoteLogout logs out from an endpoint. -func RemoteLogout(usrConfigFile, name string) (err error) { +func RemoteLogout(usrConfigFile, name string, reqAuthFile string) (err error) { // opening config file file, err := os.OpenFile(usrConfigFile, os.O_RDWR|os.O_CREATE, 0o600) if err != nil { @@ -52,7 +52,7 @@ func RemoteLogout(usrConfigFile, name string) (err error) { } else { // services sylog.Warningf("'remote logout' is deprecated for registries or keyservers and will be removed in a future release; running 'registry logout'") - return RegistryLogout(usrConfigFile, name) + return RegistryLogout(usrConfigFile, name, reqAuthFile) } // truncating file before writing new contents and syncing to commit file diff --git a/internal/pkg/build/sources/conveyorPacker_oci.go b/internal/pkg/build/sources/conveyorPacker_oci.go index 7185247a5c..4b06d41bc1 100644 --- a/internal/pkg/build/sources/conveyorPacker_oci.go +++ b/internal/pkg/build/sources/conveyorPacker_oci.go @@ -27,10 +27,10 @@ import ( "text/template" "github.com/apptainer/apptainer/internal/pkg/build/oci" + "github.com/apptainer/apptainer/internal/pkg/remote/credential/ociauth" "github.com/apptainer/apptainer/internal/pkg/util/shell" sytypes "github.com/apptainer/apptainer/pkg/build/types" "github.com/apptainer/apptainer/pkg/image" - "github.com/apptainer/apptainer/pkg/syfs" "github.com/apptainer/apptainer/pkg/sylog" useragent "github.com/apptainer/apptainer/pkg/util/user-agent" "github.com/containers/image/v5/copy" @@ -160,7 +160,7 @@ func (cp *OCIConveyorPacker) Get(ctx context.Context, b *sytypes.Bundle) (err er DockerAuthConfig: cp.b.Opts.DockerAuthConfig, DockerDaemonHost: cp.b.Opts.DockerDaemonHost, OSChoice: "linux", - AuthFilePath: syfs.SearchDockerConf(), + AuthFilePath: ociauth.ChooseAuthFile(cp.b.Opts.ReqAuthFile), DockerRegistryUserAgent: useragent.Value(), BigFilesTemporaryDir: b.TmpDir, } diff --git a/internal/pkg/build/sources/conveyorPacker_oras.go b/internal/pkg/build/sources/conveyorPacker_oras.go index aff8e30ac5..7c811708f5 100644 --- a/internal/pkg/build/sources/conveyorPacker_oras.go +++ b/internal/pkg/build/sources/conveyorPacker_oras.go @@ -33,7 +33,7 @@ func (cp *OrasConveyorPacker) Get(ctx context.Context, b *types.Bundle) (err err // full uri for name determination and output fullRef := "oras:" + ref - imagePath, err := oras.Pull(ctx, b.Opts.ImgCache, fullRef, b.Opts.TmpDir, b.Opts.DockerAuthConfig, b.Opts.NoHTTPS) + imagePath, err := oras.Pull(ctx, b.Opts.ImgCache, fullRef, b.Opts.TmpDir, b.Opts.DockerAuthConfig, b.Opts.NoHTTPS, b.Opts.ReqAuthFile) if err != nil { return fmt.Errorf("while fetching library image: %v", err) } diff --git a/internal/pkg/client/oci/pull.go b/internal/pkg/client/oci/pull.go index 31ecf0306b..3897339a94 100644 --- a/internal/pkg/client/oci/pull.go +++ b/internal/pkg/client/oci/pull.go @@ -20,21 +20,22 @@ import ( "github.com/apptainer/apptainer/internal/pkg/build" "github.com/apptainer/apptainer/internal/pkg/build/oci" "github.com/apptainer/apptainer/internal/pkg/cache" + "github.com/apptainer/apptainer/internal/pkg/remote/credential/ociauth" "github.com/apptainer/apptainer/internal/pkg/util/fs" buildtypes "github.com/apptainer/apptainer/pkg/build/types" - "github.com/apptainer/apptainer/pkg/syfs" "github.com/apptainer/apptainer/pkg/sylog" useragent "github.com/apptainer/apptainer/pkg/util/user-agent" ocitypes "github.com/containers/image/v5/types" ) type PullOptions struct { - TmpDir string - OciAuth *ocitypes.DockerAuthConfig - DockerHost string - NoHTTPS bool - NoCleanUp bool - Pullarch string + TmpDir string + OciAuth *ocitypes.DockerAuthConfig + DockerHost string + NoHTTPS bool + NoCleanUp bool + Pullarch string + ReqAuthFile string } // pull will build a SIF image into the cache if directTo="", or a specific file if directTo is set. @@ -47,9 +48,9 @@ func pull(ctx context.Context, imgCache *cache.Handle, directTo, pullFrom string sysCtx := &ocitypes.SystemContext{ OCIInsecureSkipTLSVerify: opts.NoHTTPS, DockerAuthConfig: opts.OciAuth, - AuthFilePath: syfs.SearchDockerConf(), DockerRegistryUserAgent: useragent.Value(), BigFilesTemporaryDir: opts.TmpDir, + AuthFilePath: ociauth.ChooseAuthFile(opts.ReqAuthFile), } if opts.Pullarch != "" { if arch, ok := oci.ArchMap[opts.Pullarch]; ok { diff --git a/internal/pkg/client/oras/oras.go b/internal/pkg/client/oras/oras.go index e830731182..eee6d58322 100644 --- a/internal/pkg/client/oras/oras.go +++ b/internal/pkg/client/oras/oras.go @@ -20,6 +20,7 @@ import ( "strings" "github.com/apptainer/apptainer/internal/pkg/client" + "github.com/apptainer/apptainer/internal/pkg/remote/credential/ociauth" "github.com/apptainer/apptainer/pkg/image" "github.com/apptainer/apptainer/pkg/sylog" useragent "github.com/apptainer/apptainer/pkg/util/user-agent" @@ -33,9 +34,9 @@ import ( ) // DownloadImage downloads a SIF image specified by an oci reference to a file using the included credentials -func DownloadImage(ctx context.Context, path, ref string, ociAuth *ocitypes.DockerAuthConfig, noHTTPS bool) error { +func DownloadImage(ctx context.Context, path, ref string, ociAuth *ocitypes.DockerAuthConfig, noHTTPS bool, reqAuthFile string) error { rt := client.NewRoundTripper(ctx, nil) - im, err := remoteImage(ref, ociAuth, noHTTPS, rt) + im, err := remoteImage(ref, ociAuth, noHTTPS, rt, reqAuthFile) if err != nil { rt.ProgressShutdown() return err @@ -114,7 +115,7 @@ func DownloadImage(ctx context.Context, path, ref string, ociAuth *ocitypes.Dock // UploadImage uploads the image specified by path and pushes it to the provided oci reference, // it will use credentials if supplied -func UploadImage(_ context.Context, path, ref string, ociAuth *ocitypes.DockerAuthConfig, noHTTPS bool) error { +func UploadImage(ctx context.Context, path, ref string, ociAuth *ocitypes.DockerAuthConfig, noHTTPS bool, reqAuthFile string) error { // ensure that are uploading a SIF if err := ensureSIF(path); err != nil { return err @@ -138,7 +139,7 @@ func UploadImage(_ context.Context, path, ref string, ociAuth *ocitypes.DockerAu return err } - remoteOpts := []remote.Option{AuthOptn(ociAuth), remote.WithUserAgent(useragent.Value())} + remoteOpts := []remote.Option{ociauth.AuthOptn(ociAuth, reqAuthFile), remote.WithUserAgent(useragent.Value())} if term.IsTerminal(2) { pb := &client.DownloadProgressBar{} progChan := make(chan v1.Update, 1) @@ -186,8 +187,8 @@ func ensureSIF(filepath string) error { } // RefHash returns the digest of the SIF layer of the OCI manifest for supplied ref -func RefHash(_ context.Context, ref string, ociAuth *ocitypes.DockerAuthConfig, noHTTPS bool) (v1.Hash, error) { - im, err := remoteImage(ref, ociAuth, noHTTPS, nil) +func RefHash(ctx context.Context, ref string, ociAuth *ocitypes.DockerAuthConfig, noHTTPS bool, reqAuthFile string) (v1.Hash, error) { + im, err := remoteImage(ref, ociAuth, noHTTPS, nil, reqAuthFile) if err != nil { return v1.Hash{}, err } @@ -245,7 +246,7 @@ func sha256sum(r io.Reader) (result string, nBytes int64, err error) { } // remoteImage returns a v1.Image for the provided remote ref. -func remoteImage(ref string, ociAuth *ocitypes.DockerAuthConfig, noHTTPS bool, rt *client.RoundTripper) (v1.Image, error) { +func remoteImage(ref string, ociAuth *ocitypes.DockerAuthConfig, noHTTPS bool, rt *client.RoundTripper, reqAuthFile string) (v1.Image, error) { ref = strings.TrimPrefix(ref, "oras://") ref = strings.TrimPrefix(ref, "//") @@ -258,7 +259,7 @@ func remoteImage(ref string, ociAuth *ocitypes.DockerAuthConfig, noHTTPS bool, r if err != nil { return nil, fmt.Errorf("invalid reference %q: %w", ref, err) } - remoteOpts := []remote.Option{AuthOptn(ociAuth)} + remoteOpts := []remote.Option{ociauth.AuthOptn(ociAuth, reqAuthFile)} if rt != nil { remoteOpts = append(remoteOpts, remote.WithTransport(rt)) } diff --git a/internal/pkg/client/oras/pull.go b/internal/pkg/client/oras/pull.go index 16fc28a93b..7bf9a5b3bf 100644 --- a/internal/pkg/client/oras/pull.go +++ b/internal/pkg/client/oras/pull.go @@ -21,15 +21,15 @@ import ( ) // pull will pull an oras image into the cache if directTo="", or a specific file if directTo is set. -func pull(ctx context.Context, imgCache *cache.Handle, directTo, pullFrom string, ociAuth *ocitypes.DockerAuthConfig, noHTTPS bool) (imagePath string, err error) { - hash, err := RefHash(ctx, pullFrom, ociAuth, noHTTPS) +func pull(ctx context.Context, imgCache *cache.Handle, directTo, pullFrom string, ociAuth *ocitypes.DockerAuthConfig, noHTTPS bool, reqAuthFile string) (imagePath string, err error) { + hash, err := RefHash(ctx, pullFrom, ociAuth, noHTTPS, reqAuthFile) if err != nil { return "", fmt.Errorf("failed to get checksum for %s: %s", pullFrom, err) } if directTo != "" { sylog.Infof("Downloading oras image") - if err := DownloadImage(ctx, directTo, pullFrom, ociAuth, noHTTPS); err != nil { + if err := DownloadImage(ctx, directTo, pullFrom, ociAuth, noHTTPS, reqAuthFile); err != nil { return "", fmt.Errorf("unable to Download Image: %v", err) } imagePath = directTo @@ -43,7 +43,7 @@ func pull(ctx context.Context, imgCache *cache.Handle, directTo, pullFrom string if !cacheEntry.Exists { sylog.Infof("Downloading oras image") - if err := DownloadImage(ctx, cacheEntry.TmpPath, pullFrom, ociAuth, noHTTPS); err != nil { + if err := DownloadImage(ctx, cacheEntry.TmpPath, pullFrom, ociAuth, noHTTPS, reqAuthFile); err != nil { return "", fmt.Errorf("unable to Download Image: %v", err) } if cacheFileHash, err := ImageHash(cacheEntry.TmpPath); err != nil { @@ -67,7 +67,7 @@ func pull(ctx context.Context, imgCache *cache.Handle, directTo, pullFrom string } // Pull will pull an oras image to the cache or direct to a temporary file if cache is disabled -func Pull(ctx context.Context, imgCache *cache.Handle, pullFrom, tmpDir string, ociAuth *ocitypes.DockerAuthConfig, noHTTPS bool) (imagePath string, err error) { +func Pull(ctx context.Context, imgCache *cache.Handle, pullFrom, tmpDir string, ociAuth *ocitypes.DockerAuthConfig, noHTTPS bool, reqAuthFile string) (imagePath string, err error) { directTo := "" if imgCache.IsDisabled() { @@ -79,18 +79,18 @@ func Pull(ctx context.Context, imgCache *cache.Handle, pullFrom, tmpDir string, sylog.Infof("Downloading oras image to tmp cache: %s", directTo) } - return pull(ctx, imgCache, directTo, pullFrom, ociAuth, noHTTPS) + return pull(ctx, imgCache, directTo, pullFrom, ociAuth, noHTTPS, reqAuthFile) } // PullToFile will pull an oras image to the specified location, through the cache, or directly if cache is disabled -func PullToFile(ctx context.Context, imgCache *cache.Handle, pullTo, pullFrom, _ string, ociAuth *ocitypes.DockerAuthConfig, noHTTPS bool) (imagePath string, err error) { +func PullToFile(ctx context.Context, imgCache *cache.Handle, pullTo, pullFrom, _ string, ociAuth *ocitypes.DockerAuthConfig, noHTTPS bool, reqAuthFile string) (imagePath string, err error) { directTo := "" if imgCache.IsDisabled() { directTo = pullTo sylog.Debugf("Cache disabled, pulling directly to: %s", directTo) } - src, err := pull(ctx, imgCache, directTo, pullFrom, ociAuth, noHTTPS) + src, err := pull(ctx, imgCache, directTo, pullFrom, ociAuth, noHTTPS, reqAuthFile) if err != nil { return "", fmt.Errorf("error fetching image to cache: %v", err) } diff --git a/internal/pkg/remote/credential/login_handler.go b/internal/pkg/remote/credential/login_handler.go index 1a05bb2538..c6a628b716 100644 --- a/internal/pkg/remote/credential/login_handler.go +++ b/internal/pkg/remote/credential/login_handler.go @@ -10,26 +10,17 @@ package credential import ( - "context" "crypto/tls" - "encoding/json" "fmt" "net/http" "net/url" - "os" "time" - "github.com/apptainer/apptainer/internal/pkg/util/fs" + "github.com/apptainer/apptainer/internal/pkg/remote/credential/ociauth" "github.com/apptainer/apptainer/internal/pkg/util/interactive" - "github.com/apptainer/apptainer/pkg/syfs" + "github.com/apptainer/apptainer/pkg/sylog" useragent "github.com/apptainer/apptainer/pkg/util/user-agent" - "github.com/docker/cli/cli/config" - "github.com/docker/cli/cli/config/configfile" - "github.com/docker/cli/cli/config/types" - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" ) // loginHandlers contains the registered handlers by scheme. @@ -37,8 +28,8 @@ var loginHandlers = make(map[string]loginHandler) // loginHandler interface implements login and logout for a specific scheme. type loginHandler interface { - login(url *url.URL, username, password string, insecure bool) (*Config, error) - logout(url *url.URL) error + login(url *url.URL, username, password string, insecure bool, reqAuthFile string) (*Config, error) + logout(url *url.URL, reqAuthFile string) error } func init() { @@ -73,7 +64,7 @@ func ensurePassword(password string) (string, error) { // ociHandler handle login/logout for services with docker:// and oras:// scheme. type ociHandler struct{} -func (h *ociHandler) login(u *url.URL, username, password string, insecure bool) (*Config, error) { +func (h *ociHandler) login(u *url.URL, username, password string, insecure bool, reqAuthFile string) (*Config, error) { if u == nil { return nil, fmt.Errorf("URL not provided for login") } @@ -87,110 +78,52 @@ func (h *ociHandler) login(u *url.URL, username, password string, insecure bool) return nil, err } - if err := checkOCILogin(regName, username, pass, insecure); err != nil { + err = ociauth.LoginAndStore(regName, username, pass, insecure, reqAuthFile) + if err != nil { return nil, err } - ociConfig := syfs.DockerConf() - - cf := configfile.New(syfs.DockerConf()) - if fs.IsFile(ociConfig) { - f, err := os.Open(ociConfig) - if err != nil { - return nil, err - } - defer f.Close() - cf, err = config.LoadFromReader(f) - if err != nil { - return nil, err - } - cf.Filename = syfs.DockerConf() - } - - creds := cf.GetCredentialsStore(regName) - - // DockerHub requires special logic for historical reasons. - serverAddress := regName - if serverAddress == name.DefaultRegistry { - serverAddress = authn.DefaultAuthKey - } - - if err := creds.Store(types.AuthConfig{ - Username: username, - Password: pass, - ServerAddress: serverAddress, - }); err != nil { - return nil, fmt.Errorf("while trying to store new credentials: %w", err) - } - return &Config{ URI: u.String(), Insecure: insecure, }, nil } -func checkOCILogin(regName string, username, password string, insecure bool) error { - regOpts := []name.Option{} - if insecure { - regOpts = []name.Option{name.Insecure} - } - reg, err := name.NewRegistry(regName, regOpts...) - if err != nil { - return err +func (h *ociHandler) logout(u *url.URL, reqAuthFile string) error { + if u == nil { + return fmt.Errorf("URL not provided for logout") } + registry := u.Host + u.Path - auth := authn.FromConfig(authn.AuthConfig{ - Username: username, - Password: password, - }) - - // Creating a new transport pings the registry and works through auth flow. - _, err = transport.NewWithContext(context.TODO(), reg, auth, http.DefaultTransport, nil) + cf, err := ociauth.ConfigFileFromPath(ociauth.ChooseAuthFile(reqAuthFile)) if err != nil { - return err + return fmt.Errorf("while loading existing OCI registry credentials from %q: %w", ociauth.ChooseAuthFile(reqAuthFile), err) } - return nil -} + if _, ok := cf.GetAuthConfigs()[registry]; !ok { + sylog.Warningf("There is no existing login to registry %q.", registry) + return nil + } -func (h *ociHandler) logout(u *url.URL) error { - ociConfig := syfs.DockerConf() - ociConfigNew := syfs.DockerConf() + ".new" - cf := configfile.New(syfs.DockerConf()) - if fs.IsFile(ociConfig) { - f, err := os.Open(ociConfig) - if err != nil { - return err - } - defer f.Close() - cf, err = config.LoadFromReader(f) - if err != nil { - return err - } + creds := cf.GetCredentialsStore(registry) + if _, err := creds.Get(registry); err != nil { + sylog.Warningf("There is no existing login to registry %q.", registry) + return nil } - registry := u.Host + u.Path - if _, ok := cf.AuthConfigs[registry]; !ok { - return fmt.Errorf("%q is not logged in", registry) + if err := creds.Erase(registry); err != nil { + return fmt.Errorf("while deleting OCI credentials for registry %q: %w", registry, err) } - delete(cf.AuthConfigs, registry) + sylog.Infof("Token removed from %s", cf.Filename) - configData, err := json.Marshal(cf) - if err != nil { - return err - } - if err := os.WriteFile(ociConfigNew, configData, 0o600); err != nil { - return err - } - return os.Rename(ociConfigNew, ociConfig) + return nil } // keyserverHandler handle login/logout for keyserver service. type keyserverHandler struct{} -//nolint:revive -func (h *keyserverHandler) login(u *url.URL, username, password string, insecure bool) (*Config, error) { +func (h *keyserverHandler) login(u *url.URL, username, password string, insecure bool, reqAuthFile string) (*Config, error) { pass, err := ensurePassword(password) if err != nil { return nil, err @@ -239,7 +172,6 @@ func (h *keyserverHandler) login(u *url.URL, username, password string, insecure }, nil } -//nolint:revive -func (h *keyserverHandler) logout(u *url.URL) error { +func (h *keyserverHandler) logout(u *url.URL, reqAuthFile string) error { return nil } diff --git a/internal/pkg/remote/credential/manager.go b/internal/pkg/remote/credential/manager.go index e64fe62a35..6b63b086fe 100644 --- a/internal/pkg/remote/credential/manager.go +++ b/internal/pkg/remote/credential/manager.go @@ -20,28 +20,28 @@ var Manager = new(manager) type manager struct{} // Login allows to log into a service like a Docker/OCI registry or a keyserver. -func (m *manager) Login(uri, username, password string, insecure bool) (*Config, error) { +func (m *manager) Login(uri, username, password string, insecure bool, reqAuthFile string) (*Config, error) { u, err := url.Parse(uri) if err != nil { return nil, err } if handler, ok := loginHandlers[u.Scheme]; ok { - return handler.login(u, username, password, insecure) + return handler.login(u, username, password, insecure, reqAuthFile) } return nil, fmt.Errorf("%s transport is not supported", u.Scheme) } // Logout allows to log out from a service like a Docker/OCI registry or a keyserver. -func (m *manager) Logout(uri string) error { +func (m *manager) Logout(uri string, reqAuthFile string) error { u, err := url.Parse(uri) if err != nil { return err } if handler, ok := loginHandlers[u.Scheme]; ok { - return handler.logout(u) + return handler.logout(u, reqAuthFile) } return fmt.Errorf("%s transport is not supported", u.Scheme) diff --git a/internal/pkg/remote/credential/ociauth/ociauth.go b/internal/pkg/remote/credential/ociauth/ociauth.go new file mode 100644 index 0000000000..33161316d8 --- /dev/null +++ b/internal/pkg/remote/credential/ociauth/ociauth.go @@ -0,0 +1,223 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2020, Control Command Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Copyright (c) 2023 Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. +// +// The following code is adapted from: +// +// https://github.com/google/go-containerregistry/blob/v0.15.2/pkg/authn/keychain.go +// +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ociauth + +import ( + "context" + "fmt" + "net/http" + "os" + "sync" + + "github.com/apptainer/apptainer/internal/pkg/util/fs" + "github.com/apptainer/apptainer/pkg/syfs" + "github.com/apptainer/apptainer/pkg/sylog" + ocitypes "github.com/containers/image/v5/types" + "github.com/docker/cli/cli/config" + "github.com/docker/cli/cli/config/configfile" + "github.com/docker/cli/cli/config/types" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" +) + +type apptainerKeychain struct { + mu sync.Mutex + reqAuthFile string +} + +// Resolve implements Keychain. +func (sk *apptainerKeychain) Resolve(target authn.Resource) (authn.Authenticator, error) { + sk.mu.Lock() + defer sk.mu.Unlock() + + cf, err := getCredsFile(ChooseAuthFile(sk.reqAuthFile)) + if err != nil { + if sk.reqAuthFile != "" { + // User specifically requested use of an auth file but relevant + // credentials could not be read from that file; issue warning, but + // proceed with anonymous authentication. + sylog.Warningf("Unable to find matching credentials in specified file (%v); proceeding with anonymous authentication.", err) + } + + // No credentials found; proceed anonymously. + return authn.Anonymous, nil + } + + // See: + // https://github.com/google/ko/issues/90 + // https://github.com/moby/moby/blob/fc01c2b481097a6057bec3cd1ab2d7b4488c50c4/registry/config.go#L397-L404 + var cfg, empty types.AuthConfig + for _, key := range []string{ + target.String(), + target.RegistryStr(), + } { + if key == name.DefaultRegistry { + key = authn.DefaultAuthKey + } + + cfg, err = cf.GetAuthConfig(key) + if err != nil { + return nil, err + } + // cf.GetAuthConfig automatically sets the ServerAddress attribute. Since + // we don't make use of it, clear the value for a proper "is-empty" test. + // See: https://github.com/google/go-containerregistry/issues/1510 + cfg.ServerAddress = "" + if cfg != empty { + break + } + } + + if cfg == empty { + return authn.Anonymous, nil + } + + return authn.FromConfig(authn.AuthConfig{ + Username: cfg.Username, + Password: cfg.Password, + Auth: cfg.Auth, + IdentityToken: cfg.IdentityToken, + RegistryToken: cfg.RegistryToken, + }), nil +} + +func AuthOptn(ociAuth *ocitypes.DockerAuthConfig, reqAuthFile string) remote.Option { + // If explicit credentials in ociAuth were passed in, use those. + if ociAuth != nil { + auth := authn.FromConfig(authn.AuthConfig{ + Username: ociAuth.Username, + Password: ociAuth.Password, + IdentityToken: ociAuth.IdentityToken, + }) + return remote.WithAuth(auth) + } + + return remote.WithAuthFromKeychain(&apptainerKeychain{reqAuthFile: reqAuthFile}) +} + +func getCredsFile(reqAuthFile string) (*configfile.ConfigFile, error) { + authFileToUse := ChooseAuthFile(reqAuthFile) + cf, err := ConfigFileFromPath(authFileToUse) + if err != nil { + return nil, fmt.Errorf("while trying to read OCI credentials from file %q: %w", reqAuthFile, err) + } + + return cf, nil +} + +// ConfigFileFromPath creates a configfile.Configfile object (part of docker/cli +// API) associated with the auth file at path. +func ConfigFileFromPath(path string) (*configfile.ConfigFile, error) { + cf := configfile.New(path) + if fs.IsFile(path) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + cf, err = config.LoadFromReader(f) + if err != nil { + return nil, err + } + cf.Filename = path + } + + return cf, nil +} + +// ChooseAuthFile returns reqAuthFile if it is not empty, or else the default +// location of the OCI registry auth file. +func ChooseAuthFile(reqAuthFile string) string { + if reqAuthFile != "" { + return reqAuthFile + } + + return syfs.DockerConf() +} + +func LoginAndStore(registry, username, password string, insecure bool, reqAuthFile string) error { + if err := checkOCILogin(registry, username, password, insecure); err != nil { + return err + } + + cf, err := ConfigFileFromPath(ChooseAuthFile(reqAuthFile)) + if err != nil { + return fmt.Errorf("while loading existing OCI registry credentials from %q: %w", ChooseAuthFile(reqAuthFile), err) + } + + creds := cf.GetCredentialsStore(registry) + + // DockerHub requires special logic for historical reasons. + serverAddress := registry + if serverAddress == name.DefaultRegistry { + serverAddress = authn.DefaultAuthKey + } + + if err := creds.Store(types.AuthConfig{ + Username: username, + Password: password, + ServerAddress: serverAddress, + }); err != nil { + return fmt.Errorf("while trying to store new credentials: %w", err) + } + + sylog.Infof("Token stored in %s", cf.Filename) + + return nil +} + +func checkOCILogin(regName string, username, password string, insecure bool) error { + regOpts := []name.Option{} + if insecure { + regOpts = []name.Option{name.Insecure} + } + reg, err := name.NewRegistry(regName, regOpts...) + if err != nil { + return err + } + + auth := authn.FromConfig(authn.AuthConfig{ + Username: username, + Password: password, + }) + + // Creating a new transport pings the registry and works through auth flow. + _, err = transport.NewWithContext(context.TODO(), reg, auth, http.DefaultTransport, nil) + if err != nil { + return err + } + + return nil +} diff --git a/internal/pkg/remote/remote.go b/internal/pkg/remote/remote.go index 07704667de..1dbbefbf78 100644 --- a/internal/pkg/remote/remote.go +++ b/internal/pkg/remote/remote.go @@ -226,7 +226,7 @@ func (c *Config) GetRemote(name string) (*endpoint.Config, error) { // Login validates and stores credentials for a service like Docker/OCI registries // and keyservers. -func (c *Config) Login(uri, username, password string, insecure bool) error { +func (c *Config) Login(uri, username, password string, insecure bool, reqAuthFile string) error { _, err := remoteutil.NormalizeKeyserverURI(uri) // if there is no error, we consider it as a keyserver if err == nil { @@ -256,11 +256,17 @@ func (c *Config) Login(uri, username, password string, insecure bool) error { } } - credConfig, err := credential.Manager.Login(uri, username, password, insecure) + credConfig, err := credential.Manager.Login(uri, username, password, insecure, reqAuthFile) if err != nil { return err } + // If we're manipulating an auth-file requested via `--authfile`, don't + // update remote.yaml + if reqAuthFile != "" { + return nil + } + // Remove any existing remote.yaml entry for the same URI. // Older versions of Apptainer can create duplicate entries with same URI, // so loop must handle removing multiple matches (#214). @@ -277,10 +283,17 @@ func (c *Config) Login(uri, username, password string, insecure bool) error { } // Logout removes previously stored credentials for a service. -func (c *Config) Logout(uri string) error { - if err := credential.Manager.Logout(uri); err != nil { +func (c *Config) Logout(uri string, reqAuthFile string) error { + if err := credential.Manager.Logout(uri, reqAuthFile); err != nil { return err } + + // If we're manipulating an auth-file requested via `--authfile`, don't + // update remote.yaml + if reqAuthFile != "" { + return nil + } + // Older versions of Apptainer can create duplicate entries with same URI, // so loop must handle removing multiple matches (#214). for i := 0; i < len(c.Credentials); i++ { diff --git a/pkg/build/types/bundle.go b/pkg/build/types/bundle.go index 087774dd04..8bb9d3abf9 100644 --- a/pkg/build/types/bundle.go +++ b/pkg/build/types/bundle.go @@ -88,6 +88,8 @@ type Options struct { Unprivilege bool // Arch info Arch string + // Authentication file for registry credentials + ReqAuthFile string } // NewEncryptedBundle creates an Encrypted Bundle environment.