Skip to content

Commit

Permalink
add progressbar support for apptainer push
Browse files Browse the repository at this point in the history
Signed-off-by: jason yang <jasonyangshadow@gmail.com>
  • Loading branch information
JasonYangShadow committed Jul 27, 2023
1 parent 1f4fba4 commit 0d8948b
Show file tree
Hide file tree
Showing 5 changed files with 118 additions and 3 deletions.
12 changes: 12 additions & 0 deletions LICENSE_DEPENDENCIES.md
Original file line number Diff line number Diff line change
Expand Up @@ -845,6 +845,12 @@ The dependencies and their licenses are as follows:

**License URL:** <https://github.com/mattn/go-runewidth/blob/master/LICENSE>

## github.com/mitchellh/colorstring

**License:** MIT

**License URL:** <https://github.com/mitchellh/colorstring/blob/master/LICENSE>

## github.com/mitchellh/mapstructure

**License:** MIT
Expand Down Expand Up @@ -875,6 +881,12 @@ The dependencies and their licenses are as follows:

**License URL:** <https://github.com/samber/lo/blob/master/LICENSE>

## github.com/schollz/progressbar/v3

**License:** MIT

**License URL:** <https://github.com/schollz/progressbar/blob/master/v3/LICENSE>

## github.com/secure-systems-lab/go-securesystemslib/dsse

**License:** MIT
Expand Down
4 changes: 3 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ require (
github.com/BurntSushi/toml v1.3.2
github.com/docker/distribution v2.8.2+incompatible
github.com/samber/lo v1.38.1
github.com/schollz/progressbar/v3 v3.13.1
github.com/sirupsen/logrus v1.9.3
gopkg.in/yaml.v2 v2.4.0
)
Expand Down Expand Up @@ -128,10 +129,11 @@ require (
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mattn/go-shellwords v1.0.12 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/miekg/pkcs11 v1.1.1 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/locker v1.0.1 // indirect
github.com/moby/patternmatcher v0.5.0 // indirect
Expand Down
9 changes: 8 additions & 1 deletion go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,7 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
Expand Down Expand Up @@ -363,8 +364,9 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
Expand All @@ -373,6 +375,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU=
github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
Expand Down Expand Up @@ -491,6 +495,8 @@ github.com/safchain/ethtool v0.3.0 h1:gimQJpsI6sc1yIqP/y8GYgiXn/NjgvpM0RNoWLVVmP
github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6ocLW+CJUs=
github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM=
github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=
github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ=
github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y=
github.com/seccomp/containers-golang v0.6.0 h1:VWPMMIDr8pAtNjCX0WvLEEK9EQi5lAm4HtJbDtAtFvQ=
github.com/seccomp/containers-golang v0.6.0/go.mod h1:Dd9mONHvW4YdbSzdm23yf2CFw0iqvqLhO0mEFvPIvm4=
Expand Down Expand Up @@ -722,6 +728,7 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
2 changes: 1 addition & 1 deletion internal/pkg/client/oras/oras.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ func UploadImage(ctx context.Context, path, ref string, ociAuth *ocitypes.Docker
sylog.Infof("No tag or digest found, using default: %s", SifDefaultTag)
}

resolver, err := getResolver(ctx, ociAuth, noHTTPS, true)
resolver, err := getOrasResolver(ctx, ociAuth, noHTTPS, true)
if err != nil {
return fmt.Errorf("while getting resolver: %s", err)
}
Expand Down
94 changes: 94 additions & 0 deletions internal/pkg/client/oras/resolver.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package oras

import (
"context"
"io"
"net/http"

"github.com/apptainer/apptainer/pkg/syfs"
"github.com/apptainer/apptainer/pkg/sylog"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
ocitypes "github.com/containers/image/v5/types"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/schollz/progressbar/v3"
oras_docker "oras.land/oras-go/pkg/auth/docker"
)

type OrasResolver struct {
remotes.Resolver
}

type orasPusher struct {
remotes.Pusher
}

type contentWriter struct {
content.Writer
multiWriter io.Writer
}

func NewResolver(options docker.ResolverOptions) remotes.Resolver {
return &OrasResolver{docker.NewResolver(options)}
}

func getOrasResolver(ctx context.Context, ociAuth *ocitypes.DockerAuthConfig, noHTTPS, push bool) (remotes.Resolver, error) {
opts := docker.ResolverOptions{Credentials: genCredfn(ociAuth), PlainHTTP: noHTTPS}
if ociAuth != nil && (ociAuth.Username != "" || ociAuth.Password != "") {
return NewResolver(opts), nil
}

cli, err := oras_docker.NewClient(syfs.DockerConf())
if err != nil {
sylog.Warningf("Couldn't load auth credential file: %s", err)
return NewResolver(opts), nil
}
httpClient := &http.Client{}

if push {
httpClient.Transport = newOrasUploadTransport()
}

resolver, err := cli.Resolver(ctx, httpClient, noHTTPS)
if err != nil {
return &OrasResolver{}, err
}

return &OrasResolver{resolver}, nil
}

func (r *OrasResolver) Resolve(ctx context.Context, ref string) (name string, desc ocispec.Descriptor, err error) {
return r.Resolver.Resolve(ctx, ref)
}

func (r *OrasResolver) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) {
return r.Resolver.Fetcher(ctx, ref)
}

func (r *OrasResolver) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) {
pusher, err := r.Resolver.Pusher(ctx, ref)
if err != nil {
return &orasPusher{}, err
}

return &orasPusher{pusher}, nil
}

func (p *orasPusher) Push(ctx context.Context, desc ocispec.Descriptor) (content.Writer, error) {
writer, err := p.Pusher.Push(ctx, desc)
if err != nil {
return &contentWriter{}, err
}

bar := progressbar.DefaultBytes(desc.Size)
mwriter := io.MultiWriter(writer, bar)
return &contentWriter{
multiWriter: mwriter,
Writer: writer,
}, nil
}

func (w *contentWriter) Write(p []byte) (n int, err error) {
return w.multiWriter.Write(p)
}

0 comments on commit 0d8948b

Please sign in to comment.