From 9969df4f1cbbaea87c0d704cb482b817114deddf Mon Sep 17 00:00:00 2001 From: Morlay Date: Wed, 22 May 2024 16:14:57 +0800 Subject: [PATCH] feat: ocitar reader --- Makefile | 2 +- cue.mod/module.cue | 4 +- cue.mod/module_overwrites.cue | 6 +- cuepkg/component/crkit/container-registry.cue | 2 +- .../component/crkit/containerd-operator.cue | 2 +- cuepkg/component/crkit/z_common.cue | 2 +- go.mod | 34 +-- go.sum | 64 ++--- pkg/artifact/image.go | 59 ++--- pkg/kubepkg/iter.go | 125 +++++++++ pkg/kubepkg/packer.go | 56 ++-- pkg/kubepkg/packer_test.go | 240 ++++++++++++++---- pkg/kubepkg/puller.go | 3 - pkg/kubepkg/pusher.go | 52 ++++ pkg/kubepkg/registry.go | 44 ++++ pkg/kubepkg/testdata/example.kubepkg.json | 2 +- pkg/ocitar/error.go | 15 ++ pkg/ocitar/index.go | 218 ++++++++++++++++ pkg/ocitar/progess.go | 54 ---- pkg/ocitar/tar_reader.go | 46 ++++ pkg/ocitar/write.go | 54 +--- pkg/ocitar/write_test.go | 30 ++- pkg/ociutil/discriptor.go | 1 + pkg/ociutil/raw.go | 45 ++++ 24 files changed, 896 insertions(+), 264 deletions(-) create mode 100644 pkg/kubepkg/iter.go delete mode 100644 pkg/kubepkg/puller.go create mode 100644 pkg/kubepkg/pusher.go create mode 100644 pkg/kubepkg/registry.go create mode 100644 pkg/ocitar/error.go create mode 100644 pkg/ocitar/index.go delete mode 100644 pkg/ocitar/progess.go create mode 100644 pkg/ocitar/tar_reader.go create mode 100644 pkg/ociutil/discriptor.go create mode 100644 pkg/ociutil/raw.go diff --git a/Makefile b/Makefile index 5c4aafc..70c8517 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PIPER = piper -p piper.cue +PIPER = TTY=0 piper -p piper.cue DEBUG = 0 ifeq ($(DEBUG),1) PIPER := $(PIPER) --log-level=debug diff --git a/cue.mod/module.cue b/cue.mod/module.cue index 82260f0..b91d730 100755 --- a/cue.mod/module.cue +++ b/cue.mod/module.cue @@ -1,7 +1,7 @@ module: "github.com/octohelm/crkit@v0" deps: { - "github.com/octohelm/kubepkg@v0": { - v: "v0.5.4" + "github.com/octohelm/kubepkgspec@v0": { + v: "v0.0.0-20240521102121-31a405691640" } "github.com/octohelm/piper@v0": { v: "v0.0.0-20240427003204-9004d473b957" diff --git a/cue.mod/module_overwrites.cue b/cue.mod/module_overwrites.cue index cbd3142..7b00432 100755 --- a/cue.mod/module_overwrites.cue +++ b/cue.mod/module_overwrites.cue @@ -1,7 +1,7 @@ deps: { - "github.com/octohelm/kubepkg@v0": { - path: "github.com/octohelm/kubepkg" - v: "v0.5.4" + "github.com/octohelm/kubepkgspec@v0": { + path: "github.com/octohelm/kubepkgspec" + v: "v0.0.0-20240521102121-31a405691640" } "github.com/octohelm/piper@v0": { path: "github.com/octohelm/piper" diff --git a/cuepkg/component/crkit/container-registry.cue b/cuepkg/component/crkit/container-registry.cue index 17d48de..666eec6 100755 --- a/cuepkg/component/crkit/container-registry.cue +++ b/cuepkg/component/crkit/container-registry.cue @@ -1,7 +1,7 @@ package crkit import ( - kubepkg "github.com/octohelm/kubepkg/cuepkg/kubepkg" + kubepkg "github.com/octohelm/kubepkgspec/cuepkg/kubepkg" ) #ContainerRegistry: kubepkg.#KubePkg & { diff --git a/cuepkg/component/crkit/containerd-operator.cue b/cuepkg/component/crkit/containerd-operator.cue index 7a1a2b0..8bda132 100755 --- a/cuepkg/component/crkit/containerd-operator.cue +++ b/cuepkg/component/crkit/containerd-operator.cue @@ -1,7 +1,7 @@ package crkit import ( - kubepkg "github.com/octohelm/kubepkg/cuepkg/kubepkg" + kubepkg "github.com/octohelm/kubepkgspec/cuepkg/kubepkg" ) #ContainerdOperator: kubepkg.#KubePkg & { diff --git a/cuepkg/component/crkit/z_common.cue b/cuepkg/component/crkit/z_common.cue index 4d3eea8..1aeed72 100644 --- a/cuepkg/component/crkit/z_common.cue +++ b/cuepkg/component/crkit/z_common.cue @@ -1,7 +1,7 @@ package crkit import ( - kubepkg "github.com/octohelm/kubepkg/cuepkg/kubepkg" + kubepkg "github.com/octohelm/kubepkgspec/cuepkg/kubepkg" ) #Storage: kubepkg.#Volume & { diff --git a/go.mod b/go.mod index 22da1a0..016ad29 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/octohelm/crkit -go 1.22.2 +go 1.22.3 require ( github.com/containerd/containerd v1.7.17 @@ -8,7 +8,7 @@ require ( github.com/distribution/reference v0.6.0 github.com/go-courier/logr v0.3.0 github.com/google/go-containerregistry v0.19.1 - github.com/innoai-tech/infra v0.0.0-20240516024759-9ce40e1bf32e + github.com/innoai-tech/infra v0.0.0-20240523040540-feecd286e824 github.com/octohelm/courier v0.0.0-20240516021431-5cf7af6666a1 github.com/octohelm/gengo v0.0.0-20240510051519-974fb897453b github.com/octohelm/kubekit v0.0.0-20240508035712-15cb61729772 @@ -50,7 +50,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-json-experiment/json v0.0.0-20240418180308-af2d5061e6c2 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect @@ -92,18 +92,18 @@ require ( github.com/spf13/cobra v1.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/vbatts/tar-split v0.11.3 // indirect - go.opentelemetry.io/contrib/propagators/b3 v1.26.0 // indirect - go.opentelemetry.io/otel v1.26.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.26.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.26.0 // indirect - go.opentelemetry.io/otel/exporters/prometheus v0.48.0 // indirect - go.opentelemetry.io/otel/log v0.2.0-alpha // indirect - go.opentelemetry.io/otel/metric v1.26.0 // indirect - go.opentelemetry.io/otel/sdk v1.26.0 // indirect - go.opentelemetry.io/otel/sdk/log v0.2.0-alpha // indirect - go.opentelemetry.io/otel/sdk/metric v1.26.0 // indirect - go.opentelemetry.io/otel/trace v1.26.0 // indirect + go.opentelemetry.io/contrib/propagators/b3 v1.27.0 // indirect + go.opentelemetry.io/otel v1.27.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect + go.opentelemetry.io/otel/exporters/prometheus v0.49.0 // indirect + go.opentelemetry.io/otel/log v0.3.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/otel/sdk v1.27.0 // indirect + go.opentelemetry.io/otel/sdk/log v0.3.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect + go.opentelemetry.io/otel/trace v1.27.0 // indirect go.opentelemetry.io/proto/otlp v1.2.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.17.0 // indirect @@ -115,8 +115,8 @@ require ( golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240515191416-fc5f0ca64291 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect google.golang.org/grpc v1.64.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 98767d1..e847332 100644 --- a/go.sum +++ b/go.sum @@ -78,8 +78,8 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= @@ -135,8 +135,8 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/innoai-tech/infra v0.0.0-20240516024759-9ce40e1bf32e h1:nJCbZpw3YlrmIq0+o5QTA3dnvVPt9mZs0mmBMYhSrOU= -github.com/innoai-tech/infra v0.0.0-20240516024759-9ce40e1bf32e/go.mod h1:YWZbHJVJX61BE/+yACep0bJNoexYKyiGt5IfZQD4qUo= +github.com/innoai-tech/infra v0.0.0-20240523040540-feecd286e824 h1:iXgqOR/SC+1GqiKTFnNmRTR1+hvUu03hmXrClly9how= +github.com/innoai-tech/infra v0.0.0-20240523040540-feecd286e824/go.mod h1:f1QHGzKkGdi68vEMZ9Wpujv2ZY/1v8kANH7O0ZJ6T/Q= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -271,30 +271,30 @@ github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RV github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/contrib/propagators/b3 v1.26.0 h1:wgFbVA+bK2k+fGVfDOCOG4cfDAoppyr5sI2dVlh8MWM= -go.opentelemetry.io/contrib/propagators/b3 v1.26.0/go.mod h1:DDktFXxA+fyItAAM0Sbl5OBH7KOsCTjvbBdPKtoIf/k= -go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= -go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.26.0 h1:+hm+I+KigBy3M24/h1p/NHkUx/evbLH0PNcjpMyCHc4= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.26.0/go.mod h1:NjC8142mLvvNT6biDpaMjyz78kyEHIwAJlSX0N9P5KI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 h1:1u/AyyOqAWzy+SkPxDpahCNZParHV8Vid1RnI2clyDE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0/go.mod h1:z46paqbJ9l7c9fIPCXTqTGwhQZ5XoTIsfeFYWboizjs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.26.0 h1:Waw9Wfpo/IXzOI8bCB7DIk+0JZcqqsyn1JFnAc+iam8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.26.0/go.mod h1:wnJIG4fOqyynOnnQF/eQb4/16VlX2EJAHhHgqIqWfAo= -go.opentelemetry.io/otel/exporters/prometheus v0.48.0 h1:sBQe3VNGUjY9IKWQC6z2lNqa5iGbDSxhs60ABwK4y0s= -go.opentelemetry.io/otel/exporters/prometheus v0.48.0/go.mod h1:DtrbMzoZWwQHyrQmCfLam5DZbnmorsGbOtTbYHycU5o= -go.opentelemetry.io/otel/log v0.2.0-alpha h1:ixOPvMzserpqA07SENHvRzkZOsnG0XbPr74hv1AQ+n0= -go.opentelemetry.io/otel/log v0.2.0-alpha/go.mod h1:vbFZc65yq4c4ssvXY43y/nIqkNJLxORrqw0L85P59LA= -go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= -go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= -go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8= -go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs= -go.opentelemetry.io/otel/sdk/log v0.2.0-alpha h1:jGTkL/jroJ31jnP6jDl34N/mDOfRGGYZHcHsCM+5kWA= -go.opentelemetry.io/otel/sdk/log v0.2.0-alpha/go.mod h1:Hd8Lw9FPGUM3pfY7iGMRvFaC2Nyau4Ajb5WnQ9OdIho= -go.opentelemetry.io/otel/sdk/metric v1.26.0 h1:cWSks5tfriHPdWFnl+qpX3P681aAYqlZHcAyHw5aU9Y= -go.opentelemetry.io/otel/sdk/metric v1.26.0/go.mod h1:ClMFFknnThJCksebJwz7KIyEDHO+nTB6gK8obLy8RyE= -go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= -go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.opentelemetry.io/contrib/propagators/b3 v1.27.0 h1:IjgxbomVrV9za6bRi8fWCNXENs0co37SZedQilP2hm0= +go.opentelemetry.io/contrib/propagators/b3 v1.27.0/go.mod h1:Dv9obQz25lCisDvvs4dy28UPh974CxkahRDUPsY7y9E= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 h1:bFgvUr3/O4PHj3VQcFEuYKvRZJX1SJDQ+11JXuSB3/w= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0/go.mod h1:xJntEd2KL6Qdg5lwp97HMLQDVeAhrYxmzFseAMDPQ8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= +go.opentelemetry.io/otel/exporters/prometheus v0.49.0 h1:Er5I1g/YhfYv9Affk9nJLfH/+qCCVVg1f2R9AbJfqDQ= +go.opentelemetry.io/otel/exporters/prometheus v0.49.0/go.mod h1:KfQ1wpjf3zsHjzP149P4LyAwWRupc6c7t1ZJ9eXpKQM= +go.opentelemetry.io/otel/log v0.3.0 h1:kJRFkpUFYtny37NQzL386WbznUByZx186DpEMKhEGZs= +go.opentelemetry.io/otel/log v0.3.0/go.mod h1:ziCwqZr9soYDwGNbIL+6kAvQC+ANvjgG367HVcyR/ys= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/sdk/log v0.3.0 h1:GEjJ8iftz2l+XO1GF2856r7yYVh74URiF9JMcAacr5U= +go.opentelemetry.io/otel/sdk/log v0.3.0/go.mod h1:BwCxtmux6ACLuys1wlbc0+vGBd+xytjmjajwqqIul2g= +go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= +go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -362,10 +362,10 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto/googleapis/api v0.0.0-20240515191416-fc5f0ca64291 h1:4HZJ3Xv1cmrJ+0aFo304Zn79ur1HMxptAE7aCPNLSqc= -google.golang.org/genproto/googleapis/api v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 h1:Q2RxlXqh1cgzzUgV261vBO2jI5R/3DD1J2pM0nI4NhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= diff --git a/pkg/artifact/image.go b/pkg/artifact/image.go index 770cce9..9f9a3e8 100644 --- a/pkg/artifact/image.go +++ b/pkg/artifact/image.go @@ -3,6 +3,7 @@ package artifact import ( "bytes" "encoding/json" + "github.com/octohelm/crkit/pkg/ociutil" "sync/atomic" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -12,51 +13,46 @@ import ( specv1 "github.com/opencontainers/image-spec/specs-go/v1" ) -func Artifact(img v1.Image, c Config) (v1.Image, error) { - return &artifactImage{ +func WithAnnotations(annotations map[string]string) Option { + return func(i *artifactImage) { + i.annotations = annotations + } +} + +type Option = func(i *artifactImage) + +func Artifact(img v1.Image, c Config, optFns ...Option) (v1.Image, error) { + i := &artifactImage{ Image: img, config: c, - }, nil -} + } -type artifactImage struct { - v1.Image - config Config + for _, optFn := range optFns { + optFn(i) + } - m atomic.Pointer[specv1.Manifest] + return ociutil.FromRaw(i), nil } -func (img *artifactImage) ArtifactType() (string, error) { - return img.config.ArtifactType() +type artifactImage struct { + v1.Image + config Config + annotations map[string]string + m atomic.Pointer[specv1.Manifest] } func (img *artifactImage) MediaType() (types.MediaType, error) { return types.OCIManifestSchema1, nil } -func (i *artifactImage) ConfigName() (v1.Hash, error) { - return partial.ConfigName(i) +func (img *artifactImage) ArtifactType() (string, error) { + return img.config.ArtifactType() } func (img *artifactImage) RawConfigFile() ([]byte, error) { return img.config.RawConfigFile() } -func (img *artifactImage) Manifest() (*v1.Manifest, error) { - raw, err := img.RawManifest() - if err != nil { - return nil, err - } - - m := &v1.Manifest{} - - if err := json.Unmarshal(raw, m); err != nil { - return nil, err - } - - return m, nil -} - func (img *artifactImage) RawManifest() ([]byte, error) { m, err := img.OCIManifest() if err != nil { @@ -103,6 +99,7 @@ func (img *artifactImage) OCIManifest() (*specv1.Manifest, error) { Size: cfgSize, Digest: digest.Digest(cfgHash.String()), }, + Annotations: img.annotations, } m.SchemaVersion = 2 @@ -142,11 +139,3 @@ func (img *artifactImage) OCIManifest() (*specv1.Manifest, error) { return m, nil } - -func (img *artifactImage) Size() (int64, error) { - return partial.Size(img) -} - -func (img *artifactImage) Digest() (v1.Hash, error) { - return partial.Digest(img) -} diff --git a/pkg/kubepkg/iter.go b/pkg/kubepkg/iter.go new file mode 100644 index 0000000..9f3d4df --- /dev/null +++ b/pkg/kubepkg/iter.go @@ -0,0 +1,125 @@ +package kubepkg + +import ( + "encoding/json" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + kubepkgv1alpha1 "github.com/octohelm/kubepkgspec/pkg/apis/kubepkg/v1alpha1" + specv1 "github.com/opencontainers/image-spec/specs-go/v1" + "iter" +) + +func KubePkg(idx v1.ImageIndex) (*kubepkgv1alpha1.KubePkg, error) { + indexManifest, err := idx.IndexManifest() + if err != nil { + return nil, err + } + + for _, m := range indexManifest.Manifests { + if m.ArtifactType == ArtifactType { + img, err := idx.Image(m.Digest) + if err != nil { + return nil, err + } + + rawConfig, err := img.RawConfigFile() + if err != nil { + return nil, err + } + kpkg := &kubepkgv1alpha1.KubePkg{} + if err := json.Unmarshal(rawConfig, kpkg); err != nil { + return nil, err + } + return kpkg, nil + } + } + + return nil, nil +} + +func NewImageIter(idx v1.ImageIndex) ImageIter { + return &imageIter{ + ImageIndex: idx, + } +} + +type ImageIter interface { + Images() iter.Seq2[name.Reference, remote.Taggable] + Err() error +} + +type imageIter struct { + ImageIndex v1.ImageIndex + err error +} + +func (r *imageIter) Err() error { + return r.err +} + +func (r *imageIter) Done(err error) { + r.err = err +} + +func (p *imageIter) Repository(repoName string) (name.Repository, error) { + return name.NewRepository(repoName) +} + +func (r *imageIter) Images() iter.Seq2[name.Reference, remote.Taggable] { + indexManifest, err := r.ImageIndex.IndexManifest() + if err != nil { + r.Done(err) + + return func(yield func(name.Reference, remote.Taggable) bool) { + + } + } + + return func(yield func(name.Reference, remote.Taggable) bool) { + for _, d := range indexManifest.Manifests { + if d.Annotations == nil { + continue + } + + imageName := d.Annotations[specv1.AnnotationBaseImageName] + if imageName == "" { + continue + } + + tag := d.Annotations[specv1.AnnotationRefName] + if tag == "" { + continue + } + + repo, err := r.Repository(imageName) + if err != nil { + r.Done(err) + return + } + + ref := repo.Tag(tag) + + if d.MediaType.IsImage() { + img, err := r.ImageIndex.Image(d.Digest) + if err != nil { + r.Done(err) + return + } + + if !yield(ref, img) { + return + } + } else if d.MediaType.IsIndex() { + index, err := r.ImageIndex.ImageIndex(d.Digest) + if err != nil { + r.Done(err) + return + } + if !yield(ref, index) { + return + } + } + } + } +} diff --git a/pkg/kubepkg/packer.go b/pkg/kubepkg/packer.go index 2dfefde..4a58d38 100644 --- a/pkg/kubepkg/packer.go +++ b/pkg/kubepkg/packer.go @@ -2,6 +2,7 @@ package kubepkg import ( "context" + "fmt" "github.com/octohelm/kubepkgspec/pkg/kubepkg" "iter" "sort" @@ -29,12 +30,13 @@ const ( ) type Packer struct { - CreatePuller func(repo name.Repository, options ...remote.Option) (*remote.Puller, error) + Registry Registry + Renamer Renamer + + CreatePuller func(ref name.Reference, options ...remote.Option) (*remote.Puller, error) + Cache cache.Cache - Registry *name.Registry Platforms []string - Renamer Renamer - sourceImages sync.Map } @@ -78,17 +80,13 @@ func (p *Packer) SupportedPlatforms(supportedPlatform []string) iter.Seq[v1.Plat func (p *Packer) Repository(repoName string) (name.Repository, error) { if registry := p.Registry; registry != nil { - registryName := registry.Name() - if strings.HasPrefix(repoName, registryName) { - return registry.Repo(repoName[len(registryName)+1:]), nil - } return registry.Repo(repoName), nil } return name.NewRepository(repoName) } -func (p *Packer) Puller(repo name.Repository, options ...remote.Option) (*remote.Puller, error) { - puller, err := p.CreatePuller(repo, options...) +func (p *Packer) Puller(ref name.Reference, options ...remote.Option) (*remote.Puller, error) { + puller, err := p.CreatePuller(ref, options...) if err != nil { return nil, err } @@ -110,7 +108,20 @@ func (p *Packer) PackAsIndex(ctx context.Context, kpkg *kubepkgv1alpha1.KubePkg) var finalIndex v1.ImageIndex = empty.Index - finalIndex, err = p.appendManifests(finalIndex, kubePkgImage, nil, nil) + namespace := kpkg.Namespace + if namespace == "" { + namespace = "default" + } + + r, err := p.Repository(fmt.Sprintf("%s/artifact-kubepkg-%s", namespace, kpkg.Name)) + if err != nil { + return nil, err + } + + finalIndex, err = p.appendManifests(finalIndex, kubePkgImage, nil, &kubepkgv1alpha1.Image{ + Name: p.ImageName(r), + Tag: kpkg.Spec.Version, + }) if err != nil { return nil, err } @@ -143,7 +154,7 @@ func (p *Packer) PackAsIndex(ctx context.Context, kpkg *kubepkgv1alpha1.KubePkg) imageIndexes[imageName] = empty.Index } - puller, err := p.Puller(repo) + puller, err := p.Puller(repo.Digest(desc.Digest.String())) if err != nil { return nil, err } @@ -217,7 +228,7 @@ func (p *Packer) PackAsKubePkgImage(ctx context.Context, kpkg *kubepkgv1alpha1.K image.Digest = "" for platform := range p.SupportedPlatforms(image.Platforms) { - puller, err := p.CreatePuller(repo, remote.WithPlatform(platform)) + puller, err := p.CreatePuller(repo.Tag(image.Tag), remote.WithPlatform(platform)) if err != nil { return nil, err } @@ -244,7 +255,9 @@ func (p *Packer) PackAsKubePkgImage(ctx context.Context, kpkg *kubepkgv1alpha1.K } } - return artifact.Artifact(kubepkgImage, &Config{KubePkg: kpkg}) + return artifact.Artifact(kubepkgImage, &Config{KubePkg: kpkg}, artifact.WithAnnotations(map[string]string{ + specv1.AnnotationRefName: kpkg.Spec.Version, + })) } func (p *Packer) appendArtifactLayer(kubepkgImage v1.Image, src v1.Image, d v1.Descriptor, img *kubepkgv1alpha1.Image) (v1.Image, error) { @@ -296,9 +309,18 @@ func (p *Packer) appendManifests(idx v1.ImageIndex, source partial.Describable, if add.Annotations == nil { add.Annotations = map[string]string{} } - add.Annotations[specv1.AnnotationBaseImageName] = image.Name - add.Annotations[specv1.AnnotationRefName] = image.Tag - add.Annotations[images.AnnotationImageName] = image.FullName() + + if image.Name != "" { + add.Annotations[specv1.AnnotationBaseImageName] = image.Name + + if add.ArtifactType == "" { + add.Annotations[images.AnnotationImageName] = image.FullName() + } + } + + if image.Tag != "" { + add.Annotations[specv1.AnnotationRefName] = image.Tag + } } return mutate.AppendManifests(idx, add), nil diff --git a/pkg/kubepkg/packer_test.go b/pkg/kubepkg/packer_test.go index 004a92a..2b7a40c 100644 --- a/pkg/kubepkg/packer_test.go +++ b/pkg/kubepkg/packer_test.go @@ -1,34 +1,64 @@ package kubepkg import ( - "context" _ "embed" + + "context" "encoding/json" - "fmt" - "github.com/octohelm/crkit/pkg/kubepkg/cache" + "io" "os" "testing" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/octohelm/crkit/pkg/kubepkg/cache" "github.com/octohelm/crkit/pkg/ocitar" kubepkgv1alpha1 "github.com/octohelm/kubepkgspec/pkg/apis/kubepkg/v1alpha1" testingx "github.com/octohelm/x/testing" + "net/http/httptest" ) //go:embed testdata/example.kubepkg.json var kubepkgExample []byte -func TestPacker(t *testing.T) { - //t.Skip() +func Test(t *testing.T) { + kpkg := &kubepkgv1alpha1.KubePkg{} + _ = json.Unmarshal(kubepkgExample, kpkg) + + imageIndex := mutate.AppendManifests( + empty.Index, + mutate.IndexAddendum{ + Add: must(random.Image(10, 3)), + Descriptor: v1.Descriptor{ + Platform: &v1.Platform{ + OS: "linux", + Architecture: "amd64", + }, + }, + }, + mutate.IndexAddendum{ + Add: must(random.Image(10, 3)), + Descriptor: v1.Descriptor{ + Platform: &v1.Platform{ + OS: "linux", + Architecture: "arm64", + }, + }, + }, + ) + + s := httptest.NewServer(registry.New()) - registry, _ := name.NewRegistry(os.Getenv("CONTAINER_REGISTRY")) + r, err := NewRegistry(s.URL) + testingx.Expect(t, err, testingx.BeNil[error]()) - a := authn.FromConfig(authn.AuthConfig{ - Username: os.Getenv("CONTAINER_REGISTRY_USERNAME"), - Password: os.Getenv("CONTAINER_REGISTRY_PASSWORD"), - }) + err = remote.Put(r.Repo("docker.io/library/nginx").Tag("1.25.0-alpine"), imageIndex) + testingx.Expect(t, err, testingx.BeNil[error]()) // {{registry}}/{{namespace}}/{{name}} renamer, _ := NewTemplateRenamer("docker.io/x/{{ .name }}") @@ -38,45 +68,159 @@ func TestPacker(t *testing.T) { testingx.Expect(t, renamer.Rename(r), testingx.Be("docker.io/x/nginx")) }) - p := &Packer{ - Cache: cache.NewFilesystemCache("testdata/.tmp/cache"), - Registry: ®istry, - CreatePuller: func(name name.Repository, options ...remote.Option) (*remote.Puller, error) { - return remote.NewPuller(append(options, remote.WithAuth(a))...) - }, - Platforms: []string{ - "linux/amd64", - }, - Renamer: renamer, - } - - t.Run("should pack as kubepkg image", func(t *testing.T) { - kpkg := &kubepkgv1alpha1.KubePkg{} - _ = json.Unmarshal(kubepkgExample, kpkg) - - ctx := context.Background() - - i, err := p.PackAsKubePkgImage(ctx, kpkg) - testingx.Expect(t, err, testingx.BeNil[error]()) - - raw, _ := i.RawManifest() - fmt.Println(string(raw)) + c := cache.NewFilesystemCache("testdata/.tmp/cache") + + t.Run("with single arch", func(t *testing.T) { + p := &Packer{ + Cache: c, + CreatePuller: func(ref name.Reference, options ...remote.Option) (*remote.Puller, error) { + return remote.NewPuller(options...) + }, + Registry: r, + Platforms: []string{ + "linux/amd64", + }, + Renamer: renamer, + } + + t.Run("should pack as kubepkg image", func(t *testing.T) { + kpkg := &kubepkgv1alpha1.KubePkg{} + _ = json.Unmarshal(kubepkgExample, kpkg) + + ctx := context.Background() + + i, err := p.PackAsKubePkgImage(ctx, kpkg) + testingx.Expect(t, err, testingx.BeNil[error]()) + + layers, err := i.Layers() + testingx.Expect(t, err, testingx.BeNil[error]()) + testingx.Expect(t, len(layers), testingx.Be(1)) + }) + + t.Run("should pack as index", func(t *testing.T) { + ctx := context.Background() + + idx, err := p.PackAsIndex(ctx, kpkg) + testingx.Expect(t, err, testingx.BeNil[error]()) + + filename := "testdata/.tmp/example.kubepkg.amd64.tar" + + err = writeAsOciTar(filename, idx) + testingx.Expect(t, err, testingx.BeNil[error]()) + + t.Run("should read", func(t *testing.T) { + idx, err := ocitar.Index(func() (io.ReadCloser, error) { + return os.OpenFile(filename, os.O_RDONLY, os.ModePerm) + }) + testingx.Expect(t, err, testingx.BeNil[error]()) + + found, err := KubePkg(idx) + testingx.Expect(t, err, testingx.BeNil[error]()) + testingx.Expect(t, found.Spec.Version, testingx.Be(kpkg.Spec.Version)) + + t.Run("then could push", func(t *testing.T) { + pusher := &Pusher{ + Registry: r, + Renamer: renamer, + CreatePusher: func(ref name.Reference, options ...remote.Option) (*remote.Pusher, error) { + return remote.NewPusher(options...) + }, + } + err = pusher.PushIndex(ctx, idx) + testingx.Expect(t, err, testingx.BeNil[error]()) + }) + }) + }) }) - t.Run("should pack as index", func(t *testing.T) { - kpkg := &kubepkgv1alpha1.KubePkg{} - _ = json.Unmarshal(kubepkgExample, kpkg) - - ctx := context.Background() - - idx, err := p.PackAsIndex(ctx, kpkg) - testingx.Expect(t, err, testingx.BeNil[error]()) + t.Run("with multi arch", func(t *testing.T) { + p := &Packer{ + Cache: c, + Registry: r, + CreatePuller: func(ref name.Reference, options ...remote.Option) (*remote.Puller, error) { + return remote.NewPuller(append(options)...) + }, + Platforms: []string{ + "linux/amd64", + "linux/arm64", + }, + Renamer: renamer, + } + + t.Run("should pack as kubepkg image", func(t *testing.T) { + kpkg := &kubepkgv1alpha1.KubePkg{} + _ = json.Unmarshal(kubepkgExample, kpkg) + + ctx := context.Background() + + i, err := p.PackAsKubePkgImage(ctx, kpkg) + testingx.Expect(t, err, testingx.BeNil[error]()) + + layers, err := i.Layers() + testingx.Expect(t, err, testingx.BeNil[error]()) + testingx.Expect(t, len(layers), testingx.Be(2)) + }) + + t.Run("should pack as index", func(t *testing.T) { + ctx := context.Background() + + idx, err := p.PackAsIndex(ctx, kpkg) + testingx.Expect(t, err, testingx.BeNil[error]()) + + err = writeAsOciTar("testdata/.tmp/example.kubepkg.tar", idx) + testingx.Expect(t, err, testingx.BeNil[error]()) + }) + + t.Run("should pack as index", func(t *testing.T) { + ctx := context.Background() + + idx, err := p.PackAsIndex(ctx, kpkg) + testingx.Expect(t, err, testingx.BeNil[error]()) + + filename := "testdata/.tmp/example.kubepkg.amd64.tar" + + err = writeAsOciTar(filename, idx) + testingx.Expect(t, err, testingx.BeNil[error]()) + + t.Run("should read", func(t *testing.T) { + idx, err := ocitar.Index(func() (io.ReadCloser, error) { + return os.OpenFile(filename, os.O_RDONLY, os.ModePerm) + }) + testingx.Expect(t, err, testingx.BeNil[error]()) + + found, err := KubePkg(idx) + testingx.Expect(t, err, testingx.BeNil[error]()) + testingx.Expect(t, found.Spec.Version, testingx.Be(kpkg.Spec.Version)) + + t.Run("then could push", func(t *testing.T) { + pusher := &Pusher{ + Registry: r, + Renamer: renamer, + CreatePusher: func(ref name.Reference, options ...remote.Option) (*remote.Pusher, error) { + return remote.NewPusher(options...) + }, + } + + err = pusher.PushIndex(ctx, idx) + testingx.Expect(t, err, testingx.BeNil[error]()) + }) + }) + }) + }) +} - f, err := os.OpenFile("testdata/.tmp/example.kubepkg.tar", os.O_TRUNC|os.O_WRONLY|os.O_CREATE, os.ModePerm) - testingx.Expect(t, err, testingx.BeNil[error]()) - defer f.Close() +func writeAsOciTar(filename string, idx v1.ImageIndex) error { + f, err := os.OpenFile(filename, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return err + } + defer f.Close() + return ocitar.Write(f, idx) +} - err = ocitar.Write(f, idx) - testingx.Expect(t, err, testingx.BeNil[error]()) - }) +func must[T any](x T, err error) T { + if err != nil { + panic(err) + } + return x } diff --git a/pkg/kubepkg/puller.go b/pkg/kubepkg/puller.go deleted file mode 100644 index bb6a037..0000000 --- a/pkg/kubepkg/puller.go +++ /dev/null @@ -1,3 +0,0 @@ -package kubepkg - -type Puller interface{} diff --git a/pkg/kubepkg/pusher.go b/pkg/kubepkg/pusher.go new file mode 100644 index 0000000..c0865a4 --- /dev/null +++ b/pkg/kubepkg/pusher.go @@ -0,0 +1,52 @@ +package kubepkg + +import ( + "context" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +type Pusher struct { + Registry Registry + Renamer Renamer + CreatePusher func(ref name.Reference, options ...remote.Option) (*remote.Pusher, error) +} + +func (p *Pusher) PushIndex(ctx context.Context, idx v1.ImageIndex) error { + i := NewImageIter(idx) + for ref, img := range i.Images() { + ref, err := p.normalize(ref) + if err != nil { + return err + } + + pusher, err := p.CreatePusher(ref) + if err != nil { + return err + } + if err := pusher.Push(ctx, ref, img); err != nil { + return err + } + } + return i.Err() +} + +func (p *Pusher) normalize(ref name.Reference) (name.Reference, error) { + repoName := ref.Context().String() + tag := ref.Identifier() + + if renamer := p.Renamer; renamer != nil { + repoName = renamer.Rename(ref.Context()) + } + + if registry := p.Registry; registry != nil { + repoName = registry.Repo(repoName).String() + } + + repo, err := name.NewRepository(repoName) + if err != nil { + return nil, err + } + return repo.Tag(tag), nil +} diff --git a/pkg/kubepkg/registry.go b/pkg/kubepkg/registry.go new file mode 100644 index 0000000..07ff4b6 --- /dev/null +++ b/pkg/kubepkg/registry.go @@ -0,0 +1,44 @@ +package kubepkg + +import ( + "net/url" + "strings" + + "github.com/google/go-containerregistry/pkg/name" +) + +type Registry interface { + Repo(repoName string) name.Repository +} + +func NewRegistry(baseURL string) (Registry, error) { + u, err := url.Parse(baseURL) + if err != nil { + return nil, err + } + + ops := make([]name.Option, 0) + + if u.Scheme == "http:" { + ops = append(ops, name.Insecure) + } + + r, err := name.NewRegistry(u.Host, ops...) + if err != nil { + return nil, err + } + + return &pathNormalizedRegistry{registry: &r}, nil +} + +type pathNormalizedRegistry struct { + registry *name.Registry +} + +func (p *pathNormalizedRegistry) Repo(repoName string) name.Repository { + registryName := p.registry.Name() + if strings.HasPrefix(repoName, registryName) { + return p.registry.Repo(repoName[len(registryName)+1:]) + } + return p.registry.Repo(repoName) +} diff --git a/pkg/kubepkg/testdata/example.kubepkg.json b/pkg/kubepkg/testdata/example.kubepkg.json index 624ab27..b8b141c 100644 --- a/pkg/kubepkg/testdata/example.kubepkg.json +++ b/pkg/kubepkg/testdata/example.kubepkg.json @@ -9,7 +9,7 @@ } }, "spec": { - "version": "0.0.2", + "version": "v0.0.2", "config": { "X": "x" }, diff --git a/pkg/ocitar/error.go b/pkg/ocitar/error.go new file mode 100644 index 0000000..fece0bf --- /dev/null +++ b/pkg/ocitar/error.go @@ -0,0 +1,15 @@ +package ocitar + +import ( + "fmt" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +type ErrNotFound struct { + Digest v1.Hash +} + +func (e *ErrNotFound) Error() string { + return fmt.Sprintf("%s not found", e.Digest) +} diff --git a/pkg/ocitar/index.go b/pkg/ocitar/index.go new file mode 100644 index 0000000..349d705 --- /dev/null +++ b/pkg/ocitar/index.go @@ -0,0 +1,218 @@ +package ocitar + +import ( + "encoding/json" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/opencontainers/go-digest" + specv1 "github.com/opencontainers/image-spec/specs-go/v1" + "io" + "path/filepath" +) + +func Index(opener Opener) (v1.ImageIndex, error) { + tr := &tarReader{opener: opener} + + r, err := tr.Open("index.json") + if err != nil { + return nil, err + } + return openAsIndexReader(tr, r) +} + +type index struct { + o FileOpener + d v1.Descriptor + indexManifest *v1.IndexManifest + raw []byte +} + +type readCloser struct { + io.Reader + close func() error +} + +func (r *readCloser) Close() error { + return r.close() +} + +func (i *index) MediaType() (types.MediaType, error) { + return types.OCIImageIndex, nil +} + +func (i *index) Digest() (v1.Hash, error) { + return i.d.Digest, nil +} + +func (i *index) Size() (int64, error) { + return i.d.Size, nil +} + +func (i *index) IndexManifest() (*v1.IndexManifest, error) { + return i.indexManifest, nil +} + +func (i *index) RawManifest() ([]byte, error) { + return i.raw, nil +} + +func (i *index) Image(h v1.Hash) (v1.Image, error) { + for _, d := range i.indexManifest.Manifests { + if d.MediaType.IsImage() && d.Digest == h { + f, err := i.o.Open(layoutBlobsPath(h)) + if err != nil { + return nil, err + } + return openAsImageReader(i.o, f) + } + } + return nil, &ErrNotFound{Digest: h} +} + +func (i *index) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { + for _, d := range i.indexManifest.Manifests { + if d.MediaType.IsIndex() && d.Digest == h { + f, err := i.o.Open(layoutBlobsPath(h)) + if err != nil { + return nil, err + } + return openAsIndexReader(i.o, f) + } + } + return nil, &ErrNotFound{ + Digest: h, + } +} + +func openAsIndexReader(o FileOpener, r io.ReadCloser) (v1.ImageIndex, error) { + defer r.Close() + + i := &index{o: o} + + digester := digest.SHA256.Digester() + data, err := io.ReadAll(io.TeeReader(r, digester.Hash())) + if err != nil { + return nil, err + } + + dgst := digester.Digest() + + i.raw = data + i.d.Size = int64(len(data)) + i.d.Digest = v1.Hash{ + Algorithm: string(dgst.Algorithm()), + Hex: dgst.Hex(), + } + + indexManifest := &v1.IndexManifest{} + if err := json.Unmarshal(i.raw, indexManifest); err != nil { + return nil, err + } + i.indexManifest = indexManifest + + return i, nil +} + +func openAsImageReader(o FileOpener, r io.ReadCloser) (v1.Image, error) { + defer r.Close() + + raw, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + m := &specv1.Manifest{} + if err := json.Unmarshal(raw, m); err != nil { + return nil, err + } + + configName := v1.Hash{ + Algorithm: string(m.Config.Digest.Algorithm()), + Hex: m.Config.Digest.Hex(), + } + + configReader, err := o.Open(layoutBlobsPath(configName)) + if err != nil { + return nil, err + } + defer configReader.Close() + + configRaw, err := io.ReadAll(configReader) + if err != nil { + return nil, err + } + + return partial.CompressedToImage(&image{ + o: o, + m: m, + raw: raw, + configRaw: configRaw, + }) +} + +type image struct { + o FileOpener + m *specv1.Manifest + raw []byte + configRaw []byte +} + +func (i *image) MediaType() (types.MediaType, error) { + return types.MediaType(i.m.MediaType), nil +} + +func (i *image) RawConfigFile() ([]byte, error) { + return i.configRaw, nil +} + +func (i *image) RawManifest() ([]byte, error) { + return i.raw, nil +} + +func (i *image) LayerByDigest(hash v1.Hash) (partial.CompressedLayer, error) { + for _, l := range i.m.Layers { + if l.Digest.String() == hash.String() { + return &layer{ + d: v1.Descriptor{ + MediaType: types.MediaType(l.MediaType), + ArtifactType: l.ArtifactType, + Digest: hash, + Size: l.Size, + Annotations: l.Annotations, + }, + opener: func() (io.ReadCloser, error) { + return i.o.Open(layoutBlobsPath(hash)) + }, + }, nil + } + } + return nil, &ErrNotFound{ + Digest: hash, + } +} + +type layer struct { + d v1.Descriptor + opener Opener +} + +func (l *layer) MediaType() (types.MediaType, error) { + return l.d.MediaType, nil +} + +func (l *layer) Size() (int64, error) { + return l.d.Size, nil +} + +func (l *layer) Digest() (v1.Hash, error) { + return l.d.Digest, nil +} + +func (l *layer) Compressed() (io.ReadCloser, error) { + return l.opener() +} + +func layoutBlobsPath(hash v1.Hash) string { + return filepath.Join("blobs", hash.Algorithm, hash.Hex) +} diff --git a/pkg/ocitar/progess.go b/pkg/ocitar/progess.go deleted file mode 100644 index fffb449..0000000 --- a/pkg/ocitar/progess.go +++ /dev/null @@ -1,54 +0,0 @@ -package ocitar - -import ( - "io" - "sync/atomic" - - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" -) - -type Update struct { - Repository name.Repository - Digest v1.Hash - Total int64 - Complete int64 -} - -func (u *Update) String() string { - if repoName := u.Repository.String(); repoName != "" { - return repoName + "@" + u.Digest.String() - } - return u.Digest.String() -} - -type progress struct { - updates chan<- Update -} - -func (p *progress) complete(u Update) { - p.updates <- u -} - -type progressReader struct { - r io.Reader - digest v1.Hash - repository name.Repository - total int64 - count *int64 // number of bytes this reader has read, to support resetting on retry. - progress *progress -} - -func (r *progressReader) Read(b []byte) (int, error) { - n, err := r.r.Read(b) - if err != nil { - return n, err - } - r.progress.complete(Update{ - Repository: r.repository, - Digest: r.digest, - Total: r.total, - Complete: atomic.AddInt64(r.count, int64(n)), - }) - return n, nil -} diff --git a/pkg/ocitar/tar_reader.go b/pkg/ocitar/tar_reader.go new file mode 100644 index 0000000..910e93b --- /dev/null +++ b/pkg/ocitar/tar_reader.go @@ -0,0 +1,46 @@ +package ocitar + +import ( + "archive/tar" + "io" + "os" +) + +type Opener func() (io.ReadCloser, error) + +type FileOpener interface { + Open(filename string) (io.ReadCloser, error) +} + +type tarReader struct { + opener Opener +} + +func (i *tarReader) Open(filename string) (io.ReadCloser, error) { + f, err := i.opener() + if err != nil { + return nil, err + } + + tr := tar.NewReader(f) + + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hdr.Name == filename { + return &readCloser{ + Reader: tr, + close: f.Close, + }, nil + } + } + + _ = f.Close() + return nil, os.ErrNotExist +} diff --git a/pkg/ocitar/write.go b/pkg/ocitar/write.go index a1136ea..1d3b2a1 100644 --- a/pkg/ocitar/write.go +++ b/pkg/ocitar/write.go @@ -3,32 +3,27 @@ package ocitar import ( "archive/tar" "bytes" + "encoding/json" "fmt" - "github.com/google/go-containerregistry/pkg/name" "io" "path/filepath" - "strings" "sync" - "github.com/octohelm/x/ptr" - - "github.com/pkg/errors" - "github.com/containerd/containerd/images" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/pkg/errors" ) var ociLayoutRaw = []byte(`{"imageLayoutVersion":"1.0.0"}`) -func Write(w io.Writer, idx v1.ImageIndex, options ...Option) error { +func Write(w io.Writer, idx v1.ImageIndex) error { tw := tar.NewWriter(w) defer func() { _ = tw.Close() }() ww := &ociTarWriter{Writer: tw} - ww.Build(options...) return ww.writeRootIndex(idx) } @@ -37,22 +32,6 @@ type ociTarWriter struct { *tar.Writer blobs sync.Map - - progress *progress -} - -func (w *ociTarWriter) Build(options ...Option) { - for _, o := range options { - o(w) - } -} - -type Option = func(*ociTarWriter) - -func WithProgress(updates chan<- Update) Option { - return func(w *ociTarWriter) { - w.progress = &progress{updates: updates} - } } func (w *ociTarWriter) writeRootIndex(idx v1.ImageIndex) error { @@ -65,10 +44,15 @@ func (w *ociTarWriter) writeRootIndex(idx v1.ImageIndex) error { return err } + b := &bytes.Buffer{} + if err := json.Indent(b, raw, "", " "); err != nil { + return err + } + if err := w.writeToTar(tar.Header{ Name: "index.json", - Size: int64(len(raw)), - }, bytes.NewBuffer(raw)); err != nil { + Size: int64(b.Len()), + }, b); err != nil { return err } @@ -98,24 +82,6 @@ func (w *ociTarWriter) writeToTarWithDigest(dgst v1.Hash, size int64, r io.Reade w.blobs.Store(dgst, true) }() - if w.progress != nil { - pr := &progressReader{ - r: r, - digest: dgst, - total: size, - count: ptr.Ptr(int64(0)), - progress: w.progress, - } - - if scope != nil && scope.Annotations != nil { - if imageName, ok := scope.Annotations[images.AnnotationImageName]; ok { - pr.repository, _ = name.NewRepository(strings.Split(imageName, ":")[0]) - } - } - - r = pr - } - return w.writeToTar(tar.Header{ Name: filepath.Join("blobs", dgst.Algorithm, dgst.Hex), Size: size, diff --git a/pkg/ocitar/write_test.go b/pkg/ocitar/write_test.go index b127ec3..82829ec 100644 --- a/pkg/ocitar/write_test.go +++ b/pkg/ocitar/write_test.go @@ -1,10 +1,13 @@ package ocitar import ( + "io" "os" "path/filepath" "testing" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/random" testingx "github.com/octohelm/x/testing" ) @@ -18,14 +21,33 @@ func TestOciTar(t *testing.T) { index, err := random.Index(10, 5, 2) testingx.Expect(t, err, testingx.BeNil[error]()) - t.Run("should write", func(t *testing.T) { - filename := filepath.Join(d, "x.tar") + expectImages, err := partial.FindImages(index, func(desc v1.Descriptor) bool { + return true + }) + testingx.Expect(t, err, testingx.BeNil[error]()) + + filename := filepath.Join(d, "x.tar") - f, err := os.OpenFile(filename, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, os.ModePerm) + t.Run("should write", func(t *testing.T) { + f, err := os.OpenFile(filename, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0600) testingx.Expect(t, err, testingx.BeNil[error]()) - defer f.Close() err = Write(f, index) testingx.Expect(t, err, testingx.BeNil[error]()) + _ = f.Close() + + t.Run("then should read", func(t *testing.T) { + idx, err := Index(func() (io.ReadCloser, error) { + return os.OpenFile(filename, os.O_RDONLY, os.ModePerm) + }) + testingx.Expect(t, err, testingx.BeNil[error]()) + + images, err := partial.FindImages(idx, func(desc v1.Descriptor) bool { + return true + }) + testingx.Expect(t, err, testingx.BeNil[error]()) + testingx.Expect(t, len(images), testingx.Be(len(expectImages))) + }) }) + } diff --git a/pkg/ociutil/discriptor.go b/pkg/ociutil/discriptor.go new file mode 100644 index 0000000..f4c1f2d --- /dev/null +++ b/pkg/ociutil/discriptor.go @@ -0,0 +1 @@ +package ociutil diff --git a/pkg/ociutil/raw.go b/pkg/ociutil/raw.go new file mode 100644 index 0000000..867ffb7 --- /dev/null +++ b/pkg/ociutil/raw.go @@ -0,0 +1,45 @@ +package ociutil + +import ( + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" +) + +func FromRaw(img v1.Image) v1.Image { + return &rawImage{Image: img} +} + +type rawImage struct { + v1.Image +} + +type WithArtifactType interface { + ArtifactType() (string, error) +} + +func (i *rawImage) ArtifactType() (string, error) { + if withAt, ok := i.Image.(WithArtifactType); ok { + return withAt.ArtifactType() + } + return "", nil +} + +func (i *rawImage) ConfigName() (v1.Hash, error) { + return partial.ConfigName(i) +} + +func (i *rawImage) ConfigFile() (*v1.ConfigFile, error) { + return partial.ConfigFile(i) +} + +func (i *rawImage) Manifest() (*v1.Manifest, error) { + return partial.Manifest(i) +} + +func (i *rawImage) Size() (int64, error) { + return partial.Size(i) +} + +func (i *rawImage) Digest() (v1.Hash, error) { + return partial.Digest(i) +}