diff --git a/Dockerfile b/Dockerfile index d588e00..ba63393 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -FROM alpine:3.8 +FROM alpine:3.20.0 RUN apk --no-cache add ca-certificates -COPY skbn /usr/local/bin/skbn +COPY bin/skbn /usr/local/bin/skbn RUN addgroup -g 1001 -S skbn \ && adduser -u 1001 -D -S -G skbn skbn USER skbn diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index f4b19fe..0000000 --- a/Gopkg.lock +++ /dev/null @@ -1,531 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - digest = "1:d2ccb697dc13c8fbffafa37baae97594d5592ae8f7e113471084137315536e2b" - name = "github.com/Azure/azure-pipeline-go" - packages = ["pipeline"] - pruneopts = "UT" - revision = "b8e3409182fd52e74f7d7bdfbff5833591b3b655" - version = "v0.1.8" - -[[projects]] - digest = "1:c4a5edf3b0f38e709a78dcc945997678a364c2b5adfd48842a3dd349c352f833" - name = "github.com/Azure/azure-storage-blob-go" - packages = ["azblob"] - pruneopts = "UT" - revision = "5152f14ace1c6db66bd9cb57840703a8358fa7bc" - version = "0.3.0" - -[[projects]] - digest = "1:49f9d62ba765b7d638dc8abd4b68252dc511a1b8fe44a6697e94660bde541d2d" - name = "github.com/aws/aws-sdk-go" - packages = [ - "aws", - "aws/awserr", - "aws/awsutil", - "aws/client", - "aws/client/metadata", - "aws/corehandlers", - "aws/credentials", - "aws/credentials/ec2rolecreds", - "aws/credentials/endpointcreds", - "aws/credentials/processcreds", - "aws/credentials/stscreds", - "aws/csm", - "aws/defaults", - "aws/ec2metadata", - "aws/endpoints", - "aws/request", - "aws/session", - "aws/signer/v4", - "internal/ini", - "internal/s3err", - "internal/sdkio", - "internal/sdkrand", - "internal/sdkuri", - "internal/shareddefaults", - "private/protocol", - "private/protocol/eventstream", - "private/protocol/eventstream/eventstreamapi", - "private/protocol/query", - "private/protocol/query/queryutil", - "private/protocol/rest", - "private/protocol/restxml", - "private/protocol/xml/xmlutil", - "service/s3", - "service/s3/s3iface", - "service/s3/s3manager", - "service/sts", - ] - pruneopts = "UT" - revision = "a21788f1964e28dc1e3932e595db6707ce8070cf" - version = "v1.16.7" - -[[projects]] - digest = "1:49ea765e340f3907fc52ab1cc23bf9698d2d4fa9db14673710879d2ae2d22a9a" - name = "github.com/djherbis/buffer" - packages = [ - ".", - "limio", - "wrapio", - ] - pruneopts = "UT" - revision = "4972e2bf4a27dbd7c43140c959b85e8c1d2f5536" - version = "v1.0" - -[[projects]] - digest = "1:58be7025fd84632dfbb8a398f931b5bdbbecc0390e4385df4ae56775487a0f87" - name = "github.com/docker/spdystream" - packages = [ - ".", - "spdy", - ] - pruneopts = "UT" - revision = "449fdfce4d962303d702fec724ef0ad181c92528" - -[[projects]] - digest = "1:b7a8552c62868d867795b63eaf4f45d3e92d36db82b428e680b9c95a8c33e5b1" - name = "github.com/gogo/protobuf" - packages = [ - "proto", - "sortkeys", - ] - pruneopts = "UT" - revision = "342cbe0a04158f6dcb03ca0079991a51a4248c02" - version = "v0.5" - -[[projects]] - digest = "1:17fe264ee908afc795734e8c4e63db2accabaf57326dbf21763a7d6b86096260" - name = "github.com/golang/protobuf" - packages = [ - "proto", - "ptypes", - "ptypes/any", - "ptypes/duration", - "ptypes/timestamp", - ] - pruneopts = "UT" - revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" - version = "v1.1.0" - -[[projects]] - branch = "master" - digest = "1:0bfbe13936953a98ae3cfe8ed6670d396ad81edf069a806d2f6515d7bb6950df" - name = "github.com/google/btree" - packages = ["."] - pruneopts = "UT" - revision = "4030bb1f1f0c35b30ca7009e9ebd06849dd45306" - -[[projects]] - digest = "1:41bfd4219241b7f7d6e6fdb13fc712576f1337e68e6b895136283b76928fdd66" - name = "github.com/google/gofuzz" - packages = ["."] - pruneopts = "UT" - revision = "44d81051d367757e1c7c6a5a86423ece9afcf63c" - -[[projects]] - digest = "1:75eb87381d25cc75212f52358df9c3a2719584eaa9685cd510ce28699122f39d" - name = "github.com/googleapis/gnostic" - packages = [ - "OpenAPIv2", - "compiler", - "extensions", - ] - pruneopts = "UT" - revision = "0c5108395e2debce0d731cf0287ddf7242066aba" - -[[projects]] - digest = "1:878f0defa9b853f9acfaf4a162ba450a89d0050eff084f9fe7f5bd15948f172a" - name = "github.com/gregjones/httpcache" - packages = [ - ".", - "diskcache", - ] - pruneopts = "UT" - revision = "787624de3eb7bd915c329cba748687a3b22666a6" - -[[projects]] - digest = "1:3e260afa138eab6492b531a3b3d10ab4cb70512d423faa78b8949dec76e66a21" - name = "github.com/imdario/mergo" - packages = ["."] - pruneopts = "UT" - revision = "9316a62528ac99aaecb4e47eadd6dc8aa6533d58" - version = "v0.3.5" - -[[projects]] - digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be" - name = "github.com/inconshreveable/mousetrap" - packages = ["."] - pruneopts = "UT" - revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" - version = "v1.0" - -[[projects]] - digest = "1:bb81097a5b62634f3e9fec1014657855610c82d19b9a40c17612e32651e35dca" - name = "github.com/jmespath/go-jmespath" - packages = ["."] - pruneopts = "UT" - revision = "c2b33e84" - -[[projects]] - digest = "1:eaefc85d32c03e5f0c2b88ea2f79fce3d993e2c78316d21319575dd4ea9153ca" - name = "github.com/json-iterator/go" - packages = ["."] - pruneopts = "UT" - revision = "ab8a2e0c74be9d3be70b3184d9acc634935ded82" - version = "1.1.4" - -[[projects]] - digest = "1:33422d238f147d247752996a26574ac48dcf472976eda7f5134015f06bf16563" - name = "github.com/modern-go/concurrent" - packages = ["."] - pruneopts = "UT" - revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94" - version = "1.0.3" - -[[projects]] - digest = "1:c56ad36f5722eb07926c979d5e80676ee007a9e39e7808577b9d87ec92b00460" - name = "github.com/modern-go/reflect2" - packages = ["."] - pruneopts = "UT" - revision = "94122c33edd36123c84d5368cfb2b69df93a0ec8" - version = "v1.0.1" - -[[projects]] - branch = "master" - digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2" - name = "github.com/petar/GoLLRB" - packages = ["llrb"] - pruneopts = "UT" - revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4" - -[[projects]] - digest = "1:0e7775ebbcf00d8dd28ac663614af924411c868dca3d5aa762af0fae3808d852" - name = "github.com/peterbourgon/diskv" - packages = ["."] - pruneopts = "UT" - revision = "5f041e8faa004a95c88a202771f4cc3e991971e6" - version = "v2.0.1" - -[[projects]] - digest = "1:645cabccbb4fa8aab25a956cbcbdf6a6845ca736b2c64e197ca7cbb9d210b939" - name = "github.com/spf13/cobra" - packages = ["."] - pruneopts = "UT" - revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385" - version = "v0.0.3" - -[[projects]] - digest = "1:c1b1102241e7f645bc8e0c22ae352e8f0dc6484b6cb4d132fa9f24174e0119e2" - name = "github.com/spf13/pflag" - packages = ["."] - pruneopts = "UT" - revision = "298182f68c66c05229eb03ac171abe6e309ee79a" - version = "v1.0.3" - -[[projects]] - digest = "1:3f3a05ae0b95893d90b9b3b5afdb79a9b3d96e4e36e099d841ae602e4aca0da8" - name = "golang.org/x/crypto" - packages = ["ssh/terminal"] - pruneopts = "UT" - revision = "de0752318171da717af4ce24d0a2e8626afaeb11" - -[[projects]] - branch = "release-branch.go1.10" - digest = "1:bdc99a0ff03b87c7fe65dce8cfd5f7db86d65446850b2c655db7bc740f16aded" - name = "golang.org/x/net" - packages = [ - "context", - "http2", - "http2/hpack", - "idna", - "lex/httplex", - ] - pruneopts = "UT" - revision = "0ed95abb35c445290478a5348a7b38bb154135fd" - -[[projects]] - digest = "1:9359217acc6040b4be710ce34473acef28023ad39bfafecea34ffaea7f1e1890" - name = "golang.org/x/oauth2" - packages = [ - ".", - "internal", - ] - pruneopts = "UT" - revision = "a6bd8cefa1811bd24b86f8902872e4e8225f74c4" - -[[projects]] - branch = "master" - digest = "1:ba8cbf57cfd92d5f8592b4aca1a35d92c162363d32aeabd5b12555f8896635e7" - name = "golang.org/x/sys" - packages = [ - "unix", - "windows", - ] - pruneopts = "UT" - revision = "dcdaa6325bcb8d3c07805583fe12153f08a41e2b" - -[[projects]] - digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18" - name = "golang.org/x/text" - packages = [ - "collate", - "collate/build", - "internal/colltab", - "internal/gen", - "internal/tag", - "internal/triegen", - "internal/ucd", - "language", - "secure/bidirule", - "transform", - "unicode/bidi", - "unicode/cldr", - "unicode/norm", - "unicode/rangetable", - ] - pruneopts = "UT" - revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" - version = "v0.3.0" - -[[projects]] - digest = "1:d37b0ef2944431fe9e8ef35c6fffc8990d9e2ca300588df94a6890f3649ae365" - name = "golang.org/x/time" - packages = ["rate"] - pruneopts = "UT" - revision = "f51c12702a4d776e4c1fa9b0fabab841babae631" - -[[projects]] - digest = "1:08206298775e5b462e6c0333f4471b44e63f1a70e42952b6ede4ecc9572281eb" - name = "google.golang.org/appengine" - packages = [ - "internal", - "internal/base", - "internal/datastore", - "internal/log", - "internal/remote_api", - "internal/urlfetch", - "urlfetch", - ] - pruneopts = "UT" - revision = "4a4468ece617fc8205e99368fa2200e9d1fad421" - version = "v1.3.0" - -[[projects]] - digest = "1:5aebed51f26a49cb77a0c34d1328a0d29839e7a1204b14ff8e26a3df8ec61736" - name = "gopkg.in/djherbis/nio.v2" - packages = ["."] - pruneopts = "UT" - revision = "824ca9017eeb2a422fdda7ae9dba05d3ab019dfa" - version = "v2.0.3" - -[[projects]] - digest = "1:ef72505cf098abdd34efeea032103377bec06abb61d8a06f002d5d296a4b1185" - name = "gopkg.in/inf.v0" - packages = ["."] - pruneopts = "UT" - revision = "3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4" - version = "v0.9.0" - -[[projects]] - digest = "1:4d2e5a73dc1500038e504a8d78b986630e3626dc027bc030ba5c75da257cdb96" - name = "gopkg.in/yaml.v2" - packages = ["."] - pruneopts = "UT" - revision = "51d6538a90f86fe93ac480b35f37b2be17fef232" - version = "v2.2.2" - -[[projects]] - digest = "1:0d299a04c6472e4458461d7034c76d014cc6f632a3262cbf21d123b19ce13e65" - name = "k8s.io/api" - packages = [ - "admissionregistration/v1alpha1", - "admissionregistration/v1beta1", - "apps/v1", - "apps/v1beta1", - "apps/v1beta2", - "auditregistration/v1alpha1", - "authentication/v1", - "authentication/v1beta1", - "authorization/v1", - "authorization/v1beta1", - "autoscaling/v1", - "autoscaling/v2beta1", - "autoscaling/v2beta2", - "batch/v1", - "batch/v1beta1", - "batch/v2alpha1", - "certificates/v1beta1", - "coordination/v1beta1", - "core/v1", - "events/v1beta1", - "extensions/v1beta1", - "networking/v1", - "policy/v1beta1", - "rbac/v1", - "rbac/v1alpha1", - "rbac/v1beta1", - "scheduling/v1alpha1", - "scheduling/v1beta1", - "settings/v1alpha1", - "storage/v1", - "storage/v1alpha1", - "storage/v1beta1", - ] - pruneopts = "UT" - revision = "89a74a8d264df0e993299876a8cde88379b940ee" - version = "kubernetes-1.13.0" - -[[projects]] - branch = "release-1.13" - digest = "1:d0727b638acf14f150c5e2bdf70413af1099b7a76d69f4f85ad3049b615d24a3" - name = "k8s.io/apimachinery" - packages = [ - "pkg/api/errors", - "pkg/api/meta", - "pkg/api/resource", - "pkg/apis/meta/v1", - "pkg/apis/meta/v1/unstructured", - "pkg/apis/meta/v1beta1", - "pkg/conversion", - "pkg/conversion/queryparams", - "pkg/fields", - "pkg/labels", - "pkg/runtime", - "pkg/runtime/schema", - "pkg/runtime/serializer", - "pkg/runtime/serializer/json", - "pkg/runtime/serializer/protobuf", - "pkg/runtime/serializer/recognizer", - "pkg/runtime/serializer/streaming", - "pkg/runtime/serializer/versioning", - "pkg/selection", - "pkg/types", - "pkg/util/clock", - "pkg/util/errors", - "pkg/util/framer", - "pkg/util/httpstream", - "pkg/util/httpstream/spdy", - "pkg/util/intstr", - "pkg/util/json", - "pkg/util/naming", - "pkg/util/net", - "pkg/util/remotecommand", - "pkg/util/runtime", - "pkg/util/sets", - "pkg/util/validation", - "pkg/util/validation/field", - "pkg/util/yaml", - "pkg/version", - "pkg/watch", - "third_party/forked/golang/netutil", - "third_party/forked/golang/reflect", - ] - pruneopts = "UT" - revision = "2b1284ed4c93a43499e781493253e2ac5959c4fd" - -[[projects]] - digest = "1:92c7b523fffa5badf467754612745bfa8629c4999a44c0c2521c16e2ad50bd2a" - name = "k8s.io/client-go" - packages = [ - "discovery", - "kubernetes", - "kubernetes/scheme", - "kubernetes/typed/admissionregistration/v1alpha1", - "kubernetes/typed/admissionregistration/v1beta1", - "kubernetes/typed/apps/v1", - "kubernetes/typed/apps/v1beta1", - "kubernetes/typed/apps/v1beta2", - "kubernetes/typed/auditregistration/v1alpha1", - "kubernetes/typed/authentication/v1", - "kubernetes/typed/authentication/v1beta1", - "kubernetes/typed/authorization/v1", - "kubernetes/typed/authorization/v1beta1", - "kubernetes/typed/autoscaling/v1", - "kubernetes/typed/autoscaling/v2beta1", - "kubernetes/typed/autoscaling/v2beta2", - "kubernetes/typed/batch/v1", - "kubernetes/typed/batch/v1beta1", - "kubernetes/typed/batch/v2alpha1", - "kubernetes/typed/certificates/v1beta1", - "kubernetes/typed/coordination/v1beta1", - "kubernetes/typed/core/v1", - "kubernetes/typed/events/v1beta1", - "kubernetes/typed/extensions/v1beta1", - "kubernetes/typed/networking/v1", - "kubernetes/typed/policy/v1beta1", - "kubernetes/typed/rbac/v1", - "kubernetes/typed/rbac/v1alpha1", - "kubernetes/typed/rbac/v1beta1", - "kubernetes/typed/scheduling/v1alpha1", - "kubernetes/typed/scheduling/v1beta1", - "kubernetes/typed/settings/v1alpha1", - "kubernetes/typed/storage/v1", - "kubernetes/typed/storage/v1alpha1", - "kubernetes/typed/storage/v1beta1", - "pkg/apis/clientauthentication", - "pkg/apis/clientauthentication/v1alpha1", - "pkg/apis/clientauthentication/v1beta1", - "pkg/version", - "plugin/pkg/client/auth/exec", - "rest", - "rest/watch", - "tools/auth", - "tools/clientcmd", - "tools/clientcmd/api", - "tools/clientcmd/api/latest", - "tools/clientcmd/api/v1", - "tools/metrics", - "tools/reference", - "tools/remotecommand", - "transport", - "transport/spdy", - "util/cert", - "util/connrotation", - "util/exec", - "util/flowcontrol", - "util/homedir", - "util/integer", - ] - pruneopts = "UT" - revision = "e64494209f554a6723674bd494d69445fb76a1d4" - version = "v10.0.0" - -[[projects]] - digest = "1:e2999bf1bb6eddc2a6aa03fe5e6629120a53088926520ca3b4765f77d7ff7eab" - name = "k8s.io/klog" - packages = ["."] - pruneopts = "UT" - revision = "8139d8cb77af419532b33dfa7dd09fbc5f1d344f" - -[[projects]] - digest = "1:7719608fe0b52a4ece56c2dde37bedd95b938677d1ab0f84b8a7852e4c59f849" - name = "sigs.k8s.io/yaml" - packages = ["."] - pruneopts = "UT" - revision = "fd68e9863619f6ec2fdd8625fe1f02e7c877e480" - version = "v1.1.0" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - input-imports = [ - "github.com/Azure/azure-pipeline-go/pipeline", - "github.com/Azure/azure-storage-blob-go/azblob", - "github.com/aws/aws-sdk-go/aws", - "github.com/aws/aws-sdk-go/aws/session", - "github.com/aws/aws-sdk-go/service/s3", - "github.com/aws/aws-sdk-go/service/s3/s3manager", - "github.com/djherbis/buffer", - "github.com/spf13/cobra", - "gopkg.in/djherbis/nio.v2", - "k8s.io/api/core/v1", - "k8s.io/apimachinery/pkg/runtime", - "k8s.io/client-go/kubernetes", - "k8s.io/client-go/rest", - "k8s.io/client-go/tools/clientcmd", - "k8s.io/client-go/tools/remotecommand", - ] - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index 4ba0d56..0000000 --- a/Gopkg.toml +++ /dev/null @@ -1,39 +0,0 @@ -[[constraint]] - name = "github.com/Azure/azure-pipeline-go" - version = "0.1.8" - -[[constraint]] - name = "github.com/Azure/azure-storage-blob-go" - version = "0.3.0" - -[[constraint]] - name = "github.com/aws/aws-sdk-go" - version = "1.16.7" - -[[constraint]] - name = "github.com/djherbis/buffer" - version = "1.0.0" - -[[constraint]] - name = "github.com/spf13/cobra" - version = "0.0.3" - -[[constraint]] - name = "gopkg.in/djherbis/nio.v2" - version = "2.0.3" - -[[constraint]] - name = "k8s.io/api" - version = "kubernetes-1.13.0" - -[[constraint]] - branch = "release-1.13" - name = "k8s.io/apimachinery" - -[[constraint]] - name = "k8s.io/client-go" - version = "10.0.0" - -[prune] - go-tests = true - unused-packages = true diff --git a/Makefile b/Makefile index 90d81b0..705a837 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -HAS_DEP := $(shell command -v dep;) -DEP_VERSION := v0.5.0 -GIT_TAG := $(shell git describe --tags --always) +# HAS_DEP := $(shell command -v dep;) +# DEP_VERSION := v0.5.0 +GIT_TAG := $(shell git tag | tail -n 1) GIT_COMMIT := $(shell git rev-parse --short HEAD) LDFLAGS := "-X main.GitTag=${GIT_TAG} -X main.GitCommit=${GIT_COMMIT}" DIST := $(CURDIR)/dist @@ -39,11 +39,11 @@ endif endif bootstrap: -ifndef HAS_DEP - wget -q -O $(GOPATH)/bin/dep https://github.com/golang/dep/releases/download/$(DEP_VERSION)/dep-linux-amd64 - chmod +x $(GOPATH)/bin/dep -endif - dep ensure +# ifndef HAS_DEP +# wget -q -O $(GOPATH)/bin/dep https://github.com/golang/dep/releases/download/$(DEP_VERSION)/dep-linux-amd64 +# chmod +x $(GOPATH)/bin/dep +# endif +# dep ensure dist: mkdir -p $(DIST) diff --git a/cmd/skbn.go b/cmd/skbn.go index b3ef0be..5a5e5d8 100644 --- a/cmd/skbn.go +++ b/cmd/skbn.go @@ -35,10 +35,13 @@ func NewRootCmd(args []string) *cobra.Command { } type cpCmd struct { - src string - dst string - parallel int - bufferSize float64 + src string + dst string + parallel int + bufferSize float64 + s3partSize int64 + s3maxUploadParts int + verbose bool out io.Writer } @@ -52,7 +55,7 @@ func NewCpCmd(out io.Writer) *cobra.Command { Short: "Copy files or directories Kubernetes and Cloud storage", Long: ``, Run: func(cmd *cobra.Command, args []string) { - if err := skbn.Copy(c.src, c.dst, c.parallel, c.bufferSize); err != nil { + if err := skbn.Copy(c.src, c.dst, c.parallel, c.bufferSize, c.s3partSize, c.s3maxUploadParts, c.verbose); err != nil { log.Fatal(err) } }, @@ -63,6 +66,9 @@ func NewCpCmd(out io.Writer) *cobra.Command { f.StringVar(&c.dst, "dst", "", "path to copy to. Example: s3:///path/to/copyto") f.IntVarP(&c.parallel, "parallel", "p", 1, "number of files to copy in parallel. set this flag to 0 for full parallelism") f.Float64VarP(&c.bufferSize, "buffer-size", "b", 6.75, "in memory buffer size (MB) to use for files copy (buffer per file)") + f.Int64VarP(&c.s3partSize, "s3-part-size", "s", 128*1024*1024, "size of each part in bytes for multipart upload to S3. Default is 128MB. Consider that the default MaxUploadParts is 10000 so max file size with default s3 settings is 1.28TB.") + f.IntVarP(&c.s3maxUploadParts, "s3-max-upload-parts", "m", 10000, "maximum number of parts for multipart upload to S3. Default is 10000.") + f.BoolVarP(&c.verbose, "verbose", "v", false, "verbose output") cmd.MarkFlagRequired("src") cmd.MarkFlagRequired("dst") diff --git a/examples/code/example.go b/examples/code/example.go index 19a69af..4d9345b 100644 --- a/examples/code/example.go +++ b/examples/code/example.go @@ -9,10 +9,13 @@ import ( func main() { src := "k8s://namespace/pod/container/path/to/copy/from" dst := "s3://bucket/path/to/copy/to" - parallel := 0 // all at once - bufferSize := 1.0 // 1GB of in memory buffer size + parallel := 0 // all at once + bufferSize := 1.0 // 1GB of in memory buffer size + s3partSize := int64(5 * 1024 * 1024) // 5MB + s3maxUploadParts := 10000 + verbose := true - if err := skbn.Copy(src, dst, parallel, bufferSize); err != nil { + if err := skbn.Copy(src, dst, parallel, bufferSize, s3partSize, s3maxUploadParts, verbose); err != nil { log.Fatal(err) } } diff --git a/examples/in-cluster/job.yaml b/examples/in-cluster/job.yaml index 2bc3a23..eb7bdf8 100644 --- a/examples/in-cluster/job.yaml +++ b/examples/in-cluster/job.yaml @@ -1,30 +1,40 @@ apiVersion: batch/v1 kind: Job metadata: + name: test-skbn labels: - app: skbn - name: skbn + name: test-skbn + app: test-skbn spec: template: metadata: labels: - app: skbn + name: test-skbn + app: test-skbn + spotinst.io/restrict-scale-down: "true" annotations: - iam.amazonaws.com/role: skbn # We are using kube2iam + iam.amazonaws.com/role: test-dbbackup-operator + spec: restartPolicy: OnFailure - serviceAccountName: skbn + serviceAccount: opsflow containers: - - name: skbn - image: nuvo/skbn + - name: test-skbn + image: nuvo/skbn:0.5.6 + resources: + limits: + memory: "3Gi" + cpu: "2000m" command: ["skbn"] - args: - - cp - - --src - - k8s://namespace/pod/container/path/to/copy/from - - --dst - - s3://bucket/path/to/copy/to - imagePullPolicy: IfNotPresent + args: [ + "cp", + "--parallel", "1", + "--src", "k8s://invu-sre/cassandra-primary-r1-sts-0/cassandra/var/lib/cassandra/test.file", + "--dst", "s3://nuvo-dev-dbbackup/test.file", + "--buffer-size", "512", + "--verbose" + ] env: - name: AWS_REGION - value: eu-central-1 + value: us-east-1 + imagePullPolicy: IfNotPresent diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ec46e2e --- /dev/null +++ b/go.mod @@ -0,0 +1,50 @@ +module github.com/nuvo/skbn + +go 1.20 + +require ( + github.com/Azure/azure-pipeline-go v0.2.3 + github.com/Azure/azure-storage-blob-go v0.15.0 + github.com/aws/aws-sdk-go v1.53.20 + github.com/djherbis/buffer v1.2.0 + github.com/djherbis/nio/v3 v3.0.1 + github.com/spf13/cobra v1.8.0 + k8s.io/api v0.0.0-20181204000039-89a74a8d264d + k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93 + k8s.io/client-go v10.0.0+incompatible +) + +require ( + github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 // indirect + github.com/elazarl/goproxy v0.0.0-20231117061959-7cc037d33fb5 // indirect + github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415 // indirect + github.com/golang/protobuf v1.2.0 // indirect + github.com/google/btree v1.0.0 // indirect + github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367 // indirect + github.com/google/uuid v1.2.0 // indirect + github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d // indirect + github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7 // indirect + github.com/imdario/mergo v0.3.5 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be // indirect + github.com/mattn/go-ieproxy v0.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/testify v1.9.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/oauth2 v0.0.0-20170412232759-a6bd8cefa181 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/time v0.0.0-20161028155119-f51c12702a4d // indirect + google.golang.org/appengine v1.3.0 // indirect + gopkg.in/inf.v0 v0.9.0 // indirect + gopkg.in/yaml.v2 v2.2.8 // indirect + k8s.io/klog v0.0.0-20181108234604-8139d8cb77af // indirect + sigs.k8s.io/yaml v1.1.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..edc3a7f --- /dev/null +++ b/go.sum @@ -0,0 +1,136 @@ +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= +github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= +github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/aws/aws-sdk-go v1.53.20 h1:cYWPvZLP1gPj5CfUdnfjaaA7WFK3FGoJ/R9+Ks1inU4= +github.com/aws/aws-sdk-go v1.53.20/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/djherbis/buffer v1.1.0/go.mod h1:VwN8VdFkMY0DCALdY8o00d3IZ6Amz/UNVMWcSaJT44o= +github.com/djherbis/buffer v1.2.0 h1:PH5Dd2ss0C7CRRhQCZ2u7MssF+No9ide8Ye71nPHcrQ= +github.com/djherbis/buffer v1.2.0/go.mod h1:fjnebbZjCUpPinBRD+TDwXSOeNQ7fPQWLfGQqiAiUyE= +github.com/djherbis/nio/v3 v3.0.1 h1:6wxhnuppteMa6RHA4L81Dq7ThkZH8SwnDzXDYy95vB4= +github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmWgZxOcmg= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/elazarl/goproxy v0.0.0-20231117061959-7cc037d33fb5 h1:m62nsMU279qRD9PQSWD1l66kmkXzuYcnVJqL4XLeV2M= +github.com/elazarl/goproxy v0.0.0-20231117061959-7cc037d33fb5/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415 h1:WSBJMqJbLxsn+bTCPyPYZfqHdJmc8MK4wrBjMft6BAM= +github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367 h1:ScAXWS+TR6MZKex+7Z8rneuSJH+FSDqd6ocQyl+ZHo4= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7 h1:6TSoaYExHper8PYsJu23GWVNOyYRCSnIFyxKgLSZ54w= +github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be h1:AHimNtVIpiBjPUhEF5KNCkrUyqTSA5zWUl8sQ2bfGBE= +github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/oauth2 v0.0.0-20170412232759-a6bd8cefa181 h1:/4OaQ4bC66Oq9JDhUnxTjBGt8XBhDuwgMRXHgvfcCUY= +golang.org/x/oauth2 v0.0.0-20170412232759-a6bd8cefa181/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.0.0-20161028155119-f51c12702a4d h1:TnM+PKb3ylGmZvyPXmo9m/wktg7Jn/a/fNmr33HSj8g= +golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o= +gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.0.0-20181204000039-89a74a8d264d h1:HQoGWsWUe/FmRcX9BU440AAMnzBFEf+DBo4nbkQlNzs= +k8s.io/api v0.0.0-20181204000039-89a74a8d264d/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93 h1:tT6oQBi0qwLbbZSfDkdIsb23EwaLY85hoAV4SpXfdao= +k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/client-go v10.0.0+incompatible h1:F1IqCqw7oMBzDkqlcBymRq1450wD0eNqLE9jzUrIi34= +k8s.io/client-go v10.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/klog v0.0.0-20181108234604-8139d8cb77af h1:s6rm8OxBbyDNSRkpyAd5OL4icUdBICVw9+mFADa+t5E= +k8s.io/klog v0.0.0-20181108234604-8139d8cb77af/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/pkg/skbn/abs.go b/pkg/skbn/abs.go index e44e7da..7ed279b 100644 --- a/pkg/skbn/abs.go +++ b/pkg/skbn/abs.go @@ -70,7 +70,7 @@ func GetListOfFilesFromAbs(ctx context.Context, iClient interface{}, path string } // DownloadFromAbs downloads a single file from azure blob storage -func DownloadFromAbs(ctx context.Context, iClient interface{}, path string, writer io.Writer) error { +func DownloadFromAbs(ctx context.Context, iClient interface{}, path string, writer io.Writer, verbose bool) error { pSplit := strings.Split(path, "/") if err := validateAbsPath(pSplit); err != nil { @@ -84,7 +84,7 @@ func DownloadFromAbs(ctx context.Context, iClient interface{}, path string, writ } bu := getBlobURL(cu, p) - dr, err := bu.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false) + dr, err := bu.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{}) if err != nil { return err } @@ -100,7 +100,7 @@ func DownloadFromAbs(ctx context.Context, iClient interface{}, path string, writ } // UploadToAbs uploads a single file to azure blob storage -func UploadToAbs(ctx context.Context, iClient interface{}, toPath, fromPath string, reader io.Reader) error { +func UploadToAbs(ctx context.Context, iClient interface{}, toPath, fromPath string, reader io.Reader, verbose bool) error { pSplit := strings.Split(toPath, "/") if err := validateAbsPath(pSplit); err != nil { return err diff --git a/pkg/skbn/kube.go b/pkg/skbn/kube.go index 61110ef..35ccb8a 100644 --- a/pkg/skbn/kube.go +++ b/pkg/skbn/kube.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "io" + "log" "os" "path/filepath" "strings" @@ -105,7 +106,7 @@ func GetListOfFilesFromK8s(iClient interface{}, path, findType, findName string) } // DownloadFromK8s downloads a single file from Kubernetes -func DownloadFromK8s(iClient interface{}, path string, writer io.Writer) error { +func DownloadFromK8s(iClient interface{}, path string, writer io.Writer, verbose bool) error { client := *iClient.(*K8sClient) pSplit := strings.Split(path, "/") if err := validateK8sPath(pSplit); err != nil { @@ -118,13 +119,24 @@ func DownloadFromK8s(iClient interface{}, path string, writer io.Writer) error { attempt := 0 for attempt < attempts { attempt++ + if verbose { + log.Printf("Attempt %d to download file from %s/%s/%s:%s", attempt, namespace, podName, containerName, pathToCopy) + } stderr, err := Exec(client, namespace, podName, containerName, command, nil, writer) + + if (verbose && len(stderr) != 0) || err != nil { + log.Printf("STDERR: %s", stderr) + log.Printf("Error: %v", err) + } if attempt == attempts { + log.Printf("this was last attempt") if len(stderr) != 0 { + log.Printf("STDERR: %s", stderr) return fmt.Errorf("STDERR: " + (string)(stderr)) } if err != nil { + log.Printf("Error: %v", err) return err } } @@ -138,7 +150,7 @@ func DownloadFromK8s(iClient interface{}, path string, writer io.Writer) error { } // UploadToK8s uploads a single file to Kubernetes -func UploadToK8s(iClient interface{}, toPath, fromPath string, reader io.Reader) error { +func UploadToK8s(iClient interface{}, toPath, fromPath string, reader io.Reader, verbose bool) error { client := *iClient.(*K8sClient) pSplit := strings.Split(toPath, "/") if err := validateK8sPath(pSplit); err != nil { diff --git a/pkg/skbn/s3.go b/pkg/skbn/s3.go index 3fc06e2..29d7e3e 100644 --- a/pkg/skbn/s3.go +++ b/pkg/skbn/s3.go @@ -3,6 +3,7 @@ package skbn import ( "fmt" "io" + "log" "os" "path/filepath" "strconv" @@ -80,10 +81,13 @@ func GetListOfFilesFromS3(iClient interface{}, path string) ([]string, error) { } // DownloadFromS3 downloads a single file from S3 -func DownloadFromS3(iClient interface{}, path string, writer io.Writer) error { +func DownloadFromS3(iClient interface{}, path string, writer io.Writer, verbose bool) error { s := iClient.(*session.Session) pSplit := strings.Split(path, "/") if err := validateS3Path(pSplit); err != nil { + if verbose { + log.Printf("validate s3 path error: %s", err) + } return err } bucket, s3Path := initS3Variables(pSplit) @@ -93,6 +97,10 @@ func DownloadFromS3(iClient interface{}, path string, writer io.Writer) error { for attempt < attempts { attempt++ + if verbose { + log.Printf("Attempt %d to download file from s3://%s/%s", attempt, bucket, s3Path) + } + downloader := s3manager.NewDownloader(s) downloader.Concurrency = 1 // support writerWrapper @@ -101,8 +109,19 @@ func DownloadFromS3(iClient interface{}, path string, writer io.Writer) error { Bucket: aws.String(bucket), Key: aws.String(s3Path), }) + + if verbose { + log.Printf("Downloaded file from s3://%s/%s", bucket, s3Path) + } if err != nil { + if verbose { + log.Printf("Error: %v", err) + log.Printf("Attempt: %v", attempt) + } if attempt == attempts { + if verbose { + log.Printf("This was last attempt") + } return err } utils.Sleep(attempt) @@ -123,10 +142,13 @@ func (ww writerWrapper) WriteAt(p []byte, off int64) (n int, err error) { } // UploadToS3 uploads a single file to S3 -func UploadToS3(iClient interface{}, toPath, fromPath string, reader io.Reader) error { +func UploadToS3(iClient interface{}, toPath, fromPath string, reader io.Reader, s3partSize int64, s3maxUploadParts int, verbose bool) error { s := iClient.(*session.Session) pSplit := strings.Split(toPath, "/") if err := validateS3Path(pSplit); err != nil { + if verbose { + log.Printf("validate s3 path error: %s", err) + } return err } if len(pSplit) == 1 { @@ -140,15 +162,34 @@ func UploadToS3(iClient interface{}, toPath, fromPath string, reader io.Reader) for attempt < attempts { attempt++ - uploader := s3manager.NewUploader(s) + if verbose { + log.Printf("Attempt %d to upload file to s3://%s/%s", attempt, bucket, s3Path) + } + + // uploader := s3manager.NewUploader(s) + uploader := s3manager.NewUploader(s, func(u *s3manager.Uploader) { + u.PartSize = s3partSize + u.MaxUploadParts = s3maxUploadParts + }) _, err := uploader.Upload(&s3manager.UploadInput{ Bucket: aws.String(bucket), Key: aws.String(s3Path), Body: reader, }) + + if verbose { + log.Printf("Uploaded file to s3://%s/%s", bucket, s3Path) + } if err != nil { + if verbose { + log.Printf("Error: %v", err) + log.Printf("Attempt: %v", attempt) + } if attempt == attempts { + if verbose { + log.Printf("This was last attempt") + } return err } utils.Sleep(attempt) @@ -160,6 +201,16 @@ func UploadToS3(iClient interface{}, toPath, fromPath string, reader io.Reader) return nil } +// calculatePartSize calculates an appropriate part size for the multipart upload +func calculatePartSize(fileSize int64) int64 { + const maxParts = 10000 + partSize := fileSize / maxParts + if partSize < 5*1024*1024 { + partSize = 5 * 1024 * 1024 // Minimum part size of 5 MB + } + return partSize +} + func getNewSession() (*session.Session, error) { awsConfig := &aws.Config{} diff --git a/pkg/skbn/skbn.go b/pkg/skbn/skbn.go index 2a9889c..f480c3b 100644 --- a/pkg/skbn/skbn.go +++ b/pkg/skbn/skbn.go @@ -11,7 +11,7 @@ import ( "github.com/nuvo/skbn/pkg/utils" "github.com/djherbis/buffer" - "gopkg.in/djherbis/nio.v2" + "github.com/djherbis/nio/v3" ) // FromToPair is a pair of FromPath and ToPath @@ -21,7 +21,7 @@ type FromToPair struct { } // Copy copies files from src to dst -func Copy(src, dst string, parallel int, bufferSize float64) error { +func Copy(src, dst string, parallel int, bufferSize float64, s3partSize int64, s3maxUploadParts int, verbose bool) error { srcPrefix, srcPath := utils.SplitInTwo(src, "://") dstPrefix, dstPath := utils.SplitInTwo(dst, "://") @@ -37,7 +37,7 @@ func Copy(src, dst string, parallel int, bufferSize float64) error { if err != nil { return err } - err = PerformCopy(srcClient, dstClient, srcPrefix, dstPrefix, fromToPaths, parallel, bufferSize) + err = PerformCopy(srcClient, dstClient, srcPrefix, dstPrefix, fromToPaths, parallel, bufferSize, s3partSize, s3maxUploadParts, verbose) if err != nil { return err } @@ -101,7 +101,7 @@ func GetFromToPaths(srcClient interface{}, srcPrefix, srcPath, dstPath string) ( } // PerformCopy performs the actual copy action -func PerformCopy(srcClient, dstClient interface{}, srcPrefix, dstPrefix string, fromToPaths []FromToPair, parallel int, bufferSize float64) error { +func PerformCopy(srcClient, dstClient interface{}, srcPrefix, dstPrefix string, fromToPaths []FromToPair, parallel int, bufferSize float64, s3partSize int64, s3maxUploadParts int, verbose bool) error { // Execute in parallel totalFiles := len(fromToPaths) @@ -141,7 +141,7 @@ func PerformCopy(srcClient, dstClient interface{}, srcPrefix, dstPrefix string, if len(errc) != 0 { return } - err := Download(srcClient, srcPrefix, fromPath, pw) + err := Download(srcClient, srcPrefix, fromPath, pw, verbose) if err != nil { log.Println(err, fmt.Sprintf(" src: file: %s", fromPath)) errc <- err @@ -149,13 +149,13 @@ func PerformCopy(srcClient, dstClient interface{}, srcPrefix, dstPrefix string, }() go func() { - defer pr.Close() defer bwg.Done() + defer pr.Close() if len(errc) != 0 { return } defer log.Printf("[%s/%d] done: %s://%s -> %s://%s", currentLinePadded, totalFiles, srcPrefix, fromPath, dstPrefix, toPath) - err := Upload(dstClient, dstPrefix, toPath, fromPath, pr) + err := Upload(dstClient, dstPrefix, toPath, fromPath, pr, s3partSize, s3maxUploadParts, verbose) if err != nil { log.Println(err, fmt.Sprintf(" dst: file: %s", toPath)) errc <- err @@ -211,23 +211,23 @@ func GetListOfFiles(client interface{}, prefix, path string) ([]string, error) { } // Download downloads a single file from path into an io.Writer -func Download(srcClient interface{}, srcPrefix, srcPath string, writer io.Writer) error { +func Download(srcClient interface{}, srcPrefix, srcPath string, writer io.Writer, verbose bool) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() switch srcPrefix { case "k8s": - err := DownloadFromK8s(srcClient, srcPath, writer) + err := DownloadFromK8s(srcClient, srcPath, writer, verbose) if err != nil { return err } case "s3": - err := DownloadFromS3(srcClient, srcPath, writer) + err := DownloadFromS3(srcClient, srcPath, writer, verbose) if err != nil { return err } case "abs": - err := DownloadFromAbs(ctx, srcClient, srcPath, writer) + err := DownloadFromAbs(ctx, srcClient, srcPath, writer, verbose) if err != nil { return err } @@ -239,23 +239,23 @@ func Download(srcClient interface{}, srcPrefix, srcPath string, writer io.Writer } // Upload uploads a single file provided as an io.Reader array to path -func Upload(dstClient interface{}, dstPrefix, dstPath, srcPath string, reader io.Reader) error { +func Upload(dstClient interface{}, dstPrefix, dstPath, srcPath string, reader io.Reader, s3partSize int64, s3maxUploadParts int, verbose bool) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() switch dstPrefix { case "k8s": - err := UploadToK8s(dstClient, dstPath, srcPath, reader) + err := UploadToK8s(dstClient, dstPath, srcPath, reader, verbose) if err != nil { return err } case "s3": - err := UploadToS3(dstClient, dstPath, srcPath, reader) + err := UploadToS3(dstClient, dstPath, srcPath, reader, s3partSize, s3maxUploadParts, verbose) if err != nil { return err } case "abs": - err := UploadToAbs(ctx, dstClient, dstPath, srcPath, reader) + err := UploadToAbs(ctx, dstClient, dstPath, srcPath, reader, verbose) if err != nil { return err }