From 7affcf304de0dc4f5c2093618c57419cff3edfa1 Mon Sep 17 00:00:00 2001 From: Stefan Sundin Date: Mon, 15 Aug 2022 22:13:41 -0700 Subject: [PATCH] Replace flag with zflag. --- README.md | 119 +++++++++++++++---------------------------- example-schedule.txt | 4 +- flowrate/README.md | 2 +- go.mod | 1 + go.sum | 2 + main.go | 21 ++++---- minio-server.sh | 2 +- 7 files changed, 57 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index 8246d36..0073122 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ shrimp is a small program that can reliably upload large files to Amazon S3. Features: - shrimp supports most of the arguments used for `aws s3 cp`. In many cases you can simply replace `aws s3 cp` with `shrimp` and everything will work. -- shrimp has interactive keyboard controls that lets you limit the bandwidth used for the upload (you can specify an initial limit with `-bwlimit`, e.g. `-bwlimit=2.5m` for 2.5 MB/s). While the upload is in progress, press ? to see the available keyboard controls. +- shrimp has interactive keyboard controls that lets you limit the bandwidth used for the upload (you can specify an initial limit with `--bwlimit`, e.g. `--bwlimit=2.5m` for 2.5 MB/s). While the upload is in progress, press ? to see the available keyboard controls. - shrimp can automatically adjust the bandwidth limit based on a schedule. [See here for more information.](https://github.com/stefansundin/s3sha256sum/discussions/4) - shrimp can resume the upload in case it fails for whatever reason (just re-run the command). Unlike the aws cli, shrimp will never abort the multipart upload in case of failures ([please set up a lifecycle policy for this!](https://aws.amazon.com/blogs/aws-cloud-financial-management/discovering-and-deleting-incomplete-multipart-uploads-to-lower-amazon-s3-costs/)). -- shrimp supports the [Additional Checksum Algorithms feature released in February 2022](https://aws.amazon.com/blogs/aws/new-additional-checksum-algorithms-for-amazon-s3/). Use `-checksum-algorithm` to allow verification of the object without the need to download it, e.g. using [s3verify](https://github.com/stefansundin/s3verify). -- shrimp also supports automatically attaching a SHA256 checksum to the object metadata if a `SHA256SUMS` file is present in the working directory. Use `-compute-checksum` if you want shrimp to calculate the checksum and add it to the `SHA256SUMS` file. You can use [s3sha256sum](https://github.com/stefansundin/s3sha256sum) to verify the object after it has been uploaded. The `-checksum-algorithm` feature somewhat supercedes this, but there are still uses for this checksum, especially for multi-part objects. [See here for more information.](https://github.com/stefansundin/s3sha256sum/discussions/1) +- shrimp supports the [Additional Checksum Algorithms feature released in February 2022](https://aws.amazon.com/blogs/aws/new-additional-checksum-algorithms-for-amazon-s3/). Use `--checksum-algorithm` to allow verification of the object without the need to download it, e.g. using [s3verify](https://github.com/stefansundin/s3verify). +- shrimp also supports automatically attaching a SHA256 checksum to the object metadata if a `SHA256SUMS` file is present in the working directory. Use `--compute-checksum` if you want shrimp to calculate the checksum and add it to the `SHA256SUMS` file. You can use [s3sha256sum](https://github.com/stefansundin/s3sha256sum) to verify the object after it has been uploaded. The `--checksum-algorithm` feature somewhat supercedes this, but there are still uses for this checksum, especially for multi-part objects. [See here for more information.](https://github.com/stefansundin/s3sha256sum/discussions/1) Keep in mind: - shrimp will always use a multipart upload, so do not use it for small files. @@ -27,84 +27,47 @@ go install github.com/stefansundin/shrimp@latest ## Usage ``` -$ shrimp -help +$ shrimp --help Usage: shrimp [parameters] LocalPath must be a local file. S3Uri must have the format s3:///. Parameters: - -bucket-key-enabled - Enables use of an S3 Bucket Key for object encryption with server-side encryption using AWS KMS (SSE-KMS). - -bwlimit string - Bandwidth limit. (e.g. "2.5m") - -ca-bundle string - The CA certificate bundle to use when verifying SSL certificates. - -cache-control string - Specifies caching behavior for the object. - -checksum-algorithm string - The checksum algorithm to use for the object. Supported values: CRC32, CRC32C, SHA1, SHA256. - -compute-checksum - Compute checksum and add to SHA256SUMS file. - -content-disposition string - Specifies presentational information for the object. - -content-encoding string - Specifies what content encodings have been applied to the object. - -content-language string - Specifies the language the content is in. - -content-type string - A standard MIME type describing the format of the object data. - -debug - Turn on debug logging. - -dryrun - Checks if the upload was started previously and how much was completed. (use in combination with -bwlimit to calculate remaining time) - -endpoint-url string - Override the S3 endpoint URL. (for use with S3 compatible APIs) - -expected-bucket-owner string - The account ID of the expected bucket owner. - -force - Overwrite existing object. - -metadata string - A map of metadata to store with the object in S3. (JSON syntax is not supported) - -mfa-duration duration - MFA duration. shrimp will prompt for another code after this duration. (max "12h") (default 1h0m0s) - -mfa-secret - Provide the MFA secret and shrimp will automatically generate TOTP codes. (useful if the upload takes longer than the allowed assume role duration) - -no-sign-request - Do not sign requests. This does not work with Amazon S3, but may work with other S3 APIs. - -no-verify-ssl - Do not verify SSL certificates. - -object-lock-legal-hold-status string - Specifies whether a legal hold will be applied to this object. Possible values: ON, OFF. - -object-lock-mode string - The Object Lock mode that you want to apply to this object. Possible values: GOVERNANCE, COMPLIANCE. - -object-lock-retain-until-date string - The date and time when you want this object's Object Lock to expire. Must be formatted as a timestamp parameter. (e.g. "2022-03-14T15:14:15Z") - -part-size string - Override automatic part size. (e.g. "128m") - -profile string - Use a specific profile from your credential file. - -region string - The bucket region. Avoids one API call. - -request-payer string - Confirms that the requester knows that they will be charged for the requests. Possible values: requester. - -schedule string - Schedule file to use for automatically adjusting the bandwidth limit (see https://github.com/stefansundin/shrimp/discussions/4). - -sse string - Specifies server-side encryption of the object in S3. Valid values are AES256 and aws:kms. - -sse-c string - Specifies server-side encryption using customer provided keys of the the object in S3. AES256 is the only valid value. If you provide this value, -sse-c-key must be specified as well. - -sse-c-key string - The customer-provided encryption key to use to server-side encrypt the object in S3. The key provided should not be base64 encoded. - -sse-kms-key-id string - The customer-managed AWS Key Management Service (KMS) key ID that should be used to server-side encrypt the object in S3. - -storage-class string - Storage class. Known values: STANDARD, REDUCED_REDUNDANCY, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, GLACIER, DEEP_ARCHIVE, OUTPOSTS, GLACIER_IR. - -tagging string - The tag-set for the object. The tag-set must be encoded as URL Query parameters. - -use-accelerate-endpoint - Use S3 Transfer Acceleration. - -use-path-style - Use S3 Path Style. - -version - Print version number. + --bucket-key-enabled Enables use of an S3 Bucket Key for object encryption with server-side encryption using AWS KMS (SSE-KMS). + --bwlimit string Bandwidth limit. (e.g. "2.5m") + --ca-bundle string The CA certificate bundle to use when verifying SSL certificates. + --cache-control string Specifies caching behavior for the object. + --checksum-algorithm string The checksum algorithm to use for the object. Supported values: CRC32, CRC32C, SHA1, SHA256. + --compute-checksum Compute checksum and add to SHA256SUMS file. + --content-disposition string Specifies presentational information for the object. + --content-encoding string Specifies what content encodings have been applied to the object. + --content-language string Specifies the language the content is in. + --content-type string A standard MIME type describing the format of the object data. + --debug Turn on debug logging. + --dryrun Checks if the upload was started previously and how much was completed. (use in combination with --bwlimit to calculate remaining time) + --endpoint-url string Override the S3 endpoint URL. (for use with S3 compatible APIs) + --expected-bucket-owner string The account ID of the expected bucket owner. + --force Overwrite existing object. + --metadata string A map of metadata to store with the object in S3. (JSON syntax is not supported) + --mfa-duration duration MFA duration. shrimp will prompt for another code after this duration. (max "12h") (default 1h0m0s) + --mfa-secret Provide the MFA secret and shrimp will automatically generate TOTP codes. (useful if the upload takes longer than the allowed assume role duration) + --no-sign-request Do not sign requests. This does not work with Amazon S3, but may work with other S3 APIs. + --no-verify-ssl Do not verify SSL certificates. + --object-lock-legal-hold-status string Specifies whether a legal hold will be applied to this object. Possible values: ON, OFF. + --object-lock-mode string The Object Lock mode that you want to apply to this object. Possible values: GOVERNANCE, COMPLIANCE. + --object-lock-retain-until-date string The date and time when you want this object's Object Lock to expire. Must be formatted as a timestamp parameter. (e.g. "2022-03-14T15:14:15Z") + --part-size string Override automatic part size. (e.g. "128m") + --profile string Use a specific profile from your credential file. + --region string The bucket region. Avoids one API call. + --request-payer string Confirms that the requester knows that they will be charged for the requests. Possible values: requester. + --schedule string Schedule file to use for automatically adjusting the bandwidth limit (see https://github.com/stefansundin/shrimp/discussions/4). + --sse string Specifies server-side encryption of the object in S3. Valid values are AES256 and aws:kms. + --sse-c string Specifies server-side encryption using customer provided keys of the the object in S3. AES256 is the only valid value. If you provide this value, --sse-c-key must be specified as well. + --sse-c-key string The customer-provided encryption key to use to server-side encrypt the object in S3. The key provided should not be base64 encoded. + --sse-kms-key-id string The customer-managed AWS Key Management Service (KMS) key ID that should be used to server-side encrypt the object in S3. + --storage-class string Storage class. Known values: STANDARD, REDUCED_REDUNDANCY, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, GLACIER, DEEP_ARCHIVE, OUTPOSTS, GLACIER_IR. + --tagging string The tag-set for the object. The tag-set must be encoded as URL Query parameters. + --use-accelerate-endpoint Use S3 Transfer Acceleration. + --use-path-style Use S3 Path Style. + --version Print version number. ``` diff --git a/example-schedule.txt b/example-schedule.txt index 2a06973..6d2684a 100644 --- a/example-schedule.txt +++ b/example-schedule.txt @@ -1,9 +1,9 @@ -# Example schedule file. Use "-schedule filename.txt" to use feature. +# Example schedule file. Use "--schedule filename.txt" to use feature. # See https://github.com/stefansundin/shrimp/discussions/4 # Lines starting with "#" are ignored. # The schedule can include a default bandwidth limit (if omitted then unlimited is the default). -# The -bwlimit switch overrides the default value specified here. +# The --bwlimit switch overrides the default value specified here. # After the default, define a number of non-overlapping ranges that override the default. # The format is: diff --git a/flowrate/README.md b/flowrate/README.md index 7d5df9a..c067ec1 100644 --- a/flowrate/README.md +++ b/flowrate/README.md @@ -2,7 +2,7 @@ Based on https://github.com/mxk/go-flowrate. Modified to allow seeking and to calculate a total progress. -Optionally, it can also skip rate limiting on the first pass. This is necessary when transferring files over unencrypted HTTP, since the AWS SDK will in this case calculate a checksum of the payload before it initiates the request. This case will only happen when the user uses `-endpoint-url` with an `http://` endpoint. +Optionally, it can also skip rate limiting on the first pass. This is necessary when transferring files over unencrypted HTTP, since the AWS SDK will in this case calculate a checksum of the payload before it initiates the request. This case will only happen when the user uses `--endpoint-url` with an `http://` endpoint. Here's a list of the calls that will occur: diff --git a/go.mod b/go.mod index 9c3a041..c199b8f 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.27.5 github.com/aws/smithy-go v1.12.1 github.com/pkg/term v1.1.0 + github.com/stefansundin/go-zflag v1.1.1 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab ) diff --git a/go.sum b/go.sum index c6332ff..cd089bb 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/pkg/term v1.1.0 h1:xIAAdCMh3QIAy+5FrE8Ad8XoDhEU4ufwbaSozViP9kk= github.com/pkg/term v1.1.0/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stefansundin/go-zflag v1.1.1 h1:XabhzWS588bVvV1z1UctSa6i8zHkXc5W9otqtnDSHw8= +github.com/stefansundin/go-zflag v1.1.1/go.mod h1:HXX5rABl1AoTcZ2jw+CqJ7R8irczaLquGNZlFabZooc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= diff --git a/main.go b/main.go index 65bc2b3..893be21 100644 --- a/main.go +++ b/main.go @@ -7,7 +7,6 @@ import ( "crypto/tls" "encoding/base32" "errors" - "flag" "fmt" "io" "io/fs" @@ -22,6 +21,7 @@ import ( "strings" "time" + flag "github.com/stefansundin/go-zflag" "github.com/stefansundin/shrimp/flowrate" "github.com/stefansundin/shrimp/terminal" @@ -75,7 +75,7 @@ func run() (int, error) { flag.StringVar(&metadata, "metadata", "", "A map of metadata to store with the object in S3. (JSON syntax is not supported)") flag.StringVar(&requestPayer, "request-payer", "", "Confirms that the requester knows that they will be charged for the requests. Possible values: requester.") flag.StringVar(&sse, "sse", "", "Specifies server-side encryption of the object in S3. Valid values are AES256 and aws:kms.") - flag.StringVar(&sseCustomerAlgorithm, "sse-c", "", "Specifies server-side encryption using customer provided keys of the the object in S3. AES256 is the only valid value. If you provide this value, -sse-c-key must be specified as well.") + flag.StringVar(&sseCustomerAlgorithm, "sse-c", "", "Specifies server-side encryption using customer provided keys of the the object in S3. AES256 is the only valid value. If you provide this value, --sse-c-key must be specified as well.") flag.StringVar(&sseCustomerKey, "sse-c-key", "", "The customer-provided encryption key to use to server-side encrypt the object in S3. The key provided should not be base64 encoded.") flag.StringVar(&sseKmsKeyId, "sse-kms-key-id", "", "The customer-managed AWS Key Management Service (KMS) key ID that should be used to server-side encrypt the object in S3.") flag.StringVar(&checksumAlgorithm, "checksum-algorithm", "", "The checksum algorithm to use for the object. Supported values: CRC32, CRC32C, SHA1, SHA256.") @@ -91,7 +91,7 @@ func run() (int, error) { flag.BoolVar(&useAccelerateEndpoint, "use-accelerate-endpoint", false, "Use S3 Transfer Acceleration.") flag.BoolVar(&usePathStyle, "use-path-style", false, "Use S3 Path Style.") flag.BoolVar(&force, "force", false, "Overwrite existing object.") - flag.BoolVar(&dryrun, "dryrun", false, "Checks if the upload was started previously and how much was completed. (use in combination with -bwlimit to calculate remaining time)") + flag.BoolVar(&dryrun, "dryrun", false, "Checks if the upload was started previously and how much was completed. (use in combination with --bwlimit to calculate remaining time)") flag.BoolVar(&debug, "debug", false, "Turn on debug logging.") flag.BoolVar(&versionFlag, "version", false, "Print version number.") flag.Usage = func() { @@ -118,14 +118,11 @@ func run() (int, error) { } else if flag.NArg() < 2 { flag.Usage() fmt.Fprintln(os.Stderr) - fmt.Fprintln(os.Stderr, "Error: LocalPath and S3Uri parameters are required!") - return 1, nil + return 1, errors.New("Error: LocalPath and S3Uri parameters are required!") } else if flag.NArg() > 2 { flag.Usage() fmt.Fprintln(os.Stderr) - fmt.Fprintln(os.Stderr, "Error: Unfortunately shrimp requires positional arguments (LocalPath and S3Uri) to be specified last.") - fmt.Fprintln(os.Stderr, "I will probably replace the flag parsing library in the future to address this.") - return 1, nil + return 1, errors.New("Error: Too many positional arguments!") } if endpointURL != "" { @@ -150,7 +147,7 @@ func run() (int, error) { fmt.Fprintln(os.Stderr, "Warning: MFA duration can not exceed 12 hours.") } if mfaSecretFlag { - fmt.Fprintln(os.Stderr, "Read more about the -mfa-secret feature here: https://github.com/stefansundin/shrimp/discussions/3") + fmt.Fprintln(os.Stderr, "Read more about the --mfa-secret feature here: https://github.com/stefansundin/shrimp/discussions/3") secret, ok := os.LookupEnv("AWS_MFA_SECRET") if ok { fmt.Fprintln(os.Stderr, "MFA secret read from AWS_MFA_SECRET.") @@ -315,7 +312,7 @@ func run() (int, error) { return 1, err } else if sum == "" { if !computeChecksum { - fmt.Fprintln(os.Stderr, "Warning: SHA256SUMS file is present but does not have an entry for this file. Consider using -compute-checksum.") + fmt.Fprintln(os.Stderr, "Warning: SHA256SUMS file is present but does not have an entry for this file. Consider using --compute-checksum.") } } else { if createMultipartUploadInput.Metadata == nil { @@ -477,7 +474,7 @@ func run() (int, error) { if obj != nil || err == nil || !isSmithyErrorCode(err, 404) { if obj != nil { fmt.Fprintln(os.Stderr, "The object already exists in the S3 bucket.") - fmt.Fprintln(os.Stderr, "Please delete it or use -force to overwrite the existing object.") + fmt.Fprintln(os.Stderr, "Please delete it or use --force to overwrite the existing object.") } return 1, err } @@ -531,7 +528,7 @@ func run() (int, error) { if createMultipartUploadInput.StorageClass != "" && upload.StorageClass != createMultipartUploadInput.StorageClass { - return 1, fmt.Errorf("Error: existing upload uses the storage class %s. You requested %s. Either make them match or remove -storage-class.\n", upload.StorageClass, createMultipartUploadInput.StorageClass) + return 1, fmt.Errorf("Error: existing upload uses the storage class %s. You requested %s. Either make them match or remove --storage-class.\n", upload.StorageClass, createMultipartUploadInput.StorageClass) } } diff --git a/minio-server.sh b/minio-server.sh index b9b81be..0cd5645 100755 --- a/minio-server.sh +++ b/minio-server.sh @@ -1,6 +1,6 @@ #!/bin/bash -ex # When testing with minio, add these arguments to shrimp: -# -profile minio -endpoint-url https://localhost:9000/ -no-verify-ssl +# --profile minio --endpoint-url https://localhost:9000/ --no-verify-ssl mkdir -p minio-data export MINIO_ROOT_USER=admin