From c8afd160cf5fc140e66cf493b4bf234bce8b12e0 Mon Sep 17 00:00:00 2001 From: Chris Guest Date: Sun, 23 Apr 2023 23:56:59 +0100 Subject: [PATCH 1/2] Trying to get content range and checksums working. --- 33_awscli/S3.md | 73 ++++++++++++++++++++++++++++++++++++- 33_awscli/boto-tool/main.py | 46 ++++++++++++++++++++++- 2 files changed, 116 insertions(+), 3 deletions(-) diff --git a/33_awscli/S3.md b/33_awscli/S3.md index 5a1c57e..aca59af 100644 --- a/33_awscli/S3.md +++ b/33_awscli/S3.md @@ -16,7 +16,7 @@ NOTES: TODO: -* generate crc example `aws s3api head-object --bucket chris-test-bucket-444 --key test/random.bin --checksum-mode Enabled --query ChecksumCRC32 --output text` +* generate crc example `aws s3api head-object --bucket mybucket --key test/random.bin --checksum-mode Enabled --query ChecksumCRC32 --output text` ## Configure @@ -93,6 +93,10 @@ Presigned URLs are often used in scenarios where an application or user needs to Presigned URLs are often used for secure distribution of content, sharing of large files, and temporary access to resources for specific operations or tasks. They provide a secure and efficient way to share resources with limited permissions and help to ensure that resources are not accessed or modified beyond their intended use. +### Presigned GET Urls + +NOTE: The AWSCLI only creates presigned GET urls [details here](https://github.com/aws/aws-cli/issues/3050) + ```sh # copy a file to a bucket aws s3 cp ../BATCH.md s3://bucket @@ -106,6 +110,66 @@ curl -v -s -o /dev/null $SIGNEDURL1 curl -v -s -o /dev/null $SIGNEDURL2 ``` +### Presigned PUT & POST Urls + +TODO: Can I get a crc check on a signed url? +TODO: Add a range example + +Goto [./boto-tool/README.md](./boto-tool/README.md) + +```sh +dd if=/dev/urandom of=../out/random.bin bs=1024 count=100 + +md5sum ../out/random.bin +sha256sum ../out/random.bin +sha256sum ../out/random.bin | tr " " "\n" | head -n 1 | base64 + +SIGNEDURL1=$(pipenv run start --signed --put --bucket ${BUCKET_NAME} --prefix random6.bin | jq -r -s '.[].url' | grep https --color=no) +echo $SIGNEDURL1 +SIGNEDURL1=$(pipenv run start --signed --post --bucket ${BUCKET_NAME} --prefix random6.bin | jq -r -s '.[].url' | grep https --color=no) +echo $SIGNEDURL1 + +# upload file PUT +curl -v -X PUT -T "../out/random.bin" -H "Content-Type: application/octet-stream" -H "Content-Length: 102400" -H "Content-MD5: c27a965ecedc5d08e45969a30754ab39" $SIGNEDURL1 + +curl -v -X PUT -T "../out/random.bin" -H "Content-Type: application/octet-stream" -H "Content-Length: 102400" $SIGNEDURL1 + +curl -v -X PUT -T "../out/random.bin" -H "Content-Type: application/octet-stream" -H "Content-Length: 102400" -H "x-amz-checksum-sha256: 2ba686ec462c2a1771d3b2f8d425d9b1beb6016200d35e1fdfff9d76fb2da2a1" $SIGNEDURL1 + +# POST - NOT WORKING +curl -v -X POST -T "../out/random.bin" -H "Content-Type: application/octet-stream" -H "Content-Length: 102400" $SIGNEDURL1 + +# list files +AWS_PROFILE=myprofile aws s3 ls ${BUCKET_NAME} + +# check md5 matches the etag. +aws s3api get-object-attributes --bucket ${BUCKET_NAME} --key random.bin --object-attributes Checksum ObjectSize ETag --output json | jq -r '.ETag' + +``` + +## TEST + +TODO: Work out why content range and checksums are not working. +https://gist.github.com/alexdebrie/3e8b96217f5aff01227050b17a24e380 + +```sh +stat ../out/random.bin + +pipenv run start --signed --post --bucket ${BUCKET_NAME} --prefix random.bin + +sha256sum ../out/random.bin | tr " " "\n" | head -n 1 | base64 + + + +AWS_PROFILE=trint-toys aws s3 rm s3://${BUCKET_NAME}/random.bin + +./presigned_test.sh + + +``` + + + ## Deleting ```sh @@ -147,3 +211,10 @@ aws s3api list-objects --bucket mybucket --prefix testsruntimefolder/1678128187/ * New – Additional Checksum Algorithms for Amazon S3 [here](https://aws.amazon.com/blogs/aws/new-additional-checksum-algorithms-for-amazon-s3/) * Support S3 additional checksums in high-level S3 commands #6750 [here](https://github.com/aws/aws-cli/issues/6750) * Amazon S3 Inventory Reports [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-inventory.html) +* AWS CLI s3 presign [here](https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/presign.html) +* Checking object integrity [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html) +* Uploading objects [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html) +* PutObject [here](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html) +* Generating an SHA-256 Hash From the Command Line [here](https://www.baeldung.com/linux/sha-256-from-command-line) +* Upload files to AWS S3 using pre-signed POST data and a Lambda function [here](https://www.webiny.com/blog/upload-files-to-aws-s3-using-pre-signed-post-data-and-a-lambda-function-7a9fb06d56c1) +* Differences between PUT and POST S3 signed URLs [here](https://advancedweb.hu/differences-between-put-and-post-s3-signed-urls/) diff --git a/33_awscli/boto-tool/main.py b/33_awscli/boto-tool/main.py index a752a25..d276c6d 100644 --- a/33_awscli/boto-tool/main.py +++ b/33_awscli/boto-tool/main.py @@ -25,7 +25,7 @@ def str2bool(v): return v.lower() in ("yes", "true", "t", "1") -def signedupload(bucket_name: str, s3_key: str, expires: int): +def signed_upload_put(bucket_name: str, s3_key: str, expires: int): logger = logging.getLogger() # Replace the following with your own values @@ -49,6 +49,41 @@ def signedupload(bucket_name: str, s3_key: str, expires: int): logger.info({ "message": "presigned", "url": f"{presigned_url}"}) +def signed_upload_post(bucket_name: str, s3_key: str, expires: int): + logger = logging.getLogger() + + # Replace the following with your own values + profile_name = os.environ['AWS_PROFILE'] + + # Create a session with the specified profile + session = Session(profile_name=profile_name) + + # Create an S3 client using the session + s3 = session.client('s3') + + presigned_url = s3.generate_presigned_post( + Bucket=bucket_name, + Key=s3_key, + Fields= { + }, + Conditions=[ + #["starts-with", '$content-type', ""], + #['content-length-range', 10000, 150000], + #["starts-with", '$x-amz-checksum-sha256', ""], + ], + ExpiresIn=expires, + ) + + logger.info({ "message": "presigned", "url": f"{presigned_url}"}) + + presigned_url['fields']['file'] = '@{key}'.format(key=s3_key) + + form_values = "\n ".join(["-F {key}={value} \\".format(key=key, value=value) + for key, value in presigned_url['fields'].items()]) + + curl = 'curl -v {form_values} \n {url}'.format(form_values=form_values, url=presigned_url['url']) + logger.info({ "message": "presigned", "curl": f"{curl}"}) + print(curl) def main(): with io.open( @@ -63,6 +98,8 @@ def main(): parser = argparse.ArgumentParser(description="AWS BOTO") parser.add_argument("--signed", dest="signed", action="store_true") + parser.add_argument("--put", dest="put", action="store_true") + parser.add_argument("--post", dest="post", action="store_true") parser.add_argument("--bucket", dest="bucket", type=str) parser.add_argument("--prefix", dest="prefix", type=str) parser.add_argument("--expires", dest="expires", type=int, default=3600) @@ -70,7 +107,12 @@ def main(): if args.signed: logger.info(f"Upload s3://{args.bucket}/{args.prefix}") - signedupload(args.bucket, args.prefix, args.expires) + if args.put: + signed_upload_put(args.bucket, args.prefix, args.expires) + elif args.post: + signed_upload_post(args.bucket, args.prefix, args.expires) + else: + print("Need to specify --put or --post") else: parser.print_help() From b212ba6139e550767c5dcac13046cef78dc4e1a3 Mon Sep 17 00:00:00 2001 From: Chris Guest Date: Mon, 24 Apr 2023 12:22:18 +0100 Subject: [PATCH 2/2] Updated --- 33_awscli/S3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/33_awscli/S3.md b/33_awscli/S3.md index aca59af..9d6e2b7 100644 --- a/33_awscli/S3.md +++ b/33_awscli/S3.md @@ -161,7 +161,7 @@ sha256sum ../out/random.bin | tr " " "\n" | head -n 1 | base64 -AWS_PROFILE=trint-toys aws s3 rm s3://${BUCKET_NAME}/random.bin +AWS_PROFILE=myprofile aws s3 rm s3://${BUCKET_NAME}/random.bin ./presigned_test.sh