From bf3cae907b2a0650d7583f459149a611b85730de Mon Sep 17 00:00:00 2001 From: Josh Watt Date: Tue, 1 Aug 2023 00:29:42 -0400 Subject: [PATCH 1/9] Add new var STRIP_LEADING_DIRECTORY_PATH. --- Dockerfile.oss | 1 + Dockerfile.plus | 1 + common/etc/nginx/nginx.conf | 1 + standalone_ubuntu_oss_install.sh | 2 ++ 4 files changed, 5 insertions(+) diff --git a/Dockerfile.oss b/Dockerfile.oss index f986474e..838bda85 100644 --- a/Dockerfile.oss +++ b/Dockerfile.oss @@ -10,6 +10,7 @@ ENV PROXY_CACHE_VALID_NOTFOUND "1m" ENV PROXY_CACHE_VALID_FORBIDDEN "30s" ENV CORS_ENABLED 0 ENV DIRECTORY_LISTING_PATH_PREFIX "" +ENV STRIP_LEADING_DIRECTORY_PATH "" # We modify the nginx base image by: # 1. Adding configuration files needed for proxying private S3 buckets diff --git a/Dockerfile.plus b/Dockerfile.plus index 47252b5e..6183ed00 100644 --- a/Dockerfile.plus +++ b/Dockerfile.plus @@ -12,6 +12,7 @@ ENV PROXY_CACHE_VALID_NOTFOUND "1m" ENV PROXY_CACHE_VALID_FORBIDDEN "30s" ENV CORS_ENABLED 0 ENV DIRECTORY_LISTING_PATH_PREFIX "" +ENV STRIP_LEADING_DIRECTORY_PATH "" COPY plus/etc/ssl /etc/ssl COPY plus/usr /usr diff --git a/common/etc/nginx/nginx.conf b/common/etc/nginx/nginx.conf index 43914f5f..75857f70 100644 --- a/common/etc/nginx/nginx.conf +++ b/common/etc/nginx/nginx.conf @@ -31,6 +31,7 @@ env PROXY_CACHE_VALID_NOTFOUND; env PROXY_CACHE_VALID_FORBIDDEN; env HEADER_PREFIXES_TO_STRIP; env FOUR_O_FOUR_ON_EMPTY_BUCKET; +env STRIP_LEADING_DIRECTORY_PATH; events { worker_connections 1024; diff --git a/standalone_ubuntu_oss_install.sh b/standalone_ubuntu_oss_install.sh index eed6725a..774d4981 100644 --- a/standalone_ubuntu_oss_install.sh +++ b/standalone_ubuntu_oss_install.sh @@ -171,6 +171,8 @@ PROXY_CACHE_VALID_NOTFOUND=${PROXY_CACHE_VALID_NOTFOUND:-'1m'} PROXY_CACHE_VALID_FORBIDDEN=${PROXY_CACHE_VALID_FORBIDDEN:-'30s'} # Enables or disables CORS for the S3 Gateway (true=enabled, false=disabled) CORS_ENABLED=${CORS_ENABLED:-'false'} +# Configure portion of URL to be removed (optional) +STRIP_LEADING_DIRECTORY_PATH=${STRIP_LEADING_DIRECTORY_PATH:-''} EOF # By enabling CORS, we also need to enable the OPTIONS method which From 4ec308b62b699091c75fc038cbc38bd3a2eb7e6f Mon Sep 17 00:00:00 2001 From: Josh Watt Date: Tue, 1 Aug 2023 00:31:00 -0400 Subject: [PATCH 2/9] Add STRIP_LEADING_DIRECTORY_PATH map function to config. --- common/etc/nginx/templates/default.conf.template | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/common/etc/nginx/templates/default.conf.template b/common/etc/nginx/templates/default.conf.template index 7873a190..6255d110 100644 --- a/common/etc/nginx/templates/default.conf.template +++ b/common/etc/nginx/templates/default.conf.template @@ -9,10 +9,15 @@ include /etc/nginx/conf.d/gateway/v${AWS_SIGS_VERSION}_js_vars.conf; # Extracts only the path from the requested URI. This strips out all query # parameters and anchors in order to prevent extraneous data from being sent # to S3. -map $request_uri $uri_path { +map $request_uri $uri_full_path { "~^(?P.*?)(\?.*)*$" $path; } +# Remove a portion of request URL (if configured) +map $uri_full_path $uri_path { + "~^$STRIP_LEADING_DIRECTORY_PATH(.*)" $1; +} + map $S3_STYLE $s3_host_hdr { virtual "${S3_BUCKET_NAME}.${S3_SERVER}"; path "${S3_SERVER}:${S3_SERVER_PORT}"; From 598ff019d47d12fa6cd5c7bc219d320bbbd60875 Mon Sep 17 00:00:00 2001 From: Josh Watt Date: Tue, 1 Aug 2023 00:43:21 -0400 Subject: [PATCH 3/9] Add new var to example settings and getting_started. --- docs/getting_started.md | 4 ++-- settings.example | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/getting_started.md b/docs/getting_started.md index 8bfce615..1a3e1113 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -39,8 +39,8 @@ running as a Container or as a Systemd service. | `JS_TRUSTED_CERT_PATH` | No | | | Enables the `js_fetch_trusted_certificate` directive when retrieving AWS credentials and sets the path (on the container) to the specified path | | `HEADER_PREFIXES_TO_STRIP` | No | | | A list of HTTP header prefixes that exclude headers client responses. List should be specified in lower-case and a semicolon (;) should be used to as a deliminator between values. For example: `x-goog-;x-something-` | | `CORS_ENABLED` | No | `true`, `false` | `false` | Flag that enables CORS headers on GET requests and enables pre-flight OPTIONS requests. If enabled, this will add CORS headers for "fully open" cross domain requests by default, meaning all domains are allowed, similar to the settings show in [this example](https://enable-cors.org/server_nginx.html). CORS settings can be fine-tuned by overwriting the [`cors.conf.template`](/common/etc/nginx/templates/gateway/cors.conf.template) file. | -| `CORS_ALLOWED_ORIGIN` | No | | | value to set to be returned from the CORS `Access-Control-Allow-Origin` header. This value is only used if CORS is enabled. (default: \*) | - +| `CORS_ALLOWED_ORIGIN` | No | | | value to set to be returned from the CORS `Access-Control-Allow-Origin` header. This value is only used if CORS is enabled. (default: \*) +| `STRIP_LEADING_DIRECTORY_PATH` | No | | | Removes a portion of the path in the requested URL (if configured). Useful when deploying to an ALB under a folder (eg. www.mysite.com/mybucket). If you are using [AWS instance profile credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html), you will need to omit the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` variables from diff --git a/settings.example b/settings.example index 9b619deb..b87bc5d9 100644 --- a/settings.example +++ b/settings.example @@ -18,3 +18,4 @@ PROXY_CACHE_INACTIVE=60m PROXY_CACHE_VALID_OK=1h PROXY_CACHE_VALID_NOTFOUND=1m PROXY_CACHE_VALID_FORBIDDEN=30s +STRIP_LEADING_DIRECTORY_PATH=/somepath \ No newline at end of file From 21bf1813bfb61a51f28f2f117331c6d95809e964 Mon Sep 17 00:00:00 2001 From: Josh Watt Date: Tue, 1 Aug 2023 00:43:54 -0400 Subject: [PATCH 4/9] Fix broken S3 hostname in example settings. --- settings.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.example b/settings.example index b87bc5d9..832eaaa5 100644 --- a/settings.example +++ b/settings.example @@ -2,7 +2,7 @@ S3_BUCKET_NAME=my-bucket AWS_ACCESS_KEY_ID=ZZZZZZZZZZZZZZZZZZZZ AWS_SECRET_ACCESS_KEY=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa AWS_SESSION_TOKEN=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb -S3_SERVER=s3-us-east-1.amazonaws.com +S3_SERVER=s3.us-east-1.amazonaws.com S3_SERVER_PORT=443 S3_SERVER_PROTO=https S3_REGION=us-east-1 From 4f6c259a6143d885839bc6306edcb79228e34068 Mon Sep 17 00:00:00 2001 From: Javier Evans Date: Wed, 25 Oct 2023 16:22:57 -0700 Subject: [PATCH 5/9] add tests for STRIP_LEADING_DIRECTORY_PATH --- test.sh | 27 +++++++++++++++++---------- test/docker-compose.yaml | 1 + test/integration/test_api.sh | 5 +++++ 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/test.sh b/test.sh index d2be720e..30f5cc3d 100755 --- a/test.sh +++ b/test.sh @@ -247,10 +247,12 @@ integration_test() { printf "\e[1m Integration test suite with PROVIDE_INDEX_PAGE=%s\e[22m\n" "$3" printf "\033[34;1m▶\033[0m" printf "\e[1m Integration test suite with APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=%s\e[22m\n" "$4" + printf "\033[34;1m▶\033[0m" + printf "\e[1m Integration test suite with STRIP_LEADING_DIRECTORY_PATH=%s\e[22m\n" "$5" p "Starting Docker Compose Environment" # COMPOSE_COMPATIBILITY=true Supports older style compose filenames with _ vs - - COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 compose up -d + COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 STRIP_LEADING_DIRECTORY_PATH=$5 compose up -d if [ "${wait_for_it_installed}" ]; then if [ -x "${wait_for_it_cmd}" ]; then @@ -259,8 +261,8 @@ integration_test() { fi p "Starting HTTP API tests (v$1 signatures)" - echo " test/integration/test_api.sh \"$test_server\" \"$test_dir\" $1 $2 $3 $4" - bash "${test_dir}/integration/test_api.sh" "${test_server}" "${test_dir}" "$1" "$2" "$3" "$4"; + echo " test/integration/test_api.sh \"$test_server\" \"$test_dir\" $1 $2 $3 $4 $5" + bash "${test_dir}/integration/test_api.sh" "${test_server}" "${test_dir}" "$1" "$2" "$3" "$4" "$5"; # We check to see if NGINX is in fact using the correct version of AWS # signatures as it was configured to do. @@ -402,36 +404,41 @@ runUnitTestWithSessionToken "s3gateway_test.js" integration_test_data p "Testing API with AWS Signature V2 and allow directory listing off" -integration_test 2 0 0 0 +integration_test 2 0 0 0 "" compose stop nginx-s3-gateway # Restart with new config p "Testing API with AWS Signature V2 and allow directory listing on" -integration_test 2 1 0 0 +integration_test 2 1 0 0 "" compose stop nginx-s3-gateway # Restart with new config p "Testing API with AWS Signature V2 and static site on" -integration_test 2 0 1 0 +integration_test 2 0 1 0 "" compose stop nginx-s3-gateway # Restart with new config p "Testing API with AWS Signature V2 and allow directory listing on and append slash and allow index" -integration_test 2 1 1 1 +integration_test 2 1 1 1 "" compose stop nginx-s3-gateway # Restart with new config p "Test API with AWS Signature V4 and allow directory listing off" -integration_test 4 0 0 0 +integration_test 4 0 0 0 "" compose stop nginx-s3-gateway # Restart with new config p "Test API with AWS Signature V4 and allow directory listing on and appending /" -integration_test 4 1 0 1 +integration_test 4 1 0 1 "" compose stop nginx-s3-gateway # Restart with new config p "Test API with AWS Signature V4 and static site on appending /" -integration_test 4 0 1 1 +integration_test 4 0 1 1 "" + +compose stop nginx-s3-gateway # Restart with new config + +p "Testing API with AWS Signature V2 and allow directory listing off and prefix stripping on" +integration_test 2 0 0 0 /my-bucket p "All integration tests complete" diff --git a/test/docker-compose.yaml b/test/docker-compose.yaml index df54b0d8..0a162f14 100644 --- a/test/docker-compose.yaml +++ b/test/docker-compose.yaml @@ -26,6 +26,7 @@ services: ALLOW_DIRECTORY_LIST: PROVIDE_INDEX_PAGE: APPEND_SLASH_FOR_POSSIBLE_DIRECTORY: + STRIP_LEADING_DIRECTORY_PATH: AWS_SIGS_VERSION: STATIC_SITE_HOSTING: PROXY_CACHE_MAX_SIZE: "10g" diff --git a/test/integration/test_api.sh b/test/integration/test_api.sh index 5153233c..26dd6fbd 100644 --- a/test/integration/test_api.sh +++ b/test/integration/test_api.sh @@ -25,6 +25,7 @@ signature_version=$3 allow_directory_list=$4 index_page=$5 append_slash=$6 +strip_leading_directory=$7 test_fail_exit_code=2 no_dep_exit_code=3 checksum_length=32 @@ -268,6 +269,10 @@ assertHttpRequestEquals "GET" "b/c/'(1).txt" "data/bucket-1/b/c/'(1).txt" assertHttpRequestEquals "GET" "b/e.txt" "data/bucket-1/b/e.txt" +if [ -n "${strip_leading_directory}" ]; then + assertHttpRequestEquals "GET" "/my-bucket/a.txt" "data/bucket-1/a.txt" +fi + # These URLs do not work unencoded assertHttpRequestEquals "GET" 'a/plus%2Bplus.txt' "data/bucket-1/a/plus+plus.txt" From 6f76f768dc07502c81836357abaf11b57006ca43 Mon Sep 17 00:00:00 2001 From: Javier Evans Date: Wed, 25 Oct 2023 16:23:38 -0700 Subject: [PATCH 6/9] handle when path doesn't contain a substring matching the STRIP_LEADING_DIRECTORY_PATH configuration --- common/etc/nginx/templates/default.conf.template | 1 + 1 file changed, 1 insertion(+) diff --git a/common/etc/nginx/templates/default.conf.template b/common/etc/nginx/templates/default.conf.template index 6255d110..e68413d7 100644 --- a/common/etc/nginx/templates/default.conf.template +++ b/common/etc/nginx/templates/default.conf.template @@ -16,6 +16,7 @@ map $request_uri $uri_full_path { # Remove a portion of request URL (if configured) map $uri_full_path $uri_path { "~^$STRIP_LEADING_DIRECTORY_PATH(.*)" $1; + default $uri_full_path; } map $S3_STYLE $s3_host_hdr { From 6176bc26051f3f8a8be171ea384277c716b71965 Mon Sep 17 00:00:00 2001 From: Javier Evans Date: Wed, 25 Oct 2023 17:18:08 -0700 Subject: [PATCH 7/9] Empty-Commit From af6775de55055959611be27a4602953f2c840dac Mon Sep 17 00:00:00 2001 From: Javier Evans Date: Wed, 25 Oct 2023 17:21:17 -0700 Subject: [PATCH 8/9] fix typo in comment --- common/docker-entrypoint.d/00-check-for-required-env.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/docker-entrypoint.d/00-check-for-required-env.sh b/common/docker-entrypoint.d/00-check-for-required-env.sh index 4fd3776a..0ed18cb2 100755 --- a/common/docker-entrypoint.d/00-check-for-required-env.sh +++ b/common/docker-entrypoint.d/00-check-for-required-env.sh @@ -35,7 +35,7 @@ if [[ -v AWS_CONTAINER_CREDENTIALS_RELATIVE_URI ]]; then echo "Running inside an ECS task, using container credentials" elif [[ -v S3_SESSION_TOKEN ]]; then - echo "Depreciated the S3_SESSION_TOKEN! Use the environment variable of AWS_SESSION_TOKEN instead" + echo "Deprecated the S3_SESSION_TOKEN! Use the environment variable of AWS_SESSION_TOKEN instead" failed=1 elif [[ -v AWS_SESSION_TOKEN ]]; then @@ -58,11 +58,11 @@ elif [[ -v AWS_WEB_IDENTITY_TOKEN_FILE ]]; then fi elif [[ -v S3_ACCESS_KEY_ID ]]; then - echo "Depreciated the S3_ACCESS_KEY_ID! Use the environment variable of AWS_ACCESS_KEY_ID instead" + echo "Deprecated the S3_ACCESS_KEY_ID! Use the environment variable of AWS_ACCESS_KEY_ID instead" failed=1 elif [[ -v S3_SECRET_KEY ]]; then - echo "Depreciated the S3_SECRET_KEY! Use the environment variable of AWS_SECRET_ACCESS_KEY instead" + echo "Deprecated the S3_SECRET_KEY! Use the environment variable of AWS_SECRET_ACCESS_KEY instead" failed=1 elif [[ -v AWS_SECRET_KEY ]]; then @@ -77,7 +77,7 @@ else fi if [[ -v S3_DEBUG ]]; then - echo "Depreciated the S3_DEBUG! Use the environment variable of DEBUG instead" + echo "Deprecated the S3_DEBUG! Use the environment variable of DEBUG instead" failed=1 fi From 860ce43c259daa7ef29eaa49aac9fb4ad327a571 Mon Sep 17 00:00:00 2001 From: Josh Watt Date: Wed, 1 Nov 2023 17:44:15 -0400 Subject: [PATCH 9/9] Update getting_started doc for STRIP_LEADING_DIRECTORY_PATH var. --- docs/getting_started.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/getting_started.md b/docs/getting_started.md index 1a3e1113..5e24732f 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -40,7 +40,7 @@ running as a Container or as a Systemd service. | `HEADER_PREFIXES_TO_STRIP` | No | | | A list of HTTP header prefixes that exclude headers client responses. List should be specified in lower-case and a semicolon (;) should be used to as a deliminator between values. For example: `x-goog-;x-something-` | | `CORS_ENABLED` | No | `true`, `false` | `false` | Flag that enables CORS headers on GET requests and enables pre-flight OPTIONS requests. If enabled, this will add CORS headers for "fully open" cross domain requests by default, meaning all domains are allowed, similar to the settings show in [this example](https://enable-cors.org/server_nginx.html). CORS settings can be fine-tuned by overwriting the [`cors.conf.template`](/common/etc/nginx/templates/gateway/cors.conf.template) file. | | `CORS_ALLOWED_ORIGIN` | No | | | value to set to be returned from the CORS `Access-Control-Allow-Origin` header. This value is only used if CORS is enabled. (default: \*) -| `STRIP_LEADING_DIRECTORY_PATH` | No | | | Removes a portion of the path in the requested URL (if configured). Useful when deploying to an ALB under a folder (eg. www.mysite.com/mybucket). +| `STRIP_LEADING_DIRECTORY_PATH` | No | | | Removes a portion of the path in the requested URL (if configured). Useful when deploying to an ALB under a folder (eg. www.mysite.com/somepath). If you are using [AWS instance profile credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html), you will need to omit the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` variables from @@ -104,6 +104,13 @@ It will also redirect `/some/path` to `/some/path/` when S3 returns 404 on look like a possible directory, it must not start with a `.` and not have an extension. +### Hosting a Bucket as a Subfolder on an ALB + +The `STRIP_LEADING_DIRECTORY_PATH` environment variable allows one to host an +S3 bucket in a subfolder on an ALB. For example, if you wanted to expose the +root of a bucket under the path "www.mysite.com/somepath", you would set this +variable to "/somepath". + ## Running as a Systemd Service An [install script](/standalone_ubuntu_oss_install.sh) for the gateway shows