Make allow-directory-listing and provide-index-page compatible (#139)

Dockerfile.unprivileged

@@ -8,6 +8,7 @@ FROM nginx-s3-gateway
 # Implement changes required to run NGINX as an unprivileged user
 RUN sed -i "/^server {/a \    listen       8080;" /etc/nginx/templates/default.conf.template \
     && sed -i '/user  nginx;/d' /etc/nginx/nginx.conf \
+    && sed -i 's#http://127.0.0.1:80#http://127.0.0.1:8080#g' /etc/nginx/include/s3gateway.js \
     && sed -i 's,/var/run/nginx.pid,/tmp/nginx.pid,' /etc/nginx/nginx.conf \
     && sed -i "/^http {/a \    proxy_temp_path /tmp/proxy_temp;\n    client_body_temp_path /tmp/client_temp;\n    fastcgi_temp_path /tmp/fastcgi_temp;\n    uwsgi_temp_path /tmp/uwsgi_temp;\n    scgi_temp_path /tmp/scgi_temp;\n" /etc/nginx/nginx.conf \
 # Nginx user must own the cache and etc directory to write cache and tweak the nginx config

common/docker-entrypoint.d/00-check-for-required-env.sh

@@ -109,11 +109,6 @@ parseBoolean() {
   esac
 }
 
-if [ "$(parseBoolean ${ALLOW_DIRECTORY_LIST})" == "1" ] && [ "$(parseBoolean ${PROVIDE_INDEX_PAGE})" == "1" ]; then
-  >&2 echo "ALLOW_DIRECTORY_LIST and PROVIDE_INDEX_PAGE cannot be both set"
-  failed=1
-fi
-
 if [ -n "${HEADER_PREFIXES_TO_STRIP+x}" ]; then
   if [[ "${HEADER_PREFIXES_TO_STRIP}" =~ [A-Z] ]]; then
     >&2 echo "HEADER_PREFIXES_TO_STRIP must not contain uppercase characters"

common/etc/nginx/include/s3gateway.js

@@ -170,7 +170,7 @@ function _s3ReqParamsForSigV2(r, bucket) {
      * Thus, we can't put the path /dir1/ in the string to sign. */
     let uri = _isDirectory(r.variables.uri_path) ? '/' : r.variables.uri_path;
     // To return index pages + index.html
-    if (PROVIDE_INDEX_PAGE && _isDirectory(r.variables.uri_path)){
+    if (utils.parseBoolean(r.variables.forIndexPage) && _isDirectory(r.variables.uri_path)){
         uri = r.variables.uri_path + INDEX_PAGE
     }
 
@@ -196,7 +196,10 @@ function _s3ReqParamsForSigV4(r, bucket, server) {
         host = bucket + '.' + host;
     }
     const baseUri = s3BaseUri(r);
-    const queryParams = _s3DirQueryParams(r.variables.uri_path, r.method);
+    const computed_url = !utils.parseBoolean(r.variables.forIndexPage)
+        ? r.variables.uri_path
+        : r.variables.uri_path + INDEX_PAGE;
+    const queryParams = _s3DirQueryParams(computed_url, r.method);
     let uri;
     if (queryParams.length > 0) {
         if (baseUri.length > 0) {
@@ -248,7 +251,7 @@ function s3uri(r) {
     let path;
 
     // Create query parameters only if directory listing is enabled.
-    if (ALLOW_LISTING) {
+    if (ALLOW_LISTING && !utils.parseBoolean(r.variables.forIndexPage)) {
         const queryParams = _s3DirQueryParams(uriPath, r.method);
         if (queryParams.length > 0) {
             path = basePath + '?' + queryParams;
@@ -283,7 +286,7 @@ function _s3DirQueryParams(uriPath, method) {
 
     /* Return if static website. We don't want to list the files in the
        directory, we want to append the index page and get the fil. */
-    if (PROVIDE_INDEX_PAGE){
+    if (uriPath.endsWith(INDEX_PAGE)){
         return '';
     }
 
@@ -317,8 +320,8 @@ function redirectToS3(r) {
     const uriPath = r.variables.uri_path;
     const isDirectoryListing = ALLOW_LISTING && _isDirectory(uriPath);
 
-    if (isDirectoryListing && r.method === 'GET') {
-        r.internalRedirect("@s3Listing");
+    if (isDirectoryListing && (r.method === 'GET' || r.method === 'HEAD')) {
+        r.internalRedirect("@s3PreListing");
     } else if ( PROVIDE_INDEX_PAGE == true ) {
         r.internalRedirect("@s3");
     } else if ( !ALLOW_LISTING && !PROVIDE_INDEX_PAGE && uriPath == "/" ) {
@@ -338,6 +341,36 @@ function trailslashControl(r) {
         r.internalRedirect("@error404");
 }
 
+/**
+ * Checks if there is an index.html file in the directory.
+ * Redirects appropriately. Before that, it checks if
+ * directory listing is enforced or not.
+ *
+ * @param {Object} r - The HTTP request object.
+ */
+async function loadContent(r) {
+    if (!PROVIDE_INDEX_PAGE) {
+        r.internalRedirect("@s3Directory");
+        return;
+    }
+    const url = s3uri(r);
+    let reply = await ngx.fetch(
+        `http://127.0.0.1:80${url}`
+    );
+
+    if (reply.status == 200) {
+        // found index.html, so redirect to it
+        r.internalRedirect(r.variables.request_uri + INDEX_PAGE);
+    } else if (reply.status == 404) {
+        // else just list the contents of the directory
+        r.internalRedirect("@s3Directory");
+    } else {
+        r.internalRedirect("@error500");
+    }
+
+    return;
+}
+
 /**
  * Processes the directory listing output as returned from S3. If
  * FOUR_O_FOUR_ON_EMPTY_BUCKET is enabled, this function will corrupt the
@@ -450,6 +483,7 @@ export default {
     redirectToS3,
     editHeaders,
     filterListResponse,
+    loadContent,
     // These functions do not need to be exposed, but they are exposed so that
     // unit tests can run against them.
     _s3ReqParamsForSigV2,

common/etc/nginx/templates/default.conf.template

@@ -20,6 +20,7 @@ map $S3_STYLE $s3_host_hdr {
 }
 
 js_var $indexIsEmpty true;
+js_var $forIndexPage true;
 # This creates the HTTP authentication header to be sent to S3
 js_set $s3auth s3gateway.s3auth;
 js_set $awsSessionToken awscredentials.sessionToken;
@@ -82,7 +83,7 @@ server {
         auth_request /aws/credentials/retrieve;
 
         # Redirect to the proper location based on the client request - either
-        # @s3, @s3Listing or @error405.
+        # @s3, @s3PreListing or @error405.
 
         js_content s3gateway.redirectToS3;
     }
@@ -141,7 +142,7 @@ server {
         include /etc/nginx/conf.d/gateway/s3_location.conf;
     }
 
-    location @s3Listing {
+    location @s3PreListing {
         # We include only the headers needed for the authentication signatures that
         # we plan to use.
         include /etc/nginx/conf.d/gateway/v${AWS_SIGS_VERSION}_headers.conf;
@@ -195,7 +196,116 @@ server {
         # Comment out this line to receive the error messages returned by S3
         error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404;
 
-        proxy_pass ${S3_SERVER_PROTO}://storage_urls$s3uri;
+        js_content s3gateway.loadContent;
+        include /etc/nginx/conf.d/gateway/s3listing_location.conf;
+    }
+
+    location @s3Directory {
+        # We include only the headers needed for the authentication signatures that
+        # we plan to use.
+        include /etc/nginx/conf.d/gateway/v${AWS_SIGS_VERSION}_headers.conf;
+
+        # Necessary for determining the correct URI to construct.
+        set $forIndexPage false;
+
+        # The CORS configuration needs to be imported in several places in order for
+        # it to be applied within different contexts.
+        include /etc/nginx/conf.d/gateway/cors.conf;
+
+        # Don't allow any headers from the client - we don't want them messing
+        # with S3 at all.
+        proxy_pass_request_headers off;
+
+        # Enable passing of the server name through TLS Server Name Indication extension.
+        proxy_ssl_server_name on;
+        proxy_ssl_name ${S3_SERVER};
+
+        # Set the Authorization header to the AWS Signatures credentials
+        proxy_set_header Authorization $s3auth;
+        proxy_set_header X-Amz-Security-Token $awsSessionToken;
+
+        # We set the host as the bucket name to inform the S3 API of the bucket
+        proxy_set_header Host $s3_host_hdr;
+
+        # Use keep alive connections in order to improve performance
+        proxy_http_version 1.1;
+        proxy_set_header Connection '';
+
+        # We strip off all of the AWS specific headers from the server so that
+        # there is nothing identifying the object as having originated in an
+        # object store.
+        js_header_filter s3gateway.editHeaders;
+
+        # Apply XSL transformation to the XML returned from S3 directory listing
+        # results such that we can output an HTML directory contents list.
+        xslt_stylesheet /etc/nginx/include/listing.xsl;
+        xslt_types application/xml;
+
+        # We apply an output filter to the XML input received from S3 before it
+        # is passed to XSLT in order to determine if the resource is not a valid
+        # S3 directory. If it isn't a valid directory, we do a dirty hack to
+        # corrupt the contents of the XML causing the XSLT to fail and thus
+        # nginx to return a 404 to the client. If you don't care about empty
+        # directory listings for invalid directories, remove this.
+        js_body_filter s3gateway.filterListResponse;
+
+        # Catch all errors from S3 and sanitize them so that the user can't
+        # gain intelligence about the S3 bucket being proxied.
+        proxy_intercept_errors on;
+
+        # Comment out this line to receive the error messages returned by S3
+        error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404;
+
+        proxy_pass  ${S3_SERVER_PROTO}://storage_urls$s3Uri;
+        include /etc/nginx/conf.d/gateway/s3listing_location.conf;
+    }
+
+    location ~ /index.html$ {
+        # Configuration for handling locations ending with /index.html
+
+        # Necessary for determining the correct URI to construct.
+        set $forIndexPage true;
+
+        # We include only the headers needed for the authentication signatures that
+        # we plan to use.
+        include /etc/nginx/conf.d/gateway/v${AWS_SIGS_VERSION}_headers.conf;
+
+        # The CORS configuration needs to be imported in several places in order for
+        # it to be applied within different contexts.
+        include /etc/nginx/conf.d/gateway/cors.conf;
+
+        # Don't allow any headers from the client - we don't want them messing
+        # with S3 at all.
+        proxy_pass_request_headers off;
+
+        # Enable passing of the server name through TLS Server Name Indication extension.
+        proxy_ssl_server_name on;
+        proxy_ssl_name ${S3_SERVER};
+
+        # Set the Authorization header to the AWS Signatures credentials
+        proxy_set_header Authorization $s3auth;
+        proxy_set_header X-Amz-Security-Token $awsSessionToken;
+
+        # We set the host as the bucket name to inform the S3 API of the bucket
+        proxy_set_header Host $s3_host_hdr;
+
+        # Use keep alive connections in order to improve performance
+        proxy_http_version 1.1;
+        proxy_set_header Connection '';
+
+        # We strip off all of the AWS specific headers from the server so that
+        # there is nothing identifying the object as having originated in an
+        # object store.
+        js_header_filter s3gateway.editHeaders;
+
+        # Catch all errors from S3 and sanitize them so that the user can't
+        # gain intelligence about the S3 bucket being proxied.
+        proxy_intercept_errors on;
+
+        # Comment out this line to receive the error messages returned by S3
+        error_page 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 420 422 423 424 426 428 429 431 444 449 450 451 500 501 502 503 504 505 506 507 508 509 510 511 =404 @error404;
+
+        proxy_pass  ${S3_SERVER_PROTO}://storage_urls$s3uri;
         include /etc/nginx/conf.d/gateway/s3listing_location.conf;
     }