Merge branch 'main' into user/ashdhin/IssueTemplateRegressionCheckbox

awslabs · Oct 25, 2024 · 9b53752 · 9b53752
2 parents 53e8533 + 8c1969b
commit 9b53752
Show file tree

Hide file tree

Showing 23 changed files with 799 additions and 333 deletions.
diff --git a/.gitignore b/.gitignore
@@ -68,3 +68,6 @@ cmake-build*
 # js package locks irrelevant to the overall package's security
 benchmarks/benchmarks-stack/benchmarks-stack/package-lock.json
 benchmarks/dashboard-stack/package-lock.json
+
+# virtual environment
+.venv/
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,11 +1,7 @@
 
-cmake_minimum_required(VERSION 3.1)
+cmake_minimum_required(VERSION 3.9)
 project(aws-c-s3 C)
 
-if (POLICY CMP0069)
-    cmake_policy(SET CMP0069 NEW) # Enable LTO/IPO if available in the compiler, see AwsCFlags
-endif()
-
 option(ASSERT_LOCK_HELD "Enable ASSERT_SYNCED_DATA_LOCK_HELD for checking thread issue" OFF)
 option(ENABLE_MOCK_SERVER_TESTS "Whether to run the integration tests that rely on pre-configured mock server" OFF)
 option(ENABLE_MRAP_TESTS "Whether to run the integration tests that rely on pre-configured multi-region access point" OFF)

diff --git a/README.md b/README.md
@@ -23,7 +23,7 @@ This library is licensed under the Apache 2.0 License.
 
 ### Building
 
-CMake 3.1+ is required to build.
+CMake 3.9+ is required to build.
 
 `<install-path>` must be an absolute path in the following instructions.
 

diff --git a/include/aws/s3/private/s3_meta_request_impl.h b/include/aws/s3/private/s3_meta_request_impl.h
@@ -350,6 +350,14 @@ void aws_s3_meta_request_init_signing_date_time_default(
     struct aws_s3_meta_request *meta_request,
     struct aws_date_time *date_time);
 
+AWS_S3_API
+void aws_s3_meta_request_sign_request_default_impl(
+    struct aws_s3_meta_request *meta_request,
+    struct aws_s3_request *request,
+    aws_signing_complete_fn *on_signing_complete,
+    void *user_data,
+    bool disable_s3_express_signing);
+
 AWS_S3_API
 void aws_s3_meta_request_sign_request_default(
     struct aws_s3_meta_request *meta_request,

diff --git a/include/aws/s3/s3.h b/include/aws/s3/s3.h
@@ -48,6 +48,7 @@ enum aws_s3_errors {
     AWS_ERROR_S3_REQUEST_HAS_COMPLETED,
     AWS_ERROR_S3_RECV_FILE_ALREADY_EXISTS,
     AWS_ERROR_S3_RECV_FILE_NOT_FOUND,
+    AWS_ERROR_S3_REQUEST_TIMEOUT,
 
     AWS_ERROR_S3_END_RANGE = AWS_ERROR_ENUM_END_RANGE(AWS_C_S3_PACKAGE_ID)
 };

diff --git a/include/aws/s3/s3_client.h b/include/aws/s3/s3_client.h
@@ -82,6 +82,7 @@ enum aws_s3_meta_request_type {
      * - only {bucket}/{key} format is supported for source and passing arn as
      *   source will not work
      * - source bucket is assumed to be in the same region as dest
+     * - source bucket and dest bucket must both be either directory buckets or regular buckets.
      */
     AWS_S3_META_REQUEST_TYPE_COPY_OBJECT,
 
@@ -558,19 +559,19 @@ struct aws_s3_checksum_config {
     /**
      * The location of client added checksum header.
      *
-     * If AWS_SCL_NONE. No request payload checksum will be add and calculated.
+     * If AWS_SCL_NONE. No request payload checksum will be calculated or added.
      *
-     * If AWS_SCL_HEADER, the checksum will be calculated by client and added related header to the request sent.
+     * If AWS_SCL_HEADER, the client will calculate the checksum and add it to the headers.
      *
-     * If AWS_SCL_TRAILER, the payload will be aws_chunked encoded, The checksum will be calculate while reading the
-     * payload by client. Related header will be added to the trailer part of the encoded payload. Note the payload of
-     * the original request cannot be aws-chunked encoded already. Otherwise, error will be raised.
+     * If AWS_SCL_TRAILER, the payload will be aws_chunked encoded, The client will calculate the checksum and add it to
+     * the trailer. Note the payload of the original request cannot be aws-chunked encoded already, this will cause an
+     * error.
      */
     enum aws_s3_checksum_location location;
 
     /**
      * The checksum algorithm used.
-     * Must be set if location is not AWS_SCL_NONE. Must be AWS_SCA_NONE if location is AWS_SCL_NONE.
+     * Must be set if location is not AWS_SCL_NONE.
      */
     enum aws_s3_checksum_algorithm checksum_algorithm;
 

diff --git a/source/s3.c b/source/s3.c
@@ -51,6 +51,7 @@ static struct aws_error_info s_errors[] = {
     AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3_REQUEST_HAS_COMPLETED, "Request has already completed, action cannot be performed."),
     AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3_RECV_FILE_ALREADY_EXISTS, "File already exists, cannot create as new."),
     AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3_RECV_FILE_NOT_FOUND, "The receive file doesn't exist, cannot create as configuration required."),
+    AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3_REQUEST_TIMEOUT, "RequestTimeout error received from S3."),
 };
 /* clang-format on */
 

diff --git a/source/s3_buffer_pool.c b/source/s3_buffer_pool.c
@@ -232,6 +232,7 @@ void s_buffer_pool_trim_synced(struct aws_s3_buffer_pool *buffer_pool) {
         aws_array_list_get_at_ptr(&buffer_pool->blocks, (void **)&block, i);
 
         if (block->alloc_bit_mask == 0) {
+            buffer_pool->primary_allocated -= block->block_size;
             aws_mem_release(buffer_pool->base_allocator, block->block_ptr);
             aws_array_list_erase(&buffer_pool->blocks, i);
             /* do not increment since we just released element */

diff --git a/source/s3_client.c b/source/s3_client.c
@@ -346,7 +346,7 @@ struct aws_s3_client *aws_s3_client_new(
     client->buffer_pool = aws_s3_buffer_pool_new(allocator, part_size, mem_limit);
 
     if (client->buffer_pool == NULL) {
-        goto on_early_fail;
+        goto on_error;
     }
 
     struct aws_s3_buffer_pool_usage_stats pool_usage = aws_s3_buffer_pool_get_usage(client->buffer_pool);
@@ -357,15 +357,15 @@ struct aws_s3_client *aws_s3_client_new(
             "Cannot create client from client_config; configured max part size should not exceed memory limit."
             "size.");
         aws_raise_error(AWS_ERROR_S3_INVALID_MEMORY_LIMIT_CONFIG);
-        goto on_early_fail;
+        goto on_error;
     }
 
     client->vtable = &s_s3_client_default_vtable;
 
     aws_ref_count_init(&client->ref_count, client, (aws_simple_completion_callback *)s_s3_client_start_destroy);
 
     if (aws_mutex_init(&client->synced_data.lock) != AWS_OP_SUCCESS) {
-        goto on_early_fail;
+        goto on_error;
     }
 
     aws_linked_list_init(&client->synced_data.pending_meta_request_work);
@@ -488,6 +488,44 @@ struct aws_s3_client *aws_s3_client_new(
         }
     }
 
+    client->num_network_interface_names = client_config->num_network_interface_names;
+    if (client_config->num_network_interface_names > 0) {
+        AWS_LOGF_DEBUG(
+            AWS_LS_S3_CLIENT,
+            "id=%p Client received network interface names array with length %zu.",
+            (void *)client,
+            client->num_network_interface_names);
+        aws_array_list_init_dynamic(
+            &client->network_interface_names,
+            client->allocator,
+            client_config->num_network_interface_names,
+            sizeof(struct aws_string *));
+        client->network_interface_names_cursor_array = aws_mem_calloc(
+            client->allocator, client_config->num_network_interface_names, sizeof(struct aws_byte_cursor));
+        for (size_t i = 0; i < client_config->num_network_interface_names; i++) {
+            struct aws_byte_cursor interface_name = client_config->network_interface_names_array[i];
+            struct aws_string *interface_name_str = aws_string_new_from_cursor(client->allocator, &interface_name);
+            aws_array_list_push_back(&client->network_interface_names, &interface_name_str);
+            if (aws_is_network_interface_name_valid(aws_string_c_str(interface_name_str)) == false) {
+                AWS_LOGF_ERROR(
+                    AWS_LS_S3_CLIENT,
+                    "id=%p network_interface_names_array[%zu]=" PRInSTR " is not valid.",
+                    (void *)client,
+                    i,
+                    AWS_BYTE_CURSOR_PRI(interface_name));
+                aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
+                goto on_error;
+            }
+            client->network_interface_names_cursor_array[i] = aws_byte_cursor_from_string(interface_name_str);
+            AWS_LOGF_DEBUG(
+                AWS_LS_S3_CLIENT,
+                "id=%p network_interface_names_array[%zu]=" PRInSTR "",
+                (void *)client,
+                i,
+                AWS_BYTE_CURSOR_PRI(client->network_interface_names_cursor_array[i]));
+        }
+    }
+
     /* Set up body streaming ELG */
     {
         uint16_t num_event_loops =
@@ -579,34 +617,6 @@ struct aws_s3_client *aws_s3_client_new(
     *((bool *)&client->enable_read_backpressure) = client_config->enable_read_backpressure;
     *((size_t *)&client->initial_read_window) = client_config->initial_read_window;
 
-    client->num_network_interface_names = client_config->num_network_interface_names;
-    if (client_config->num_network_interface_names > 0) {
-        AWS_LOGF_DEBUG(
-            AWS_LS_S3_CLIENT,
-            "id=%p Client received network interface names array with length %zu.",
-            (void *)client,
-            client->num_network_interface_names);
-        aws_array_list_init_dynamic(
-            &client->network_interface_names,
-            client->allocator,
-            client_config->num_network_interface_names,
-            sizeof(struct aws_string *));
-        client->network_interface_names_cursor_array = aws_mem_calloc(
-            client->allocator, client_config->num_network_interface_names, sizeof(struct aws_byte_cursor));
-        for (size_t i = 0; i < client_config->num_network_interface_names; i++) {
-            struct aws_byte_cursor interface_name = client_config->network_interface_names_array[i];
-            struct aws_string *interface_name_str = aws_string_new_from_cursor(client->allocator, &interface_name);
-            aws_array_list_push_back(&client->network_interface_names, &interface_name_str);
-            client->network_interface_names_cursor_array[i] = aws_byte_cursor_from_string(interface_name_str);
-            AWS_LOGF_DEBUG(
-                AWS_LS_S3_CLIENT,
-                "id=%p network_interface_names_array[%zu]=" PRInSTR "",
-                (void *)client,
-                i,
-                AWS_BYTE_CURSOR_PRI(client->network_interface_names_cursor_array[i]));
-        }
-    }
-
     return client;
 
 on_error:
@@ -628,10 +638,22 @@ struct aws_s3_client *aws_s3_client_new(
     aws_mem_release(client->allocator, client->proxy_ev_settings);
     aws_mem_release(client->allocator, client->tcp_keep_alive_options);
 
-    aws_event_loop_group_release(client->client_bootstrap->event_loop_group);
+    if (client->client_bootstrap != NULL) {
+        aws_event_loop_group_release(client->client_bootstrap->event_loop_group);
+    }
     aws_client_bootstrap_release(client->client_bootstrap);
     aws_mutex_clean_up(&client->synced_data.lock);
-on_early_fail:
+
+    aws_mem_release(client->allocator, client->network_interface_names_cursor_array);
+    for (size_t i = 0; i < aws_array_list_length(&client->network_interface_names); i++) {
+        struct aws_string *interface_name = NULL;
+        aws_array_list_get_at(&client->network_interface_names, &interface_name, i);
+        aws_string_destroy(interface_name);
+    }
+
+    aws_array_list_clean_up(&client->network_interface_names);
+    aws_s3_buffer_pool_destroy(client->buffer_pool);
+
     aws_mem_release(client->allocator, client);
     return NULL;
 }
@@ -941,12 +963,6 @@ struct aws_s3_meta_request *aws_s3_client_make_meta_request(
             }
         }
 
-        if (options->checksum_config->location == AWS_SCL_HEADER) {
-            /* TODO: support calculate checksum to add to header */
-            aws_raise_error(AWS_ERROR_UNSUPPORTED_OPERATION);
-            return NULL;
-        }
-
         if (options->checksum_config->location != AWS_SCL_NONE &&
             options->checksum_config->checksum_algorithm == AWS_SCA_NONE) {
             AWS_LOGF_ERROR(

diff --git a/source/s3_copy_object.c b/source/s3_copy_object.c
@@ -37,12 +37,18 @@ static void s_s3_copy_object_request_finished(
     struct aws_s3_request *request,
     int error_code);
 
+static void s_s3_copy_object_sign_request(
+    struct aws_s3_meta_request *meta_request,
+    struct aws_s3_request *request,
+    aws_signing_complete_fn *on_signing_complete,
+    void *user_data);
+
 static struct aws_s3_meta_request_vtable s_s3_copy_object_vtable = {
     .update = s_s3_copy_object_update,
     .send_request_finish = aws_s3_meta_request_send_request_finish_default,
     .prepare_request = s_s3_copy_object_prepare_request,
     .init_signing_date_time = aws_s3_meta_request_init_signing_date_time_default,
-    .sign_request = aws_s3_meta_request_sign_request_default,
+    .sign_request = s_s3_copy_object_sign_request,
     .finished_request = s_s3_copy_object_request_finished,
     .destroy = s_s3_meta_request_copy_object_destroy,
     .finish = aws_s3_meta_request_finish_default,
@@ -796,3 +802,21 @@ static void s_s3_copy_object_request_finished(
 
     aws_s3_meta_request_unlock_synced_data(meta_request);
 }
+
+static void s_s3_copy_object_sign_request(
+    struct aws_s3_meta_request *meta_request,
+    struct aws_s3_request *request,
+    aws_signing_complete_fn *on_signing_complete,
+    void *user_data) {
+
+    /**
+     * https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html
+     * https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html
+     * For CopyObject and UploadPartCopy, the request has to be signed with IAM credentials for directory buckets.
+     * Disable S3 express signing for those types.
+     */
+    bool disable_s3_express_signing = request->request_tag == AWS_S3_COPY_OBJECT_REQUEST_TAG_BYPASS ||
+                                      request->request_tag == AWS_S3_COPY_OBJECT_REQUEST_TAG_MULTIPART_COPY;
+    aws_s3_meta_request_sign_request_default_impl(
+        meta_request, request, on_signing_complete, user_data, disable_s3_express_signing);
+}
diff --git a/source/s3_meta_request.c b/source/s3_meta_request.c
@@ -900,12 +900,12 @@ static int s_meta_request_resolve_signing_config(
     return AWS_OP_SUCCESS;
 }
 
-/* Handles signing a message for the caller. */
-void aws_s3_meta_request_sign_request_default(
+void aws_s3_meta_request_sign_request_default_impl(
     struct aws_s3_meta_request *meta_request,
     struct aws_s3_request *request,
     aws_signing_complete_fn *on_signing_complete,
-    void *user_data) {
+    void *user_data,
+    bool disable_s3_express_signing) {
     AWS_PRECONDITION(meta_request);
     AWS_PRECONDITION(request);
     AWS_PRECONDITION(on_signing_complete);
@@ -947,7 +947,7 @@ void aws_s3_meta_request_sign_request_default(
         return;
     }
 
-    if (signing_config.algorithm == AWS_SIGNING_ALGORITHM_V4_S3EXPRESS) {
+    if (signing_config.algorithm == AWS_SIGNING_ALGORITHM_V4_S3EXPRESS && !disable_s3_express_signing) {
         /* Fetch credentials from S3 Express provider. */
         struct aws_get_s3express_credentials_user_data *context =
             aws_mem_calloc(meta_request->allocator, 1, sizeof(struct aws_get_s3express_credentials_user_data));
@@ -998,6 +998,9 @@ void aws_s3_meta_request_sign_request_default(
         }
     } else {
         /* Regular signing. */
+        if (disable_s3_express_signing) {
+            signing_config.algorithm = AWS_SIGNING_ALGORITHM_V4;
+        }
         s_s3_meta_request_init_signing_date_time(meta_request, &signing_config.date);
         if (aws_sign_request_aws(
                 meta_request->allocator,
@@ -1015,6 +1018,15 @@ void aws_s3_meta_request_sign_request_default(
     }
 }
 
+/* Handles signing a message for the caller. */
+void aws_s3_meta_request_sign_request_default(
+    struct aws_s3_meta_request *meta_request,
+    struct aws_s3_request *request,
+    aws_signing_complete_fn *on_signing_complete,
+    void *user_data) {
+    aws_s3_meta_request_sign_request_default_impl(meta_request, request, on_signing_complete, user_data, false);
+}
+
 /* Handle the signing result */
 static void s_s3_meta_request_request_on_signed(
     struct aws_signing_result *signing_result,