Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

use CommonRunTime to support CopyObject #2564

Merged
merged 2 commits into from
Jul 6, 2023
Merged

use CommonRunTime to support CopyObject #2564

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1d76393
[aws-cpp-sdk-s3-crt]: use CommonRunTime to support CopyObject
grrtrr May 11, 2023
35e521e
Add codegen for crt copy object
sbiscigl Jul 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 2 additions & 13 deletions generated/src/aws-cpp-sdk-s3-crt/include/aws/s3-crt/S3CrtClient.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -391,23 +391,11 @@ namespace Aws
*/
virtual Model::CopyObjectOutcome CopyObject(const Model::CopyObjectRequest& request) const;

/**
* A Callable wrapper for CopyObject that returns a future to the operation so that it can be executed in parallel to other requests.
*/
template<typename CopyObjectRequestT = Model::CopyObjectRequest>
Model::CopyObjectOutcomeCallable CopyObjectCallable(const CopyObjectRequestT& request) const
{
return SubmitCallable(&S3CrtClient::CopyObject, request);
}

/**
* An Async wrapper for CopyObject that queues the request into a thread executor and triggers associated callback when operation has finished.
*/
template<typename CopyObjectRequestT = Model::CopyObjectRequest>
void CopyObjectAsync(const CopyObjectRequestT& request, const CopyObjectResponseReceivedHandler& handler, const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context = nullptr) const
{
return SubmitAsync(&S3CrtClient::CopyObject, request, handler, context);
}
virtual void CopyObjectAsync(const Model::CopyObjectRequest& request, const CopyObjectResponseReceivedHandler& handler, const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context = nullptr) const;

/**
* <p>Creates a new S3 bucket. To create a bucket, you must register with Amazon S3
Expand Down Expand Up @@ -5587,6 +5575,7 @@ namespace Aws
const S3CrtClient *s3CrtClient;
GetObjectResponseReceivedHandler getResponseHandler;
PutObjectResponseReceivedHandler putResponseHandler;
CopyObjectResponseReceivedHandler copyResponseHandler;
std::shared_ptr<const Aws::Client::AsyncCallerContext> asyncCallerContext;
const Aws::AmazonWebServiceRequest *originalRequest;
std::shared_ptr<Aws::Http::HttpRequest> request;
Expand Down
1 change: 0 additions & 1 deletion generated/src/aws-cpp-sdk-s3-crt/include/aws/s3-crt/S3CrtServiceClientModel.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,6 @@ namespace Aws
/* Service model Outcome callable definitions */
typedef std::future<AbortMultipartUploadOutcome> AbortMultipartUploadOutcomeCallable;
typedef std::future<CompleteMultipartUploadOutcome> CompleteMultipartUploadOutcomeCallable;
typedef std::future<CopyObjectOutcome> CopyObjectOutcomeCallable;
typedef std::future<CreateBucketOutcome> CreateBucketOutcomeCallable;
typedef std::future<CreateMultipartUploadOutcome> CreateMultipartUploadOutcomeCallable;
typedef std::future<DeleteBucketOutcome> DeleteBucketOutcomeCallable;
Expand Down
122 changes: 97 additions & 25 deletions generated/src/aws-cpp-sdk-s3-crt/source/S3CrtClient.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -527,6 +527,103 @@ void S3CrtClient::InitCommonCrtRequestOption(CrtRequestCallbackUserData *userDat
aws_uri_init_parse(options->endpoint, Aws::get_aws_allocator(), &endpointCursor);
}

static void CopyObjectRequestShutdownCallback(void *user_data)
{
auto *userData = static_cast<S3CrtClient::CrtRequestCallbackUserData*>(user_data);
// call user callback and release user_data
S3Crt::Model::CopyObjectOutcome outcome(userData->s3CrtClient->GenerateXmlOutcome(userData->response));
userData->copyResponseHandler(userData->s3CrtClient, *(reinterpret_cast<const CopyObjectRequest*>(userData->originalRequest)), std::move(outcome), userData->asyncCallerContext);

Aws::Delete(userData);
}

void S3CrtClient::CopyObjectAsync(const CopyObjectRequest& request, const CopyObjectResponseReceivedHandler& handler, const std::shared_ptr<const Aws::Client::AsyncCallerContext>& handlerContext) const
{
if (!m_endpointProvider) {
return handler(this, request, CopyObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Endpoint provider is not initialized", false)), handlerContext);
}
if (!request.BucketHasBeenSet())
{
AWS_LOGSTREAM_ERROR("CopyObject", "Required field: Bucket, is not set");
return handler(this, request, CopyObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [Bucket]", false)), handlerContext);
}
if (!request.CopySourceHasBeenSet())
{
AWS_LOGSTREAM_ERROR("CopyObject", "Required field: CopySource, is not set");
return handler(this, request, CopyObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [CopySource]", false)), handlerContext);
}
if (!request.KeyHasBeenSet())
{
AWS_LOGSTREAM_ERROR("CopyObject", "Required field: Key, is not set");
return handler(this, request, CopyObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [Key]", false)), handlerContext);
}
ResolveEndpointOutcome endpointResolutionOutcome = m_endpointProvider->ResolveEndpoint(request.GetEndpointContextParams());
if (!endpointResolutionOutcome.IsSuccess()) {
handler(this, request, CopyObjectOutcome(Aws::Client::AWSError<CoreErrors>(
CoreErrors::ENDPOINT_RESOLUTION_FAILURE, "ENDPOINT_RESOLUTION_FAILURE", endpointResolutionOutcome.GetError().GetMessage(), false)), handlerContext);
return;
}
endpointResolutionOutcome.GetResult().AddPathSegments(request.GetKey());

// make aws_s3_meta_request with callbacks
CrtRequestCallbackUserData *userData = Aws::New<CrtRequestCallbackUserData>(ALLOCATION_TAG);
aws_s3_meta_request_options options;
AWS_ZERO_STRUCT(options);
aws_uri endpoint;
AWS_ZERO_STRUCT(endpoint);
options.endpoint = &endpoint;
std::unique_ptr<aws_uri, void(*)(aws_uri*)> endpointCleanup { options.endpoint, &aws_uri_clean_up };

userData->copyResponseHandler = handler;
userData->asyncCallerContext = handlerContext;
InitCommonCrtRequestOption(userData, &options, &request, endpointResolutionOutcome.GetResult().GetURI(), Aws::Http::HttpMethod::HTTP_PUT);
if (userData != nullptr &&
userData->request != nullptr &&
userData->request->GetContentBody() != nullptr &&
userData->request->GetContentBody()->fail())
{
return handler(this, request, CopyObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Unable to create s3 meta request", false)), handlerContext);
}
options.shutdown_callback = CopyObjectRequestShutdownCallback;
options.type = AWS_S3_META_REQUEST_TYPE_COPY_OBJECT;
struct aws_signing_config_aws signing_config_override = m_s3CrtSigningConfig;
if (endpointResolutionOutcome.GetResult().GetAttributes() && endpointResolutionOutcome.GetResult().GetAttributes()->authScheme.GetSigningRegion()) {
signing_config_override.region = Aws::Crt::ByteCursorFromCString(endpointResolutionOutcome.GetResult().GetAttributes()->authScheme.GetSigningRegion()->c_str());
}
if (endpointResolutionOutcome.GetResult().GetAttributes() && endpointResolutionOutcome.GetResult().GetAttributes()->authScheme.GetSigningRegionSet()) {
signing_config_override.region = Aws::Crt::ByteCursorFromCString(endpointResolutionOutcome.GetResult().GetAttributes()->authScheme.GetSigningRegionSet()->c_str());
}
if (endpointResolutionOutcome.GetResult().GetAttributes() && endpointResolutionOutcome.GetResult().GetAttributes()->authScheme.GetSigningName()) {
signing_config_override.service = Aws::Crt::ByteCursorFromCString(endpointResolutionOutcome.GetResult().GetAttributes()->authScheme.GetSigningName()->c_str());
}
options.signing_config = &signing_config_override;

std::shared_ptr<Aws::Crt::Http::HttpRequest> crtHttpRequest = userData->request->ToCrtHttpRequest();
options.message= crtHttpRequest->GetUnderlyingMessage();
userData->crtHttpRequest = crtHttpRequest;

if (aws_s3_client_make_meta_request(m_s3CrtClient, &options) == nullptr)
{
return handler(this, request, CopyObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Unable to create s3 meta request", false)), handlerContext);
}
}

CopyObjectOutcome S3CrtClient::CopyObject(const CopyObjectRequest& request) const
{
AWS_OPERATION_GUARD(CopyObject);
Aws::Utils::Threading::Semaphore sem(0, 1);
CopyObjectOutcome res;

auto handler = CopyObjectResponseReceivedHandler{[&](const S3CrtClient*, const CopyObjectRequest&, const CopyObjectOutcome& outcome, const std::shared_ptr<const Aws::Client::AsyncCallerContext> &) {
res = std::move(outcome);
sem.ReleaseAll();
}};

S3CrtClient::CopyObjectAsync(request, handler, nullptr);
sem.WaitOne();
return res;
}

static void GetObjectRequestShutdownCallback(void *user_data)
{
auto *userData = static_cast<S3CrtClient::CrtRequestCallbackUserData*>(user_data);
Expand Down Expand Up @@ -761,31 +858,6 @@ CompleteMultipartUploadOutcome S3CrtClient::CompleteMultipartUpload(const Comple
return CompleteMultipartUploadOutcome(MakeRequest(request, endpointResolutionOutcome.GetResult(), Aws::Http::HttpMethod::HTTP_POST));
}

CopyObjectOutcome S3CrtClient::CopyObject(const CopyObjectRequest& request) const
{
AWS_OPERATION_GUARD(CopyObject);
AWS_OPERATION_CHECK_PTR(m_endpointProvider, CopyObject, CoreErrors, CoreErrors::ENDPOINT_RESOLUTION_FAILURE);
if (!request.BucketHasBeenSet())
{
AWS_LOGSTREAM_ERROR("CopyObject", "Required field: Bucket, is not set");
return CopyObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [Bucket]", false));
}
if (!request.CopySourceHasBeenSet())
{
AWS_LOGSTREAM_ERROR("CopyObject", "Required field: CopySource, is not set");
return CopyObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [CopySource]", false));
}
if (!request.KeyHasBeenSet())
{
AWS_LOGSTREAM_ERROR("CopyObject", "Required field: Key, is not set");
return CopyObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [Key]", false));
}
ResolveEndpointOutcome endpointResolutionOutcome = m_endpointProvider->ResolveEndpoint(request.GetEndpointContextParams());
AWS_OPERATION_CHECK_SUCCESS(endpointResolutionOutcome, CopyObject, CoreErrors, CoreErrors::ENDPOINT_RESOLUTION_FAILURE, endpointResolutionOutcome.GetError().GetMessage());
endpointResolutionOutcome.GetResult().AddPathSegments(request.GetKey());
return CopyObjectOutcome(MakeRequest(request, endpointResolutionOutcome.GetResult(), Aws::Http::HttpMethod::HTTP_PUT));
}

CreateBucketOutcome S3CrtClient::CreateBucket(const CreateBucketRequest& request) const
{
AWS_OPERATION_GUARD(CreateBucket);
Expand Down
12 changes: 2 additions & 10 deletions generated/src/aws-cpp-sdk-s3/include/aws/s3/S3Client.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,20 +408,12 @@ namespace Aws
/**
* A Callable wrapper for CopyObject that returns a future to the operation so that it can be executed in parallel to other requests.
*/
template<typename CopyObjectRequestT = Model::CopyObjectRequest>
Model::CopyObjectOutcomeCallable CopyObjectCallable(const CopyObjectRequestT& request) const
{
return SubmitCallable(&S3Client::CopyObject, request);
}
virtual Model::CopyObjectOutcomeCallable CopyObjectCallable(const Model::CopyObjectRequest& request) const;

/**
* An Async wrapper for CopyObject that queues the request into a thread executor and triggers associated callback when operation has finished.
*/
template<typename CopyObjectRequestT = Model::CopyObjectRequest>
void CopyObjectAsync(const CopyObjectRequestT& request, const CopyObjectResponseReceivedHandler& handler, const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context = nullptr) const
{
return SubmitAsync(&S3Client::CopyObject, request, handler, context);
}
virtual void CopyObjectAsync(const Model::CopyObjectRequest& request, const CopyObjectResponseReceivedHandler& handler, const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context = nullptr) const;

/**
* <p>Creates a new S3 bucket. To create a bucket, you must register with Amazon S3
Expand Down
16 changes: 16 additions & 0 deletions generated/src/aws-cpp-sdk-s3/source/S3Client.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,22 @@ CopyObjectOutcome S3Client::CopyObject(const CopyObjectRequest& request) const
return CopyObjectOutcome(MakeRequest(request, endpointResolutionOutcome.GetResult(), Aws::Http::HttpMethod::HTTP_PUT));
}

CopyObjectOutcomeCallable S3Client::CopyObjectCallable(const CopyObjectRequest& request) const
{
auto task = Aws::MakeShared< std::packaged_task< CopyObjectOutcome() > >(ALLOCATION_TAG, [this, request](){ return this->CopyObject(request); } );
auto packagedFunction = [task]() { (*task)(); };
m_executor->Submit(packagedFunction);
return task->get_future();
}

void S3Client::CopyObjectAsync(const CopyObjectRequest& request, const CopyObjectResponseReceivedHandler& handler, const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context) const
{
m_executor->Submit( [this, request, handler, context]()
{
handler(this, request, CopyObject(request), context);
} );
}

CreateBucketOutcome S3Client::CreateBucket(const CreateBucketRequest& request) const
{
AWS_OPERATION_GUARD(CreateBucket);
Expand Down
1 change: 1 addition & 0 deletions .../com/amazonaws/util/awsclientgenerator/generators/cpp/s3/S3RestXmlCppClientGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ public class S3RestXmlCppClientGenerator extends RestXmlCppClientGenerator {

s3CrtEnabledOps.add("GetObject");
s3CrtEnabledOps.add("PutObject");
s3CrtEnabledOps.add("CopyObject");

bucketLocationConstraints.add("us-east-1");
bucketLocationConstraints.add("us-east-2");
Expand Down
1 change: 1 addition & 0 deletions ...rc/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/s3/S3ClientHeader.vm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@ namespace ${rootNamespace}
const S3CrtClient *s3CrtClient;
GetObjectResponseReceivedHandler getResponseHandler;
PutObjectResponseReceivedHandler putResponseHandler;
CopyObjectResponseReceivedHandler copyResponseHandler;
std::shared_ptr<const Aws::Client::AsyncCallerContext> asyncCallerContext;
const Aws::AmazonWebServiceRequest *originalRequest;
std::shared_ptr<Aws::Http::HttpRequest> request;
Expand Down
16 changes: 16 additions & 0 deletions ...s/com/amazonaws/util/awsclientgenerator/velocity/cpp/s3/s3-crt/S3CrtSpecificOperations.vm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,8 @@ static void ${operation.name}RequestShutdownCallback(void *user_data)
userData->putResponseHandler(userData->s3CrtClient, *(reinterpret_cast<const ${operation.name}Request*>(userData->originalRequest)), std::move(outcome), userData->asyncCallerContext);
#elseif($operation.name == "GetObject")
userData->getResponseHandler(userData->s3CrtClient, *(reinterpret_cast<const ${operation.name}Request*>(userData->originalRequest)), std::move(outcome), userData->asyncCallerContext);
#elseif($operation.name == "CopyObject")
userData->copyResponseHandler(userData->s3CrtClient, *(reinterpret_cast<const ${operation.name}Request*>(userData->originalRequest)), std::move(outcome), userData->asyncCallerContext);
#end
#else
(*handler)(userData->s3CrtClient, outcome, userData->userCallbackContext);
Expand Down Expand Up @@ -262,6 +264,8 @@ void ${className}::${operation.name}Async(${constText}${operation.request.shape.
userData->putResponseHandler = handler;
#elseif($operation.name == "GetObject")
userData->getResponseHandler = handler;
#elseif($operation.name == "CopyObject")
userData->copyResponseHandler = handler;
#end
userData->asyncCallerContext = handlerContext;
#if($serviceModel.endpointRules)
Expand All @@ -275,6 +279,8 @@ void ${className}::${operation.name}Async(${constText}${operation.request.shape.
return handler(this, request, PutObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INVALID_PARAMETER_VALUE, "INVALID_PARAMETER_VALUE", "Input stream in bad state", false)), handlerContext);
#elseif($operation.name == "GetObject")
return handler(this, request, GetObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INVALID_PARAMETER_VALUE, "INVALID_PARAMETER_VALUE", "Output stream in bad state", false)), handlerContext);
#elseif($operation.name == "CopyObject")
return handler(this, request, CopyObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Unable to create s3 meta request", false)), handlerContext);
#end
}
#else
Expand All @@ -285,6 +291,8 @@ void ${className}::${operation.name}Async(${constText}${operation.request.shape.
options.type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT;
#elseif($operation.name == "GetObject")
options.type = AWS_S3_META_REQUEST_TYPE_GET_OBJECT;
#elseif($operation.name == "CopyObject")
options.type = AWS_S3_META_REQUEST_TYPE_COPY_OBJECT;
#else
options.type = AWS_S3_META_REQUEST_TYPE_DEFAULT;
#end
Expand Down Expand Up @@ -315,6 +323,8 @@ void ${className}::${operation.name}Async(${constText}${operation.request.shape.
return handler(this, request, PutObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Unable to create s3 meta request", false)), handlerContext);
#elseif($operation.name == "GetObject")
return handler(this, request, GetObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Unable to create s3 meta request", false)), handlerContext);
#elseif($operation.name == "CopyObject")
return handler(this, request, CopyObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Unable to create s3 meta request", false)), handlerContext);
#end
}
}
Expand Down Expand Up @@ -361,13 +371,17 @@ void ${className}::${operation.name}Async(${constText}${operation.name}ResponseR
userData->putResponseHandler = handler;
#elseif($operation.name == "GetObject")
userData->getResponseHandler = handler;
#elseif($operation.name == "CopyObject")
userData->copyResponseHandler = handler;
#end
InitCommonCrtRequestOption(userData, &options, nullptr, ss.str(), Aws::Http::HttpMethod::HTTP_${operation.http.method});
options.shutdown_callback = ${operation.name}RequestShutdownCallback;
#if($operation.name == "PutObject")
options.type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT;
#elseif($operation.name == "GetObject")
options.type = AWS_S3_META_REQUEST_TYPE_GET_OBJECT;
#elseif($operation.name == "CopyObject")
options.type = AWS_S3_META_REQUEST_TYPE_COPY_OBJECT;
#else
options.type = AWS_S3_META_REQUEST_TYPE_DEFAULT;
#end
Expand All @@ -386,6 +400,8 @@ void ${className}::${operation.name}Async(${constText}${operation.name}ResponseR
return handler(this, request, PutObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Unable to create s3 meta request", false)), handlerContext);
#elseif($operation.name == "GetObject")
return handler(this, request, GetObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Unable to create s3 meta request", false)), handlerContext);
#elseif($operation.name == "CopyObject")
return handler(this, request, CopyObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Unable to create s3 meta request", false)), handlerContext);
#end
}
}
Expand Down