From 22fb8488db7b58d18f7c950158d44ecd0fd8a7a3 Mon Sep 17 00:00:00 2001 From: Sean O'Brien Date: Tue, 17 Sep 2024 18:31:08 -0400 Subject: [PATCH] enhancement: flexible checksums --- .changes/nextrelease/checksumsv2.json | 7 + src/S3/ApplyChecksumMiddleware.php | 193 +++++++---- src/S3/CalculatesChecksumTrait.php | 48 +-- src/S3/MultipartUploader.php | 2 + src/S3/MultipartUploadingTrait.php | 18 +- .../ValidateResponseChecksumResultMutator.php | 128 +++++--- src/S3/S3Client.php | 244 +++++++++----- .../S3ValidateResponseChecksumMiddleware.php | 188 +++++++++++ src/Signature/S3SignatureV4.php | 10 + tests/S3/ApplyChecksumMiddlewareTest.php | 251 ++++++++------- tests/S3/MultipartUploaderTest.php | 33 +- tests/S3/ObjectUploaderTest.php | 30 +- ...idateResponseChecksumResultMutatorTest.php | 133 +++++++- tests/S3/S3ClientTest.php | 300 ++++++++++++++++-- tests/S3/TransferTest.php | 35 +- 15 files changed, 1224 insertions(+), 396 deletions(-) create mode 100644 .changes/nextrelease/checksumsv2.json create mode 100644 src/S3/S3ValidateResponseChecksumMiddleware.php diff --git a/.changes/nextrelease/checksumsv2.json b/.changes/nextrelease/checksumsv2.json new file mode 100644 index 0000000000..a322dac5a2 --- /dev/null +++ b/.changes/nextrelease/checksumsv2.json @@ -0,0 +1,7 @@ +[ + { + "type": "enhancement", + "category": "S3", + "description": "Adds additional config to " + } +] diff --git a/src/S3/ApplyChecksumMiddleware.php b/src/S3/ApplyChecksumMiddleware.php index a0ff65d6dc..b17e2936b7 100644 --- a/src/S3/ApplyChecksumMiddleware.php +++ b/src/S3/ApplyChecksumMiddleware.php @@ -2,8 +2,8 @@ namespace Aws\S3; use Aws\Api\Service; +use Aws\Api\Shape; use Aws\CommandInterface; -use GuzzleHttp\Psr7; use InvalidArgumentException; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\StreamInterface; @@ -18,14 +18,28 @@ class ApplyChecksumMiddleware { use CalculatesChecksumTrait; - private static $sha256AndMd5 = [ - 'PutObject', - 'UploadPart', + + public const DEFAULT_CALCULATION_MODE = 'when_supported'; + public const DEFAULT_ALGORITHM = 'crc32'; + + /** + * @var true[] + * + * S3 Operations for which pre-calculated SHA256 + * Checksums can be added to the command + */ + private static $sha256 = [ + 'PutObject' => true, + 'UploadPart' => true, ]; /** @var Service */ private $api; + /** @var array */ + private $config; + + /** @var callable */ private $nextHandler; /** @@ -34,17 +48,22 @@ class ApplyChecksumMiddleware * @param Service $api * @return callable */ - public static function wrap(Service $api) + public static function wrap(Service $api, array $config = []) { - return function (callable $handler) use ($api) { - return new self($handler, $api); + return function (callable $handler) use ($api, $config) { + return new self($handler, $api, $config); }; } - public function __construct(callable $nextHandler, Service $api) + public function __construct( + callable $nextHandler, + Service $api, array + $config = [] + ) { $this->api = $api; $this->nextHandler = $nextHandler; + $this->config = $config; } public function __invoke( @@ -54,59 +73,39 @@ public function __invoke( $next = $this->nextHandler; $name = $command->getName(); $body = $request->getBody(); + $op = $this->api->getOperation($name); + $mode = $this->config['request_checksum_calculation'] + ?? self::DEFAULT_CALCULATION_MODE; - //Checks if AddContentMD5 has been specified for PutObject or UploadPart - $addContentMD5 = $command['AddContentMD5'] ?? null; - - $op = $this->api->getOperation($command->getName()); + // Trigger warning if AddContentMD5 is specified for PutObject or UploadPart + $this->handleDeprecatedAddContentMD5($command); $checksumInfo = $op['httpChecksum'] ?? []; - $checksumMemberName = array_key_exists('requestAlgorithmMember', $checksumInfo) - ? $checksumInfo['requestAlgorithmMember'] - : ""; + $checksumMemberName = $checksumInfo['requestAlgorithmMember'] ?? ''; + $checksumMember = !empty($checksumMemberName) + ? $op->getInput()->getMember($checksumMemberName) + : null; + $checksumRequired = $checksumInfo['requestChecksumRequired'] ?? false; $requestedAlgorithm = $command[$checksumMemberName] ?? null; - if (!empty($checksumMemberName) && !empty($requestedAlgorithm)) { - $requestedAlgorithm = strtolower($requestedAlgorithm); - $checksumMember = $op->getInput()->getMember($checksumMemberName); - $supportedAlgorithms = isset($checksumMember['enum']) - ? array_map('strtolower', $checksumMember['enum']) - : null; - if (is_array($supportedAlgorithms) - && in_array($requestedAlgorithm, $supportedAlgorithms) - ) { - $request = $this->addAlgorithmHeader($requestedAlgorithm, $request, $body); - } else { - throw new InvalidArgumentException( - "Unsupported algorithm supplied for input variable {$checksumMemberName}." - . " Supported checksums for this operation include: " - . implode(", ", $supportedAlgorithms) . "." - ); - } - return $next($command, $request); - } - if (!empty($checksumInfo)) { - //if the checksum member is absent, check if it's required - $checksumRequired = $checksumInfo['requestChecksumRequired'] ?? null; - if ((!empty($checksumRequired)) - || (in_array($name, self::$sha256AndMd5) && $addContentMD5) - ) { - //S3Express doesn't support MD5; default to crc32 instead - if ($this->isS3Express($command)) { - $request = $this->addAlgorithmHeader('crc32', $request, $body); - } elseif (!$request->hasHeader('Content-MD5')) { - // Set the content MD5 header for operations that require it. - $request = $request->withHeader( - 'Content-MD5', - base64_encode(Psr7\Utils::hash($body, 'md5', true)) - ); - } - return $next($command, $request); - } + $shouldAddChecksum = $this->shouldAddChecksum( + $mode, + $checksumMember, + $name, + $checksumRequired, + $requestedAlgorithm + ); + if ($shouldAddChecksum) { + $algorithm = $this->determineChecksumAlgorithm( + $checksumMember, + $requestedAlgorithm, + $checksumMemberName + ); + $request = $this->addAlgorithmHeader($algorithm, $request, $body); } - if (in_array($name, self::$sha256AndMd5) && $command['ContentSHA256']) { - // Set the content hash header if provided in the parameters. + // Set the content hash header if ContentSHA256 is provided + if (isset(self::$sha256[$name]) && $command['ContentSHA256']) { $request = $request->withHeader( 'X-Amz-Content-Sha256', $command['ContentSHA256'] @@ -116,10 +115,83 @@ public function __invoke( return $next($command, $request); } + /** + * @param CommandInterface $command + * + * @return void + */ + private function handleDeprecatedAddContentMD5(CommandInterface $command): void + { + if (!empty($command['AddContentMD5'])) { + trigger_error( + 'S3 no longer supports MD5 checksums. ' . + 'A CRC32 checksum will be computed and applied on your behalf.', + E_USER_DEPRECATED + ); + $command['ChecksumAlgorithm'] = self::DEFAULT_ALGORITHM; + } + } + + /** + * @param string $mode + * @param Shape|null $checksumMember + * @param string $name + * @param bool $checksumRequired + * @param string|null $requestedAlgorithm + * + * @return bool + */ + private function shouldAddChecksum( + string $mode, + ?Shape $checksumMember, + string $name, + bool $checksumRequired, + ?string $requestedAlgorithm + ): bool + { + return ($mode === 'when_supported' && $checksumMember) + || ($mode === 'when_required' + && ($checksumRequired || ($checksumMember && $requestedAlgorithm))); + } + + /** + * @param Shape|null $checksumMember + * @param string|null $requestedAlgorithm + * @param string|null $checksumMemberName + * + * @return string + */ + private function determineChecksumAlgorithm( + ?Shape $checksumMember, + ?string $requestedAlgorithm, + ?string $checksumMemberName + ): string + { + $algorithm = self::DEFAULT_ALGORITHM; + $supportedAlgorithms = isset($checksumMember['enum']) + ? array_map('strtolower', $checksumMember['enum']) + : []; + + if ($requestedAlgorithm) { + $requestedAlgorithm = strtolower($requestedAlgorithm); + if (!in_array($requestedAlgorithm, $supportedAlgorithms)) { + throw new InvalidArgumentException( + "Unsupported algorithm supplied for input variable {$checksumMemberName}. " . + "Supported checksums for this operation include: " + . implode(", ", $supportedAlgorithms) . "." + ); + } + $algorithm = $requestedAlgorithm; + } + + return $algorithm; + } + /** * @param string $requestedAlgorithm * @param RequestInterface $request * @param StreamInterface $body + * * @return RequestInterface */ private function addAlgorithmHeader( @@ -129,19 +201,10 @@ private function addAlgorithmHeader( ) { $headerName = "x-amz-checksum-{$requestedAlgorithm}"; if (!$request->hasHeader($headerName)) { - $encoded = $this->getEncodedValue($requestedAlgorithm, $body); + $encoded = self::getEncodedValue($requestedAlgorithm, $body); $request = $request->withHeader($headerName, $encoded); } - return $request; - } - /** - * @param CommandInterface $command - * @return bool - */ - private function isS3Express(CommandInterface $command): bool - { - return isset($command['@context']['signing_service']) - && $command['@context']['signing_service'] === 's3express'; + return $request; } } diff --git a/src/S3/CalculatesChecksumTrait.php b/src/S3/CalculatesChecksumTrait.php index 3d0179ee64..d0d4907216 100644 --- a/src/S3/CalculatesChecksumTrait.php +++ b/src/S3/CalculatesChecksumTrait.php @@ -8,6 +8,13 @@ trait CalculatesChecksumTrait { + private static $supportedAlgorithms = [ + 'crc32c' => true, + 'crc32' => true, + 'sha256' => true, + 'sha1' => true + ]; + /** * @param string $requestedAlgorithm the algorithm to encode with * @param string $value the value to be encoded @@ -16,35 +23,36 @@ trait CalculatesChecksumTrait public static function getEncodedValue($requestedAlgorithm, $value) { $requestedAlgorithm = strtolower($requestedAlgorithm); $useCrt = extension_loaded('awscrt'); - if ($useCrt) { - $crt = new Crt(); - switch ($requestedAlgorithm) { - case 'crc32c': - return base64_encode(pack('N*',($crt->crc32c($value)))); - case 'crc32': - return base64_encode(pack('N*',($crt->crc32($value)))); - case 'sha256': - case 'sha1': - return base64_encode(Psr7\Utils::hash($value, $requestedAlgorithm, true)); - default: - break; - throw new InvalidArgumentException( - "Invalid checksum requested: {$requestedAlgorithm}." - . " Valid algorithms are CRC32C, CRC32, SHA256, and SHA1." - ); + + if (array_key_exists($requestedAlgorithm, self::$supportedAlgorithms)) { + if ($useCrt) { + $crt = new Crt(); + switch ($requestedAlgorithm) { + case 'crc32c': + return base64_encode(pack('N*',($crt::crc32c($value)))); + case 'crc32': + return base64_encode(pack('N*',($crt::crc32($value)))); + default: + break; + } } - } else { - if ($requestedAlgorithm == 'crc32c') { + + if ($requestedAlgorithm === 'crc32c') { throw new CommonRuntimeException("crc32c is not supported for checksums " . "without use of the common runtime for php. Please enable the CRT or choose " . "a different algorithm." ); } - if ($requestedAlgorithm == "crc32") { + + if ($requestedAlgorithm === "crc32") { $requestedAlgorithm = "crc32b"; } return base64_encode(Psr7\Utils::hash($value, $requestedAlgorithm, true)); } + //TODO add tests for this + throw new InvalidArgumentException( + "Invalid checksum requested: {$requestedAlgorithm}." + . " Valid algorithms are CRC32C, CRC32, SHA256, and SHA1." + ); } - } diff --git a/src/S3/MultipartUploader.php b/src/S3/MultipartUploader.php index ae47d7e5fd..e74fffbf99 100644 --- a/src/S3/MultipartUploader.php +++ b/src/S3/MultipartUploader.php @@ -70,6 +70,8 @@ public function __construct( 'key' => null, 'exception_class' => S3MultipartUploadException::class, ]); + + //TODO move } protected function loadUploadWorkflowInfo() diff --git a/src/S3/MultipartUploadingTrait.php b/src/S3/MultipartUploadingTrait.php index 002bd43c46..a36fa2f403 100644 --- a/src/S3/MultipartUploadingTrait.php +++ b/src/S3/MultipartUploadingTrait.php @@ -54,10 +54,17 @@ protected function handleResult(CommandInterface $command, ResultInterface $resu $partData = []; $partData['PartNumber'] = $command['PartNumber']; $partData['ETag'] = $this->extractETag($result); + + $checksumResult = $this instanceof MultipartCopy + ? $result['CopyPartResult'] + : $result; if (isset($command['ChecksumAlgorithm'])) { $checksumMemberName = 'Checksum' . strtoupper($command['ChecksumAlgorithm']); - $partData[$checksumMemberName] = $result[$checksumMemberName]; + $partData[$checksumMemberName] = $checksumResult[$checksumMemberName]; + } else { + $this->applyChecksumToResult($checksumResult, $partData); } + $this->getState()->markPartAsUploaded($command['PartNumber'], $partData); } @@ -114,6 +121,15 @@ protected function getInitiateParams() return $params; } + protected function applyChecksumToResult($result, array &$partData): void + { + foreach($result as $key => $value) { + if (!empty($value) && strpos($key, 'Checksum') === 0) { + $partData[$key] = $value; + } + } + } + /** * @return UploadState */ diff --git a/src/S3/Parser/ValidateResponseChecksumResultMutator.php b/src/S3/Parser/ValidateResponseChecksumResultMutator.php index d1abd995b1..f8b84d3390 100644 --- a/src/S3/Parser/ValidateResponseChecksumResultMutator.php +++ b/src/S3/Parser/ValidateResponseChecksumResultMutator.php @@ -17,15 +17,23 @@ final class ValidateResponseChecksumResultMutator implements S3ResultMutator { use CalculatesChecksumTrait; + + public const DEFAULT_VALIDATION_MODE = 'when_supported'; + /** @var Service $api */ private $api; + /** @var array $api */ + private $config; + /** * @param Service $api + * @param array $config */ - public function __construct(Service $api) + public function __construct(Service $api, array $config = []) { $this->api = $api; + $this->config = $config; } /** @@ -42,56 +50,40 @@ public function __invoke( ): ResultInterface { $operation = $this->api->getOperation($command->getName()); + // Skip this middleware if the operation doesn't have an httpChecksum - $checksumInfo = empty($operation['httpChecksum']) - ? null - : $operation['httpChecksum'];; - if (null === $checksumInfo) { + $checksumInfo = $operation['httpChecksum'] ?? null; + if (is_null($checksumInfo)) { return $result; } - // Skip this middleware if the operation doesn't send back a checksum, - // or the user doesn't opt in + $mode = $this->config['response_checksum_validation'] ?? self::DEFAULT_VALIDATION_MODE; $checksumModeEnabledMember = $checksumInfo['requestValidationModeMember'] ?? ""; - $checksumModeEnabled = $command[$checksumModeEnabledMember] ?? ""; + $checksumModeEnabled = strtolower($command[$checksumModeEnabledMember] ?? ""); $responseAlgorithms = $checksumInfo['responseAlgorithms'] ?? []; - if (empty($responseAlgorithms) - || strtolower($checksumModeEnabled) !== "enabled") { - return $result; - } + $shouldSkipValidation = $this->shouldSkipValidation( + $mode, + $checksumModeEnabled, + $responseAlgorithms + ); - if (extension_loaded('awscrt')) { - $checksumPriority = ['CRC32C', 'CRC32', 'SHA1', 'SHA256']; - } else { - $checksumPriority = ['CRC32', 'SHA1', 'SHA256']; + if ($shouldSkipValidation) { + return $result; } - $checksumsToCheck = array_intersect( - $responseAlgorithms, - $checksumPriority + $checksumPriority = $this->getChecksumPriority(); + $checksumsToCheck = array_intersect($responseAlgorithms, array_map( + 'strtoupper', + array_keys($checksumPriority)) ); - $checksumValidationInfo = $this->validateChecksum( - $checksumsToCheck, - $response - ); - if ($checksumValidationInfo['status'] == "SUCCEEDED") { + $checksumValidationInfo = $this->validateChecksum($checksumsToCheck, $response); + + if ($checksumValidationInfo['status'] === "SUCCEEDED") { $result['ChecksumValidated'] = $checksumValidationInfo['checksum']; - } elseif ($checksumValidationInfo['status'] == "FAILED") { - // Ignore failed validations on GetObject if it's a multipart get - // which returned a full multipart object - if ($command->getName() === "GetObject" - && !empty($checksumValidationInfo['checksumHeaderValue']) - ) { - $headerValue = $checksumValidationInfo['checksumHeaderValue']; - $lastDashPos = strrpos($headerValue, '-'); - $endOfChecksum = substr($headerValue, $lastDashPos + 1); - if (is_numeric($endOfChecksum) - && intval($endOfChecksum) > 1 - && intval($endOfChecksum) < 10000) { - return $result; - } + } elseif ($checksumValidationInfo['status'] === "FAILED") { + if ($this->isMultipartGetObject($command, $checksumValidationInfo)) { + return $result; } - throw new S3Exception( "Calculated response checksum did not match the expected value", $command @@ -119,11 +111,10 @@ private function validateChecksum( $validationStatus = "SKIPPED"; $checksumHeaderValue = null; if (!empty($checksumToValidate)) { - $checksumHeaderValue = $response->getHeader( + $checksumHeaderValue = $response->getHeaderLine( 'x-amz-checksum-' . $checksumToValidate ); - if (isset($checksumHeaderValue)) { - $checksumHeaderValue = $checksumHeaderValue[0]; + if (!empty($checksumHeaderValue)) { $calculatedChecksumValue = $this->getEncodedValue( $checksumToValidate, $response->getBody() @@ -160,4 +151,57 @@ private function chooseChecksumHeaderToValidate( return null; } + + /** + * @param string $mode + * @param string $checksumModeEnabled + * @param array $responseAlgorithms + * + * @return bool + */ + private function shouldSkipValidation( + string $mode, + string $checksumModeEnabled, + array $responseAlgorithms + ): bool + { + return empty($responseAlgorithms) + || ($mode === 'when_required' && $checksumModeEnabled !== 'enabled'); + } + + /** + * @return string[] + */ + private function getChecksumPriority(): array + { + return extension_loaded('awscrt') + ? self::$supportedAlgorithms + : array_slice(self::$supportedAlgorithms, 1); + } + + /** + * @param CommandInterface $command + * @param array $checksumValidationInfo + * + * @return bool + */ + private function isMultipartGetObject( + CommandInterface $command, + array $checksumValidationInfo + ): bool + { + if ($command->getName() !== "GetObject" + || empty($checksumValidationInfo['checksumHeaderValue']) + ) { + return false; + } + + $headerValue = $checksumValidationInfo['checksumHeaderValue']; + $lastDashPos = strrpos($headerValue, '-'); + $endOfChecksum = substr($headerValue, $lastDashPos + 1); + + return is_numeric($endOfChecksum) + && (int) $endOfChecksum > 1 + && (int) $endOfChecksum < 10000; + } } diff --git a/src/S3/S3Client.php b/src/S3/S3Client.php index bcd0bbcc6f..c509720d45 100644 --- a/src/S3/S3Client.php +++ b/src/S3/S3Client.php @@ -239,80 +239,105 @@ class S3Client extends AwsClient implements S3ClientInterface /** @var array */ private static $mandatoryAttributes = ['Bucket', 'Key']; + /** @var array */ + private static $checksumValidationOptionEnum = [ + 'when_supported' => true, + 'when_required' => true + ]; + public static function getArguments() { $args = parent::getArguments(); $args['retries']['fn'] = [__CLASS__, '_applyRetryConfig']; $args['api_provider']['fn'] = [__CLASS__, '_applyApiProvider']; - return $args + [ - 'bucket_endpoint' => [ - 'type' => 'config', - 'valid' => ['bool'], - 'doc' => 'Set to true to send requests to a hardcoded ' - . 'bucket endpoint rather than create an endpoint as a ' - . 'result of injecting the bucket into the URL. This ' - . 'option is useful for interacting with CNAME endpoints.', - ], - 'use_arn_region' => [ - 'type' => 'config', - 'valid' => [ - 'bool', - Configuration::class, - CacheInterface::class, - 'callable' + return + [ + 'request_checksum_calculation' => [ + 'type' => 'config', + 'valid' => ['string'], + 'doc' => 'Set to true to disable the usage of' + . ' s3 express session authentication. This is enabled by default.', + 'fn' => [__CLASS__, '_apply_request_checksum_calculation'], + 'default' => [__CLASS__, '_default_request_checksum_calculation'], ], - 'doc' => 'Set to true to allow passed in ARNs to override' - . ' client region. Accepts...', - 'fn' => [__CLASS__, '_apply_use_arn_region'], - 'default' => [UseArnRegionConfigurationProvider::class, 'defaultProvider'], - ], - 'use_accelerate_endpoint' => [ - 'type' => 'config', - 'valid' => ['bool'], - 'doc' => 'Set to true to send requests to an S3 Accelerate' - . ' endpoint by default. Can be enabled or disabled on' - . ' individual operations by setting' - . ' \'@use_accelerate_endpoint\' to true or false. Note:' - . ' you must enable S3 Accelerate on a bucket before it can' - . ' be accessed via an Accelerate endpoint.', - 'default' => false, - ], - 'use_path_style_endpoint' => [ - 'type' => 'config', - 'valid' => ['bool'], - 'doc' => 'Set to true to send requests to an S3 path style' - . ' endpoint by default.' - . ' Can be enabled or disabled on individual operations by setting' - . ' \'@use_path_style_endpoint\' to true or false.', - 'default' => false, - ], - 'disable_multiregion_access_points' => [ - 'type' => 'config', - 'valid' => ['bool'], - 'doc' => 'Set to true to disable the usage of' - . ' multi region access points. These are enabled by default.' - . ' Can be enabled or disabled on individual operations by setting' - . ' \'@disable_multiregion_access_points\' to true or false.', - 'default' => false, - ], - 'disable_express_session_auth' => [ - 'type' => 'config', - 'valid' => ['bool'], - 'doc' => 'Set to true to disable the usage of' - . ' s3 express session authentication. This is enabled by default.', - 'default' => [__CLASS__, '_default_disable_express_session_auth'], - ], - 's3_express_identity_provider' => [ - 'type' => 'config', - 'valid' => [ - 'bool', - 'callable' + 'response_checksum_validation' => [ + 'type' => 'config', + 'valid' => ['string'], + 'doc' => 'Set to true to disable the usage of' + . ' s3 express session authentication. This is enabled by default.', + 'fn' => [__CLASS__, '_apply_response_checksum_validation'], + 'default' => [__CLASS__, '_default_response_checksum_validation'], + ] + ] + + $args + [ + 'bucket_endpoint' => [ + 'type' => 'config', + 'valid' => ['bool'], + 'doc' => 'Set to true to send requests to a hardcoded ' + . 'bucket endpoint rather than create an endpoint as a ' + . 'result of injecting the bucket into the URL. This ' + . 'option is useful for interacting with CNAME endpoints.', + ], + 'use_arn_region' => [ + 'type' => 'config', + 'valid' => [ + 'bool', + Configuration::class, + CacheInterface::class, + 'callable' + ], + 'doc' => 'Set to true to allow passed in ARNs to override' + . ' client region. Accepts...', + 'fn' => [__CLASS__, '_apply_use_arn_region'], + 'default' => [UseArnRegionConfigurationProvider::class, 'defaultProvider'], ], - 'doc' => 'Specifies the provider used to generate identities to sign s3 express requests. ' - . 'Set to `false` to disable s3 express auth, or a callable provider used to create s3 express ' - . 'identities or return null.', - 'default' => [__CLASS__, '_default_s3_express_identity_provider'], + 'use_accelerate_endpoint' => [ + 'type' => 'config', + 'valid' => ['bool'], + 'doc' => 'Set to true to send requests to an S3 Accelerate' + . ' endpoint by default. Can be enabled or disabled on' + . ' individual operations by setting' + . ' \'@use_accelerate_endpoint\' to true or false. Note:' + . ' you must enable S3 Accelerate on a bucket before it can' + . ' be accessed via an Accelerate endpoint.', + 'default' => false, + ], + 'use_path_style_endpoint' => [ + 'type' => 'config', + 'valid' => ['bool'], + 'doc' => 'Set to true to send requests to an S3 path style' + . ' endpoint by default.' + . ' Can be enabled or disabled on individual operations by setting' + . ' \'@use_path_style_endpoint\' to true or false.', + 'default' => false, + ], + 'disable_multiregion_access_points' => [ + 'type' => 'config', + 'valid' => ['bool'], + 'doc' => 'Set to true to disable the usage of' + . ' multi region access points. These are enabled by default.' + . ' Can be enabled or disabled on individual operations by setting' + . ' \'@disable_multiregion_access_points\' to true or false.', + 'default' => false, + ], + 'disable_express_session_auth' => [ + 'type' => 'config', + 'valid' => ['bool'], + 'doc' => 'Set to true to disable the usage of' + . ' s3 express session authentication. This is enabled by default.', + 'default' => [__CLASS__, '_default_disable_express_session_auth'], + ], + 's3_express_identity_provider' => [ + 'type' => 'config', + 'valid' => [ + 'bool', + 'callable' + ], + 'doc' => 'Specifies the provider used to generate identities to sign s3 express requests. ' + . 'Set to `false` to disable s3 express auth, or a callable provider used to create s3 express ' + . 'identities or return null.', + 'default' => [__CLASS__, '_default_s3_express_identity_provider'], ], ]; } @@ -384,7 +409,10 @@ public function __construct(array $args) parent::__construct($args); $stack = $this->getHandlerList(); $stack->appendInit(SSECMiddleware::wrap($this->getEndpoint()->getScheme()), 's3.ssec'); - $stack->appendBuild(ApplyChecksumMiddleware::wrap($this->getApi()), 's3.checksum'); + $stack->appendBuild( + ApplyChecksumMiddleware::wrap($this->getApi(), $this->getConfig()), + 's3.checksum' + ); $stack->appendBuild( Middleware::contentType(['PutObject', 'UploadPart']), 's3.content_type' @@ -499,6 +527,66 @@ public static function _apply_use_arn_region($value, array &$args, HandlerList $ } } + public static function _default_request_checksum_calculation(array $args): string + { + return ConfigurationResolver::resolve( + 'request_checksum_calculation', + ApplyChecksumMiddleware::DEFAULT_CALCULATION_MODE, + 'string', + $args + ); + } + + public static function _apply_request_checksum_calculation( + string $value, + array &$args + ): void + { + $value = strtolower($value); + if (array_key_exists($value, self::$checksumValidationOptionEnum)) { + $args['request_checksum_calculation'] = $value; + } + } + + public static function _default_response_checksum_validation(array $args): string + { + return ConfigurationResolver::resolve( + 'response_checksum_validation', + ValidateResponseChecksumResultMutator::DEFAULT_VALIDATION_MODE, + 'string', + $args + ); + } + + public static function _apply_response_checksum_validation( + $value, + array &$args + ): void + { + $value = strtolower($value); + if (array_key_exists($value, self::$checksumValidationOptionEnum)) { + $args['request_checksum_calculation'] = $value; + } + } + + public static function _default_disable_express_session_auth(array &$args) + { + return ConfigurationResolver::resolve( + 's3_disable_express_session_auth', + false, + 'bool', + $args + ); + } + + public static function _default_s3_express_identity_provider(array $args) + { + if ($args['config']['disable_express_session_auth']) { + return false; + } + return new S3ExpressIdentityProvider($args['region']); + } + public function createPresignedRequest(CommandInterface $command, $expires, array $options = []) { $command = clone $command; @@ -749,23 +837,6 @@ private function getSigningName($host) return $this->getConfig('signing_name'); } - public static function _default_disable_express_session_auth(array &$args) { - return ConfigurationResolver::resolve( - 's3_disable_express_session_auth', - false, - 'bool', - $args - ); - } - - public static function _default_s3_express_identity_provider(array $args) - { - if ($args['config']['disable_express_session_auth']) { - return false; - } - return new S3ExpressIdentityProvider($args['region']); - } - /** * If EndpointProviderV2 is used, removes `Bucket` from request URIs. * This is now handled by the endpoint ruleset. @@ -962,7 +1033,10 @@ public static function _applyApiProvider($value, array &$args, HandlerList $list ); $s3Parser->addS3ResultMutator( 'validate-response-checksum', - new ValidateResponseChecksumResultMutator($args['api']) + new ValidateResponseChecksumResultMutator( + $args['api'], + ['response_checksum_validation' => $args['response_checksum_validation']] + ) ); $args['parser'] = $s3Parser; } diff --git a/src/S3/S3ValidateResponseChecksumMiddleware.php b/src/S3/S3ValidateResponseChecksumMiddleware.php new file mode 100644 index 0000000000..4584ffa8ae --- /dev/null +++ b/src/S3/S3ValidateResponseChecksumMiddleware.php @@ -0,0 +1,188 @@ +api = $api; + } + + /** + * @param ResultInterface $result + * @param CommandInterface|null $command + * @param ResponseInterface|null $response + * + * @return ResultInterface + */ + public function __invoke( + CommandInterface $command, + RequestInterface $request = null + ): ResultInterface + { + $next = $this->nextHandler; + $api = $this->api; + + return $next($command, $request)->then( + function (ResultInterface $result) use ($command, $api) { + $operation = $api->getOperation($command->getName()); + // Skip this middleware if the operation doesn't have an httpChecksum + $checksumInfo = empty($operation['httpChecksum']) + ? null + : $operation['httpChecksum']; + if (null === $checksumInfo) { + return $result; + } + + // Skip this middleware if the operation doesn't send back a checksum, + // or the user doesn't opt in + $checksumModeEnabledMember = $checksumInfo['requestValidationModeMember'] ?? ""; + $checksumModeEnabled = $command[$checksumModeEnabledMember] ?? ""; + $responseAlgorithms = $checksumInfo['responseAlgorithms'] ?? []; + if (empty($responseAlgorithms) + || strtolower($checksumModeEnabled) !== "enabled") { + return $result; + } + + if (extension_loaded('awscrt')) { + $checksumPriority = ['CRC32C', 'CRC32', 'SHA1', 'SHA256']; + } else { + $checksumPriority = ['CRC32', 'SHA1', 'SHA256']; + } + + $checksumsToCheck = array_intersect( + $responseAlgorithms, + $checksumPriority + ); + $checksumValidationInfo = $this->validateChecksum( + $checksumsToCheck, + $response + ); + if ($checksumValidationInfo['status'] == "SUCCEEDED") { + $result['ChecksumValidated'] = $checksumValidationInfo['checksum']; + } elseif ($checksumValidationInfo['status'] == "FAILED") { + // Ignore failed validations on GetObject if it's a multipart get + // which returned a full multipart object + if ($command->getName() === "GetObject" + && !empty($checksumValidationInfo['checksumHeaderValue']) + ) { + $headerValue = $checksumValidationInfo['checksumHeaderValue']; + $lastDashPos = strrpos($headerValue, '-'); + $endOfChecksum = substr($headerValue, $lastDashPos + 1); + if (is_numeric($endOfChecksum) + && intval($endOfChecksum) > 1 + && intval($endOfChecksum) < 10000) { + return $result; + } + } + + throw new S3Exception( + "Calculated response checksum did not match the expected value", + $command + ); + } + + return $result; + } + ); + } + + /** + * @param $checksumPriority + * @param ResponseInterface $response + * + * @return array + */ + private function validateChecksum( + $checksumPriority, + ResponseInterface $response + ): array + { + $checksumToValidate = $this->chooseChecksumHeaderToValidate( + $checksumPriority, + $response + ); + $validationStatus = "SKIPPED"; + $checksumHeaderValue = null; + if (!empty($checksumToValidate)) { + $checksumHeaderValue = $response->getHeader( + 'x-amz-checksum-' . $checksumToValidate + ); + if (isset($checksumHeaderValue)) { + $checksumHeaderValue = $checksumHeaderValue[0]; + $calculatedChecksumValue = $this->getEncodedValue( + $checksumToValidate, + $response->getBody() + ); + $validationStatus = $checksumHeaderValue == $calculatedChecksumValue + ? "SUCCEEDED" + : "FAILED"; + } + } + return [ + "status" => $validationStatus, + "checksum" => $checksumToValidate, + "checksumHeaderValue" => $checksumHeaderValue, + ]; + } + + /** + * @param $checksumPriority + * @param ResponseInterface $response + * + * @return string + */ + private function chooseChecksumHeaderToValidate( + $checksumPriority, + ResponseInterface $response + ):? string + { + foreach ($checksumPriority as $checksum) { + $checksumHeader = 'x-amz-checksum-' . $checksum; + if ($response->hasHeader($checksumHeader)) { + return $checksum; + } + } + + return null; + } +} + + diff --git a/src/Signature/S3SignatureV4.php b/src/Signature/S3SignatureV4.php index b47c0d5dbe..2e60c31f60 100644 --- a/src/Signature/S3SignatureV4.php +++ b/src/Signature/S3SignatureV4.php @@ -94,6 +94,16 @@ public function presign( $this->getPresignedPayload($request) ); } + + //Payload is unknown, checksum will cause requests to fail. + if ($request->getMethod() === 'PUT') { + foreach($request->getHeaders() as $header => $value) { + if (strpos($header, 'x-amz-checksum') === 0) { + $request = $request->withoutHeader($header); + } + } + } + if (strpos($request->getUri()->getHost(), "accesspoint.s3-global")) { $request = $request->withHeader("x-amz-region-set", "*"); } diff --git a/tests/S3/ApplyChecksumMiddlewareTest.php b/tests/S3/ApplyChecksumMiddlewareTest.php index fcf62c13a3..106c266125 100644 --- a/tests/S3/ApplyChecksumMiddlewareTest.php +++ b/tests/S3/ApplyChecksumMiddlewareTest.php @@ -1,11 +1,10 @@ getTestClient( - 's3', - ['api_provider' => ApiProvider::filesystem(__DIR__ . '/fixtures')] - ); - $this->addMockResults($s3, [[]]); - $command = $s3->getCommand($operation, $args); - $command->getHandlerList()->appendBuild( - Middleware::tap(function ($cmd, RequestInterface $request) use ($md5Added, $md5Value) { - $this->assertSame($md5Added, $request->hasHeader('Content-MD5')); - $this->assertEquals($md5Value, $request->getHeaderLine('Content-MD5')); - }) + public function testFlexibleChecksums( + $operation, + $config, + $commandArgs, + $body, + $headerAdded, + $headerValue + ){ + if (isset($commandArgs['ChecksumAlgorithm']) + && $commandArgs['ChecksumAlgorithm'] === 'crc32c' + && !extension_loaded('awscrt') + ) { + $this->markTestSkipped("Cannot test crc32c without the CRT"); + } + + $client = $this->getTestClient('s3'); + $nextHandler = function ($cmd, $request) use ($headerAdded, $headerValue, $commandArgs) { + $checksumName = $commandArgs['ChecksumAlgorithm'] ?? "crc32"; + if ($headerAdded) { + $this->assertTrue( $request->hasHeader("x-amz-checksum-{$checksumName}")); + } + $this->assertEquals($headerValue, $request->getHeaderLine("x-amz-checksum-{$checksumName}")); + }; + $service = $client->getApi(); + $mw = new ApplyChecksumMiddleware($nextHandler, $service, $config); + $command = $client->getCommand($operation, $commandArgs); + $request = new Request( + $operation === 'getObject' + ? 'GET' + : 'PUT', + 'https://foo.bar', + [], + $body ); - $s3->execute($command); + + $mw($command, $request); } - public function getContentMd5UseCases() + public function getFlexibleChecksumUseCases() { return [ - // Test that explicitly proviced Content MD5 is passed through + // httpChecksum not modeled [ - 'PutBucketLogging', + 'GetObject', + [], [ 'Bucket' => 'foo', - 'BucketLoggingStatus' => [ - 'LoggingEnabled' => [ - 'TargetBucket' => 'bar', - 'TargetPrefix' => 'baz' - ] - ], - 'ContentMD5' => 'custommd5' + 'Key' => 'bar', + 'ChecksumMode' => 'ENABLED' ], - true, - 'custommd5' + null, + false, + '' ], - // Test MD5 added for operations that require it + // default: when_supported. defaults to crc32 [ - 'DeleteObjects', - ['Bucket' => 'foo', 'Delete' => ['Objects' => [['Key' => 'bar']]]], + 'PutObject', + [], + [ + 'Bucket' => 'foo', + 'Key' => 'bar', + 'Body' => 'abc' + ], + 'abc', true, - '/12roh/ATpPMcGD9Rj4ZlQ==' + 'NSRBwg==' ], - // Test MD5 not added for operations that do not require it + // when_required when not required and no requested algorithm [ - 'GetObject', - ['Bucket' => 'foo', 'Key' => 'bar'], + 'PutObject', + ['request_checksum_calculation' => 'when_required'], + [ + 'Bucket' => 'foo', + 'Key' => 'bar', + 'Body' => 'abc' + ], + 'abc', false, - null, + '' ], - ]; - } - - public function testAddCrc32AsAppropriate() - { - $s3 = $this->getTestClient( - 's3', - ['api_provider' => ApiProvider::filesystem(__DIR__ . '/fixtures')] - ); - $this->addMockResults($s3, [[]]); - $command = $s3->getCommand('putBucketPolicy', [ - 'Bucket' => 'mybucket--use1-az1--x-s3', - 'Policy' => 'policy' - ]); - $command->getHandlerList()->appendBuild( - Middleware::tap(function ($cmd, RequestInterface $request) { - $this->assertFalse($request->hasHeader('Content-MD5')); - $this->assertSame('8H0FFg==', $request->getHeaderLine('x-amz-checksum-crc32')); - }) - ); - $s3->execute($command); - } - - /** - * @dataProvider getFlexibleChecksumUseCases - */ - public function testAddsFlexibleChecksumAsAppropriate($operation, $args, $headerAdded, $headerValue) - { - if (isset($args['ChecksumAlgorithm']) - && $args['ChecksumAlgorithm'] === 'crc32c' - && !extension_loaded('awscrt') - ) { - $this->markTestSkipped("Cannot test crc32c without the CRT"); - } - $s3 = $this->getTestClient( - 's3', - ['api_provider' => ApiProvider::filesystem(__DIR__ . '/fixtures')] - ); - $this->addMockResults($s3, [[]]); - $command = $s3->getCommand($operation, $args); - $command->getHandlerList()->appendBuild( - Middleware::tap(function ($cmd, RequestInterface $request) use ($headerAdded, $headerValue, $args) { - $checksumName = isset($args['ChecksumAlgorithm']) ? $args['ChecksumAlgorithm'] : ""; - $this->assertSame($headerAdded, $request->hasHeader("x-amz-checksum-{$checksumName}")); - $this->assertEquals($headerValue, $request->getHeaderLine("x-amz-checksum-{$checksumName}")); - }) - ); - $s3->execute($command); - } - - public function getFlexibleChecksumUseCases() - { - return [ - // Test that explicitly proviced Content MD5 is passed through + // when_required when required and no requested algorithm [ - 'GetObject', + 'PutObjectLockConfiguration', + ['request_checksum_calculation' => 'when_required'], [ 'Bucket' => 'foo', 'Key' => 'bar', - 'ChecksumMode' => 'ENABLED' + 'ObjectLockConfiguration' => 'blah' ], - false, - '' + 'blah', + true, + 'zilhXA==' ], + // when_required when not required and requested algorithm [ 'PutObject', + ['request_checksum_calculation' => 'when_required'], [ 'Bucket' => 'foo', 'Key' => 'bar', + 'Body' => 'blah', 'ChecksumAlgorithm' => 'crc32', - 'Body' => 'abc' ], + 'blah', true, - 'EZo2zw==' + 'zilhXA==' ], + // when_supported and requested algorithm [ 'PutObject', + [], [ 'Bucket' => 'foo', 'Key' => 'bar', 'ChecksumAlgorithm' => 'crc32c', 'Body' => 'abc' ], + 'abc', true, - 'oD04yw==' + 'Nks/tw==' ], + // when_supported and requested algorithm [ 'PutObject', + [], [ 'Bucket' => 'foo', 'Key' => 'bar', 'ChecksumAlgorithm' => 'sha256' ], + '', true, - 'OmJVpLQxEjty3SMySBLGRfWtoBQ/ZXT2cT2Cjuly4XY=' + '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=' ], + // when_supported and requested algorithm [ 'PutObject', + [], [ 'Bucket' => 'foo', 'Key' => 'bar', 'ChecksumAlgorithm' => 'SHA1' ], + '', true, - 'KckkIVRT7cC010EHaNNNuun5VBY=' + '2jmj7l5rSw0yVb/vlWAYkK/YBwk=' ], ]; } @@ -179,18 +169,19 @@ public function getFlexibleChecksumUseCases() /** * @dataProvider getContentSha256UseCases */ - public function testAddsContentSHA256AsAppropriate($operation, $args, $hashAdded, $hashValue) + public function testAddsContentSHA256($operation, $args, $hashAdded, $hashValue) { - $s3 = $this->getTestClient('s3'); - $this->addMockResults($s3, [[]]); - $command = $s3->getCommand($operation, $args); - $command->getHandlerList()->appendBuild( - Middleware::tap(function ($cmd, RequestInterface $request) use ($hashAdded, $hashValue) { - $this->assertSame($hashAdded, $request->hasHeader('x-amz-content-sha256')); - $this->assertEquals($hashValue, $request->getHeaderLine('x-amz-content-sha256')); - }) - ); - $s3->execute($command); + $client = $this->getTestClient('s3'); + $nextHandler = function ($cmd, $request) use ($hashAdded, $hashValue) { + $this->assertSame($hashAdded, $request->hasHeader('x-amz-content-sha256')); + $this->assertEquals($hashValue, $request->getHeaderLine('x-amz-content-sha256')); + }; + $service = $client->getApi(); + $mw = new ApplyChecksumMiddleware($nextHandler, $service); + $command = $client->getCommand($operation, $args); + $request = new Request('PUT', 'foo'); + + $mw($command, $request); } public function getContentSha256UseCases() @@ -221,4 +212,38 @@ public function getContentSha256UseCases() ], ]; } + + public function testAddContentMd5EmitsDeprecationWarning() + { + $this->expectDeprecation(); + $this->expectDeprecationMessage('S3 no longer supports MD5 checksums.'); + $client = $this->getTestClient('s3'); + $nextHandler = function ($cmd, $request) { + $this->assertTrue($request->hasHeader('x-amz-checksum-crc32')); + }; + $service = $client->getApi(); + $mw = new ApplyChecksumMiddleware($nextHandler, $service); + $command = $client->getCommand('putObject', ['AddContentMD5' => true]); + $request = new Request('PUT', 'foo'); + + $mw($command, $request); + } + + public function testInvalidChecksumThrows() + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage( + 'Unsupported algorithm supplied for input variable ChecksumAlgorithm' + ); + $client = $this->getTestClient('s3'); + $nextHandler = function ($cmd, $request) { + $this->assertTrue($request->hasHeader('x-amz-checksum-crc32')); + }; + $service = $client->getApi(); + $mw = new ApplyChecksumMiddleware($nextHandler, $service); + $command = $client->getCommand('putObject', ['ChecksumAlgorithm' => 'NotAnAlgorithm']); + $request = new Request('PUT', 'foo'); + + $mw($command, $request); + } } diff --git a/tests/S3/MultipartUploaderTest.php b/tests/S3/MultipartUploaderTest.php index 6abfaa8277..50e41a28e8 100644 --- a/tests/S3/MultipartUploaderTest.php +++ b/tests/S3/MultipartUploaderTest.php @@ -163,20 +163,9 @@ public function testS3MultipartUploadParams($stream, $size) { /** @var \Aws\S3\S3Client $client */ $client = $this->getTestClient('s3'); - $client->getHandlerList()->appendSign( - Middleware::tap(function ($cmd, $req) { - $name = $cmd->getName(); - if ($name === 'UploadPart') { - $this->assertTrue( - $req->hasHeader('Content-MD5') - ); - } - }) - ); $uploadOptions = [ 'bucket' => 'foo', 'key' => 'bar', - 'add_content_md5' => true, 'params' => [ 'RequestPayer' => 'test', 'ContentLength' => $size @@ -327,7 +316,6 @@ public function testUploaderAddsFlexibleChecksums($stream, $size) $uploadOptions = [ 'bucket' => 'foo', 'key' => 'bar', - 'add_content_md5' => true, 'params' => [ 'RequestPayer' => 'test', 'ChecksumAlgorithm' => 'Sha256' @@ -363,4 +351,25 @@ public function testUploaderAddsFlexibleChecksums($stream, $size) $this->assertSame('xyz', $result['ChecksumSHA256']); $this->assertSame($url, $result['ObjectURL']); } + + public function testAddContentMd5EmitsDeprecationNotice() + { + $this->expectDeprecation(); + $this->expectDeprecationMessage('S3 no longer supports MD5 checksums.'); + $data = str_repeat('.', 12 * self::MB); + $filename = sys_get_temp_dir() . '/' . self::FILENAME; + file_put_contents($filename, $data); + $source = Psr7\Utils::streamFor(fopen($filename, 'r')); + $client = $this->getTestClient('s3'); + $options = ['bucket' => 'foo', 'key' => 'bar', 'add_content_md5' => true]; + $this->addMockResults($client, [ + new Result(['UploadId' => 'baz']), + new Result(['ETag' => 'A']), + new Result(['ETag' => 'B']), + new Result(['ETag' => 'C']), + ]); + + $uploader = new MultipartUploader($client, $source, $options); + $result = $uploader->upload(); + } } diff --git a/tests/S3/ObjectUploaderTest.php b/tests/S3/ObjectUploaderTest.php index b02b111faf..e96be36a29 100644 --- a/tests/S3/ObjectUploaderTest.php +++ b/tests/S3/ObjectUploaderTest.php @@ -10,7 +10,7 @@ use GuzzleHttp\Psr7\FnStream; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\StreamInterface; -use PHPUnit\Framework\TestCase; +use Yoast\PHPUnitPolyfills\TestCases\TestCase; class ObjectUploaderTest extends TestCase { @@ -250,7 +250,6 @@ public function testS3ObjectUploaderPutObjectParams() ); $uploadOptions = [ 'params' => ['RequestPayer' => 'test'], - 'add_content_md5' => true, 'before_upload' => function($command) { $this->assertSame('test', $command['RequestPayer']); }, @@ -283,20 +282,9 @@ public function testS3ObjectUploaderMultipartParams() { /** @var \Aws\S3\S3Client $client */ $client = $this->getTestClient('s3'); - $client->getHandlerList()->appendSign( - Middleware::tap(function ($cmd, $req) { - $name = $cmd->getName(); - if ($name === 'UploadPart') { - $this->assertTrue( - $req->hasHeader('Content-MD5') - ); - } - }) - ); $uploadOptions = [ 'mup_threshold' => self::MB * 4, 'params' => ['RequestPayer' => 'test'], - 'add_content_md5' => true, 'before_initiate' => function($command) { $this->assertSame('test', $command['RequestPayer']); }, @@ -330,4 +318,20 @@ public function testS3ObjectUploaderMultipartParams() $this->assertSame($url, $result['ObjectURL']); } + + public function testAddContentMd5EmitsDeprecationNotice() + { + $this->expectDeprecation(); + $this->expectExceptionMessage('S3 no longer supports MD5 checksums.'); + $client = $this->getTestClient('S3'); + $this->addMockResults($client, [new Result()]); + $result = (new ObjectUploader( + $client, + 'bucket', + 'key', + $this->generateStream(1024 * 1024 * 1), + 'private', + ['add_content_md5' => true] + ))->upload(); + } } diff --git a/tests/S3/Parser/ValidateResponseChecksumResultMutatorTest.php b/tests/S3/Parser/ValidateResponseChecksumResultMutatorTest.php index e38597618a..26f247b5ee 100644 --- a/tests/S3/Parser/ValidateResponseChecksumResultMutatorTest.php +++ b/tests/S3/Parser/ValidateResponseChecksumResultMutatorTest.php @@ -25,25 +25,40 @@ class ValidateResponseChecksumResultMutatorTest extends TestCase * @dataProvider checksumCasesDataProvider * @param array $responseAlgorithms * @param array $checksumHeadersReturned - * @param string $expectedChecksum - * + * @param string|null $expectedChecksumAlgorithm + * @param array $config + * @param string $operation + * @param string|null $checksumMode * @return void */ - public function testValidatesChoosesRightChecksum( + public function testChecksumValidation( array $responseAlgorithms, array $checksumHeadersReturned, - ?string $expectedChecksumAlgorithm + ?string $expectedChecksumAlgorithm, + array $config, + string $operation, + ?string $checksumMode ) { + if (!empty($responseAlgorithms) + && $responseAlgorithms[0] === 'CRC32C' + && !extension_loaded('awscrt') + ) { + $this->markTestSkipped(); + } + $s3Client = $this->getTestS3ClientWithResponseAlgorithms( 'GetObject', $responseAlgorithms ); - $mutator = new ValidateResponseChecksumResultMutator($s3Client->getApi()); + $mutator = new ValidateResponseChecksumResultMutator( + $s3Client->getApi(), + $config + ); $result = new Result(); $command = new Command( - 'GetObject', + $operation, [ - 'ChecksumMode' => 'enabled' + 'ChecksumMode' => $checksumMode ], new HandlerList() ); @@ -56,7 +71,6 @@ public function testValidatesChoosesRightChecksum( } $result = $mutator($result, $command, $response); - $this->assertEquals( $expectedChecksumAlgorithm, $result['ChecksumValidated'] @@ -72,37 +86,119 @@ public function testValidatesChoosesRightChecksum( public function checksumCasesDataProvider(): array { return [ + //Default, when_supported, no checksum headers, skips validation [ ['CRC32', 'CRC32C'], [], + null, + [], + 'GetObject', + null + ], + //Default, when_supported, no modeled checksums, skips validation + [ + [], + [], + null, + [], + 'GetObject', null ], + //Default, when_supported with modeled checksums [ ['SHA256', 'CRC32'], [ ['sha256', 'E6TOUbfBBDPqSyozecOzDgB3K9CZKCI6d7PbKBAYvo0='] ], - "SHA256" + "SHA256", + [], + 'GetObject', + null ], + //Default, when_supported with modeled checksums [ ['CRC32', 'CRC32C'], [ ["sha256", 'E6TOUbfBBDPqSyozecOzDgB3K9CZKCI6d7PbKBAYvo0='], ['crc32', 'DIt2Ng=='] ], - "CRC32" + "CRC32", + [], + 'GetObject', + null ], + //Default, when_supported with modeled checksums [ ['CRC32', 'CRC32C'], [ ['crc64', ''] ], + null, + [], + 'GetObject', null ], + //Default, when_supported, with modeled checksums, CRC32C + [ + ['CRC32C', 'CRC32'], + [ + ["crc32c", 'k2udew=='], + ], + "CRC32C", + [], + 'GetObject', + null + ], + // when_required, with modeled checksums, with mode "enabled" + [ + ['CRC32', 'CRC32C'], + [ + ["sha256", 'E6TOUbfBBDPqSyozecOzDgB3K9CZKCI6d7PbKBAYvo0='], + ['crc32', 'DIt2Ng=='] + ], + "CRC32", + ['response_checksum_validation' => 'when_required'], + 'GetObject', + 'enabled' + ], + // when_required, with modeled checksums, with mode "enabled" + [ + ['SHA256', 'CRC32'], + [ + ['sha256', 'E6TOUbfBBDPqSyozecOzDgB3K9CZKCI6d7PbKBAYvo0='] + ], + "SHA256", + ['response_checksum_validation' => 'when_required'], + 'GetObject', + 'enabled' + ], + // when_required, with modeled checksums, with mode "enabled" CRC32C + [ + ['CRC32C', 'CRC32'], + [ + ["crc32c", 'k2udew=='], + ], + "CRC32C", + ['response_checksum_validation' => 'when_required'], + 'GetObject', + 'enabled' + ], + // when_required, with modeled checksums, with mode unset, skips validation + [ + ['CRC32'], + [ + ["crc32", ''], + ], + null, + ['response_checksum_validation' => 'when_required'], + 'GetObject', + '' + ], ]; } - public function testValidatesChecksumFailsOnBadValue() { + public function testValidatesChecksumFailsOnBadValue() + { $this->expectException(S3Exception::class); $this->expectExceptionMessage( 'Calculated response checksum did not match the expected value' @@ -124,7 +220,8 @@ public function testValidatesChecksumFailsOnBadValue() { $mutator($result, $command, $response); } - public function testValidatesChecksumSucceeds() { + public function testValidatesChecksumSucceeds() + { $mutator = $this->getValidateResponseChecksumMutator(); $expectedValue = "E6TOUbfBBDPqSyozecOzDgB3K9CZKCI6d7PbKBAYvo0="; $expectedAlgorithm = "SHA256"; @@ -147,7 +244,8 @@ public function testValidatesChecksumSucceeds() { } - public function testValidatesChecksumSkipsValidation() { + public function testValidatesChecksumSkipsValidation() + { $mutator = $this->getValidateResponseChecksumMutator(); $result = new Result(); $command = new Command( @@ -163,7 +261,8 @@ public function testValidatesChecksumSkipsValidation() { $this->assertEmpty($result['ChecksumValidated']); } - public function testSkipsGetObjectReturnsFullMultipart() { + public function testSkipsGetObjectReturnsFullMultipart() + { $expectedValue = "E6TOUbfBBDPqSyozecOzDgB3K9CZKCI6d7PbK-1034"; $mutator = $this->getValidateResponseChecksumMutator(); $result = new Result(); @@ -185,7 +284,8 @@ public function testSkipsGetObjectReturnsFullMultipart() { self::assertTrue(true); } - public function testValidatesSha256() { + public function testValidatesSha256() + { $expectedValue = "E6TOUbfBBDPqSyozecOzDgB3K9CZKCI6d7PbKBAYvo0="; $mutator = $this->getValidateResponseChecksumMutator(); $result = new Result(); @@ -213,10 +313,11 @@ public function testValidatesSha256() { * @return ValidateResponseChecksumResultMutator */ private function getValidateResponseChecksumMutator( + array $config = [] ): ValidateResponseChecksumResultMutator { return new ValidateResponseChecksumResultMutator( - $this->getTestS3Client()->getApi() + $this->getTestS3Client()->getApi(), $config ); } diff --git a/tests/S3/S3ClientTest.php b/tests/S3/S3ClientTest.php index f5341557c5..84b576214e 100644 --- a/tests/S3/S3ClientTest.php +++ b/tests/S3/S3ClientTest.php @@ -1,6 +1,7 @@ expectDeprecation(); + $this->expectExceptionMessage('S3 no longer supports MD5 checksums.'); $s3 = $this->getTestClient('s3'); $this->addMockResults($s3, [[]]); $options['AddContentMD5'] = true; $command = $s3->getCommand($operation, $options); - $command->getHandlerList()->appendSign( - Middleware::tap(function ($cmd, $req) { - $this->assertSame( - 'CY9rzUYh03PK3k6DJie09g==', - $req->getHeader('Content-MD5')[0] - ); - }) - ); - $s3->execute($command); - } - - /** - * @dataProvider addMD5Provider - */ - public function testDoesNotComputeMD5($options, $operation) - { - $s3 = $this->getTestClient('s3'); - $this->addMockResults($s3, [[]]); - $options['AddContentMD5'] = false; - $command = $s3->getCommand($operation, $options); - $command->getHandlerList()->appendSign( - Middleware::tap(function ($cmd, $req) { - $this->assertFalse( - $req->hasHeader('Content-MD5') - ); - }) - ); $s3->execute($command); } @@ -2491,4 +2467,272 @@ public function testS3RetriesOnNotParsableBody(array $retrySettings) $client->listBuckets(); $this->assertEquals(0, $retries); } + + public function testAddCrc32ForDirectoryBucketsAsAppropriate() + { + $s3 = $this->getTestClient('s3'); + $this->addMockResults($s3, [[]]); + $command = $s3->getCommand('putBucketPolicy', [ + 'Bucket' => 'mybucket--use1-az1--x-s3', + 'Policy' => 'policy' + ]); + $command->getHandlerList()->appendBuild( + Middleware::tap(function ($cmd, RequestInterface $request) { + $this->assertFalse($request->hasHeader('Content-MD5')); + $this->assertSame('8H0FFg==', $request->getHeaderLine('x-amz-checksum-crc32')); + }) + ); + $s3->execute($command); + } + + /** + * @dataProvider getContentSha256UseCases + */ + public function testAddsContentSHA256AsAppropriate($operation, $args, $hashAdded, $hashValue) + { + $s3 = $this->getTestClient('s3'); + $this->addMockResults($s3, [[]]); + $command = $s3->getCommand($operation, $args); + $command->getHandlerList()->appendBuild( + Middleware::tap(function ($cmd, RequestInterface $request) use ($hashAdded, $hashValue) { + $this->assertSame($hashAdded, $request->hasHeader('x-amz-content-sha256')); + $this->assertEquals($hashValue, $request->getHeaderLine('x-amz-content-sha256')); + }) + ); + $s3->execute($command); + } + + public function getContentSha256UseCases() + { + $hash = 'SHA256HASH'; + + return [ + // Do nothing if ContentSHA256 was not provided. + [ + 'PutObject', + ['Bucket' => 'foo', 'Key' => 'bar', 'Body' => 'baz'], + false, + '' + ], + // Gets added for operations that allow it. + [ + 'PutObject', + ['Bucket' => 'foo', 'Key' => 'bar', 'Body' => 'baz', 'ContentSHA256' => $hash], + true, + $hash + ], + // Not added for operations that do not allow it. + [ + 'GetObject', + ['Bucket' => 'foo', 'Key' => 'bar', 'ContentSHA256' => $hash], + false, + '', + ], + ]; + } + + /** + * @dataProvider getFlexibleChecksumUseCases + */ + public function testAddsFlexibleChecksumAsAppropriate($operation, $clientArgs, $operationArgs, $headerAdded, $headerValue) + { + if (isset($operationArgs['ChecksumAlgorithm']) + && $operationArgs['ChecksumAlgorithm'] === 'crc32c' + && !extension_loaded('awscrt') + ) { + $this->markTestSkipped("Cannot test crc32c without the CRT"); + } + $s3 = $this->getTestClient('s3', $clientArgs); + $this->addMockResults($s3, [[]]); + $command = $s3->getCommand($operation, $operationArgs); + $command->getHandlerList()->appendBuild( + Middleware::tap(function ($cmd, RequestInterface $request) use ($headerAdded, $headerValue, $operationArgs) { + $checksumName = $operationArgs['ChecksumAlgorithm'] ?? "crc32"; + if ($headerAdded) { + $this->assertTrue($request->hasHeader("x-amz-checksum-{$checksumName}")); + } + $this->assertEquals($headerValue, $request->getHeaderLine("x-amz-checksum-{$checksumName}")); + }) + ); + $s3->execute($command); + } + + public function getFlexibleChecksumUseCases() + { + return [ + // httpChecksum not modeled + [ + 'GetObject', + [], + [ + 'Bucket' => 'foo', + 'Key' => 'bar', + 'ChecksumMode' => 'ENABLED' + ], + false, + '' + ], + // default: when_supported. defaults to crc32 + [ + 'PutObject', + [], + [ + 'Bucket' => 'foo', + 'Key' => 'bar', + 'Body' => 'abc' + ], + true, + 'NSRBwg==' + ], + // when_required when not required and no requested algorithm + [ + 'PutObject', + ['request_checksum_calculation' => 'when_required'], + [ + 'Bucket' => 'foo', + 'Key' => 'bar', + 'Body' => 'abc' + ], + false, + '' + ], + // when_required when required and no requested algorithm + [ + 'PutObjectLockConfiguration', + ['request_checksum_calculation' => 'when_required'], + [ + 'Bucket' => 'foo', + 'Key' => 'bar', + 'ObjectLockConfiguration' => [] + ], + true, + 'UHB63w==' + ], + // when_required when not required and requested algorithm + [ + 'PutObject', + ['request_checksum_calculation' => 'when_required'], + [ + 'Bucket' => 'foo', + 'Key' => 'bar', + 'Body' => 'blah', + 'ChecksumAlgorithm' => 'crc32', + ], + true, + 'zilhXA==' + ], + // when_supported and requested algorithm + [ + 'PutObject', + [], + [ + 'Bucket' => 'foo', + 'Key' => 'bar', + 'ChecksumAlgorithm' => 'crc32c', + 'Body' => 'abc' + ], + true, + 'Nks/tw==' + ], + // when_supported and requested algorithm + [ + 'PutObject', + [], + [ + 'Bucket' => 'foo', + 'Key' => 'bar', + 'ChecksumAlgorithm' => 'sha256' + ], + true, + '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=' + ], + // when_supported and requested algorithm + [ + 'PutObject', + [], + [ + 'Bucket' => 'foo', + 'Key' => 'bar', + 'ChecksumAlgorithm' => 'SHA1' + ], + true, + '2jmj7l5rSw0yVb/vlWAYkK/YBwk=' + ] + ]; + } + + /** + * @param array $clientConfig + * @return void + * + * @dataProvider responseChecksumValidationProvider + */ + public function testResponseChecksumValidation( + array $clientConfig, + ?string $checksumAlgorithm, + ?string $mode + ): void + { + if ($checksumAlgorithm === 'CRC32C' + && !extension_loaded('awscrt') + ) { + $this->markTestSkipped("Cannot test crc32c without the awscrt"); + } + + $handler = static function (RequestInterface $request) use ($checksumAlgorithm) { + return Promise\Create::promiseFor(new Response( + 200, + ['x-amz-checksum-' . $checksumAlgorithm => 'AAAAAA=='] + )); + }; + $client = $this->getTestClient('s3', $clientConfig + ['http_handler' => $handler]); + + + $result = $client->getObject([ + 'Bucket' => 'bucket', + 'Key' => 'key', + 'ChecksumMode' => $mode + ]); + + $this->assertEquals($checksumAlgorithm, $result['ChecksumValidated']); + } + + public function responseChecksumValidationProvider(): array + { + return [ + [ + //default, when_supported, validates checksum for operation with modeled response checksums + [], + 'CRC32', + null + ], + [ + //default, when_supported, validates checksum for operation with modeled response checksums when + // CRT installed + [], + 'CRC32C', + null + ], + [ + // when_required, validates checksum for operation with modeled response checksums + // and mode is "enabled" + ['response_checksum_validation' => 'when_required'], + 'CRC32', + 'enabled' + ], + [ + // when_required, validates checksum validation for operation with modeled response checksums + // and mode is "enabled" when CRT installed + ['response_checksum_validation' => 'when_required'], + 'CRC32C', + 'enabled' + ], + [ + // when_required, skips checksum validation for operation with modeled response checksums + ['response_checksum_validation' => 'when_required'], + null, + null + ], + ]; + } } diff --git a/tests/S3/TransferTest.php b/tests/S3/TransferTest.php index 170d20c4d2..5225e52463 100644 --- a/tests/S3/TransferTest.php +++ b/tests/S3/TransferTest.php @@ -146,7 +146,6 @@ function (CommandInterface $cmd, RequestInterface $req) { $t = new Transfer($s3, $dir, 's3://foo/bar', [ 'mup_threshold' => 5248000, 'debug' => $res, - 'add_content_md5' => true ]); $t->transfer(); @@ -402,6 +401,40 @@ public function testCanDownloadFilesYieldedBySourceIterator() $downloader->transfer(); } + public function testAddContentMd5EmitsDeprecationWarning() + { + $s3 = $this->getTestClient('s3'); + $this->addMockResults($s3, []); + + $this->expectDeprecation(); + $this->expectDeprecationMessage('S3 no longer supports MD5 checksums.'); + $s3->getHandlerList()->appendSign(Middleware::tap( + function (CommandInterface $cmd, RequestInterface $req) { + $this->assertTrue(isset($command['x-amz-checksum-crc32'])); + } + )); + + $dir = sys_get_temp_dir() . '/unittest'; + `rm -rf $dir`; + mkdir($dir); + $filename = $dir . '/foo.txt'; + $f = fopen($filename, 'w+'); + fwrite($f, 'foo'); + fclose($f); + + $res = fopen('php://temp', 'r+'); + $t = new Transfer($s3, $dir, 's3://foo/bar', [ + 'debug' => $res, + 'add_content_md5' => true + ]); + + $t->transfer(); + rewind($res); + $output = stream_get_contents($res); + $this->assertStringContainsString("Transferring $filename -> s3://foo/bar/foo.txt", $output); + `rm -rf $dir`; + } + private function mockResult(callable $fn) { return function (callable $handler) use ($fn) {