Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

enhancement: flexible checksums #2999

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

enhancement: flexible checksums #2999

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ccae080
enhancement: flexible checksums
Sep 17, 2024
abe5426
Merge branch 'master' into checksums-v2
stobrien89 Oct 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changes/nextrelease/checksumsv2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[
{
"type": "feature",
"category": "S3",
"description": "Adds a default checksum of CRC32 to operations that support checksums. Adds additional configuration for request checksum calculation and response checksum validation."
}
]
202 changes: 142 additions & 60 deletions src/S3/ApplyChecksumMiddleware.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
namespace Aws\S3;

use Aws\Api\Service;
use Aws\Api\Shape;
use Aws\CommandInterface;
use GuzzleHttp\Psr7;
use InvalidArgumentException;
use Psr\Http\Message\RequestInterface;
use Psr\Http\Message\StreamInterface;
Expand All @@ -18,14 +18,28 @@
class ApplyChecksumMiddleware
{
use CalculatesChecksumTrait;
private static $sha256AndMd5 = [
'PutObject',
'UploadPart',

public const DEFAULT_CALCULATION_MODE = 'when_supported';
public const DEFAULT_ALGORITHM = 'crc32';

/**
* @var true[]
*
* S3 Operations for which pre-calculated SHA256
* Checksums can be added to the command
*/
private static $sha256 = [
'PutObject' => true,
'UploadPart' => true,
yenfryherrerafeliz marked this conversation as resolved.
Show resolved Hide resolved
];

/** @var Service */
private $api;

/** @var array */
private $config;

/** @var callable */
private $nextHandler;

/**
Expand All @@ -34,17 +48,22 @@ class ApplyChecksumMiddleware
* @param Service $api
* @return callable
*/
public static function wrap(Service $api)
public static function wrap(Service $api, array $config = [])
{
return function (callable $handler) use ($api) {
return new self($handler, $api);
return function (callable $handler) use ($api, $config) {
return new self($handler, $api, $config);
};
}

public function __construct(callable $nextHandler, Service $api)
public function __construct(
callable $nextHandler,
Service $api,
array $config = []
)
{
$this->api = $api;
$this->nextHandler = $nextHandler;
$this->config = $config;
}

public function __invoke(
Expand All @@ -54,59 +73,43 @@ public function __invoke(
$next = $this->nextHandler;
$name = $command->getName();
$body = $request->getBody();
$operation = $this->api->getOperation($name);
$mode = $this->config['request_checksum_calculation']
?? self::DEFAULT_CALCULATION_MODE;

//Checks if AddContentMD5 has been specified for PutObject or UploadPart
$addContentMD5 = $command['AddContentMD5'] ?? null;

$op = $this->api->getOperation($command->getName());
// Trigger warning if AddContentMD5 is specified for PutObject or UploadPart
$this->handleDeprecatedAddContentMD5($command);

$checksumInfo = $op['httpChecksum'] ?? [];
$checksumMemberName = array_key_exists('requestAlgorithmMember', $checksumInfo)
? $checksumInfo['requestAlgorithmMember']
: "";
$checksumInfo = $operation['httpChecksum'] ?? [];
$checksumMemberName = $checksumInfo['requestAlgorithmMember'] ?? '';
$checksumMember = !empty($checksumMemberName)
? $operation->getInput()->getMember($checksumMemberName)
: null;
$checksumRequired = $checksumInfo['requestChecksumRequired'] ?? false;
$requestedAlgorithm = $command[$checksumMemberName] ?? null;
if (!empty($checksumMemberName) && !empty($requestedAlgorithm)) {
$requestedAlgorithm = strtolower($requestedAlgorithm);
$checksumMember = $op->getInput()->getMember($checksumMemberName);
$supportedAlgorithms = isset($checksumMember['enum'])
? array_map('strtolower', $checksumMember['enum'])
: null;
if (is_array($supportedAlgorithms)
&& in_array($requestedAlgorithm, $supportedAlgorithms)
) {
$request = $this->addAlgorithmHeader($requestedAlgorithm, $request, $body);
} else {
throw new InvalidArgumentException(
"Unsupported algorithm supplied for input variable {$checksumMemberName}."
. " Supported checksums for this operation include: "
. implode(", ", $supportedAlgorithms) . "."
);
}
return $next($command, $request);
}

if (!empty($checksumInfo)) {
//if the checksum member is absent, check if it's required
$checksumRequired = $checksumInfo['requestChecksumRequired'] ?? null;
if ((!empty($checksumRequired))
|| (in_array($name, self::$sha256AndMd5) && $addContentMD5)
) {
//S3Express doesn't support MD5; default to crc32 instead
if ($this->isS3Express($command)) {
$request = $this->addAlgorithmHeader('crc32', $request, $body);
} elseif (!$request->hasHeader('Content-MD5')) {
// Set the content MD5 header for operations that require it.
$request = $request->withHeader(
'Content-MD5',
base64_encode(Psr7\Utils::hash($body, 'md5', true))
);
}
return $next($command, $request);
$shouldAddChecksum = $this->shouldAddChecksum(
$mode,
$checksumRequired,
$checksumMember,
$requestedAlgorithm
);
if ($shouldAddChecksum) {
if (!$this->hasAlgorithmHeader($request)) {
$supportedAlgorithms = isset($checksumMember['enum'])
? array_map('strtolower', $checksumMember['enum'])
: [];
Comment on lines +99 to +101
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: I think this could be simplified to:

$supportedAlgorithms =  array_map('strtolower', $checksumMember['enum'] ?? []);

$algorithm = $this->determineChecksumAlgorithm(
$supportedAlgorithms,
$requestedAlgorithm,
$checksumMemberName
);
$request = $this->addAlgorithmHeader($algorithm, $request, $body);
}
}

if (in_array($name, self::$sha256AndMd5) && $command['ContentSHA256']) {
// Set the content hash header if provided in the parameters.
// Set the content hash header if ContentSHA256 is provided
if (isset(self::$sha256[$name]) && $command['ContentSHA256']) {
$request = $request->withHeader(
'X-Amz-Content-Sha256',
$command['ContentSHA256']
Expand All @@ -116,32 +119,111 @@ public function __invoke(
return $next($command, $request);
}

/**
* @param CommandInterface $command
*
* @return void
*/
private function handleDeprecatedAddContentMD5(CommandInterface $command): void
{
if (!empty($command['AddContentMD5'])) {
trigger_error(
'S3 no longer supports MD5 checksums. ' .
'A CRC32 checksum will be computed and applied on your behalf.',
E_USER_DEPRECATED
);
$command['ChecksumAlgorithm'] = self::DEFAULT_ALGORITHM;
}
}

/**
* @param string $mode
* @param Shape|null $checksumMember
* @param string $name
* @param bool $checksumRequired
* @param string|null $requestedAlgorithm
*
* @return bool
*/
private function shouldAddChecksum(
string $mode,
bool $checksumRequired,
?Shape $checksumMember,
?string $requestedAlgorithm
): bool
{
return ($mode === 'when_supported' && $checksumMember)
|| ($mode === 'when_required'
&& ($checksumRequired || ($checksumMember && $requestedAlgorithm)));
}

/**
* @param Shape|null $checksumMember
* @param string|null $requestedAlgorithm
* @param string|null $checksumMemberName
*
* @return string
*/
private function determineChecksumAlgorithm(
stobrien89 marked this conversation as resolved.
Show resolved Hide resolved
array $supportedAlgorithms,
?string $requestedAlgorithm,
?string $checksumMemberName
): string
{
$algorithm = self::DEFAULT_ALGORITHM;

if ($requestedAlgorithm) {
$requestedAlgorithm = strtolower($requestedAlgorithm);
if (!in_array($requestedAlgorithm, $supportedAlgorithms)) {
throw new InvalidArgumentException(
"Unsupported algorithm supplied for input variable {$checksumMemberName}. " .
"Supported checksums for this operation include: "
. implode(", ", $supportedAlgorithms) . "."
);
}
$algorithm = $requestedAlgorithm;
}

return $algorithm;
}

/**
* @param string $requestedAlgorithm
* @param RequestInterface $request
* @param StreamInterface $body
*
* @return RequestInterface
*/
private function addAlgorithmHeader(
string $requestedAlgorithm,
RequestInterface $request,
StreamInterface $body
) {
): RequestInterface
{
$headerName = "x-amz-checksum-{$requestedAlgorithm}";
if (!$request->hasHeader($headerName)) {
$encoded = $this->getEncodedValue($requestedAlgorithm, $body);
$encoded = self::getEncodedValue($requestedAlgorithm, $body);
$request = $request->withHeader($headerName, $encoded);
}

return $request;
}

/**
* @param CommandInterface $command
* @param RequestInterface $request
*
* @return bool
*/
private function isS3Express(CommandInterface $command): bool
private function hasAlgorithmHeader(RequestInterface $request): bool
{
return isset($command['@context']['signing_service'])
&& $command['@context']['signing_service'] === 's3express';
$headers = $request->getHeaders();

foreach ($headers as $name => $values) {
if (stripos($name, 'x-amz-checksum-') === 0) {
return true;
}
}

return false;
}
}
49 changes: 29 additions & 20 deletions src/S3/CalculatesChecksumTrait.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,13 @@

trait CalculatesChecksumTrait
{
private static $supportedAlgorithms = [
'crc32c' => true,
'crc32' => true,
'sha256' => true,
'sha1' => true
];

/**
* @param string $requestedAlgorithm the algorithm to encode with
* @param string $value the value to be encoded
Expand All @@ -16,35 +23,37 @@ trait CalculatesChecksumTrait
public static function getEncodedValue($requestedAlgorithm, $value) {
$requestedAlgorithm = strtolower($requestedAlgorithm);
$useCrt = extension_loaded('awscrt');
if ($useCrt) {
$crt = new Crt();
switch ($requestedAlgorithm) {
case 'crc32c':
return base64_encode(pack('N*',($crt->crc32c($value))));
case 'crc32':
return base64_encode(pack('N*',($crt->crc32($value))));
case 'sha256':
case 'sha1':
return base64_encode(Psr7\Utils::hash($value, $requestedAlgorithm, true));
default:
break;
throw new InvalidArgumentException(
"Invalid checksum requested: {$requestedAlgorithm}."
. " Valid algorithms are CRC32C, CRC32, SHA256, and SHA1."
);

if (isset(self::$supportedAlgorithms[$requestedAlgorithm])) {
if ($useCrt) {
$crt = new Crt();
switch ($requestedAlgorithm) {
case 'crc32c':
return base64_encode(pack('N*',($crt::crc32c($value))));
case 'crc32':
return base64_encode(pack('N*',($crt::crc32($value))));
default:
break;
}
}
} else {
if ($requestedAlgorithm == 'crc32c') {

if ($requestedAlgorithm === 'crc32c') {
throw new CommonRuntimeException("crc32c is not supported for checksums "
. "without use of the common runtime for php. Please enable the CRT or choose "
. "a different algorithm."
);
}
if ($requestedAlgorithm == "crc32") {

if ($requestedAlgorithm === "crc32") {
$requestedAlgorithm = "crc32b";
}
return base64_encode(Psr7\Utils::hash($value, $requestedAlgorithm, true));
}
}

$validAlgorithms = implode(', ', array_keys(self::$supportedAlgorithms));
throw new InvalidArgumentException(
"Invalid checksum requested: {$requestedAlgorithm}."
. " Valid algorithms are {$validAlgorithms}."
);
Comment on lines +54 to +57
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is another place that validates if the checksum is valid or not. For example here. I know one validates if is valid based on what the operation supports and the other one validates if is supported by the SDK in specific. Would not be worth to make that more clear here?, like state why it is not supported.
For example:

  throw new InvalidArgumentException(
      "Invalid checksum requested: {$requestedAlgorithm}."
      . "  Valid algorithms supported by this runtime are {$validAlgorithms}."
  );

}
}
8 changes: 7 additions & 1 deletion src/S3/MultipartUploadingTrait.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,16 @@ protected function handleResult(CommandInterface $command, ResultInterface $resu
$partData = [];
$partData['PartNumber'] = $command['PartNumber'];
$partData['ETag'] = $this->extractETag($result);
$commandName = $command->getName();
$checksumResult = $commandName === 'UploadPart'
? $result
: $result[$commandName . 'Result'];

if (isset($command['ChecksumAlgorithm'])) {
$checksumMemberName = 'Checksum' . strtoupper($command['ChecksumAlgorithm']);
$partData[$checksumMemberName] = $result[$checksumMemberName];
$partData[$checksumMemberName] = $checksumResult[$checksumMemberName] ?? null;
}

$this->getState()->markPartAsUploaded($command['PartNumber'], $partData);
}

Expand Down
Loading