enhancement: flexible checksums

aws · Sep 18, 2024 · 22fb848 · 22fb848
1 parent c64ee32
commit 22fb848
Show file tree

Hide file tree

Showing 15 changed files with 1,224 additions and 396 deletions.
diff --git a/.changes/nextrelease/checksumsv2.json b/.changes/nextrelease/checksumsv2.json
@@ -0,0 +1,7 @@
+[
+  {
+    "type": "enhancement",
+    "category": "S3",
+    "description": "Adds additional config to "
+  }
+]
diff --git a/src/S3/ApplyChecksumMiddleware.php b/src/S3/ApplyChecksumMiddleware.php
@@ -2,8 +2,8 @@
 namespace Aws\S3;
 
 use Aws\Api\Service;
+use Aws\Api\Shape;
 use Aws\CommandInterface;
-use GuzzleHttp\Psr7;
 use InvalidArgumentException;
 use Psr\Http\Message\RequestInterface;
 use Psr\Http\Message\StreamInterface;
@@ -18,14 +18,28 @@
 class ApplyChecksumMiddleware
 {
     use CalculatesChecksumTrait;
-    private static $sha256AndMd5 = [
-        'PutObject',
-        'UploadPart',
+
+    public const DEFAULT_CALCULATION_MODE = 'when_supported';
+    public const DEFAULT_ALGORITHM = 'crc32';
+
+    /**
+     * @var true[]
+     *
+     * S3 Operations for which pre-calculated SHA256
+     * Checksums can be added to the command
+     */
+    private static $sha256 = [
+        'PutObject' => true,
+        'UploadPart' => true,
     ];
 
     /** @var Service */
     private $api;
 
+    /** @var array */
+    private $config;
+
+    /** @var callable */
     private $nextHandler;
 
     /**
@@ -34,17 +48,22 @@ class ApplyChecksumMiddleware
      * @param Service $api
      * @return callable
      */
-    public static function wrap(Service $api)
+    public static function wrap(Service $api, array $config = [])
     {
-        return function (callable $handler) use ($api) {
-            return new self($handler, $api);
+        return function (callable $handler) use ($api, $config) {
+            return new self($handler, $api, $config);
         };
     }
 
-    public function __construct(callable $nextHandler, Service $api)
+    public function __construct(
+        callable $nextHandler,
+        Service $api, array
+        $config = []
+    )
     {
         $this->api = $api;
         $this->nextHandler = $nextHandler;
+        $this->config = $config;
     }
 
     public function __invoke(
@@ -54,59 +73,39 @@ public function __invoke(
         $next = $this->nextHandler;
         $name = $command->getName();
         $body = $request->getBody();
+        $op = $this->api->getOperation($name);
+        $mode = $this->config['request_checksum_calculation']
+            ?? self::DEFAULT_CALCULATION_MODE;
 
-        //Checks if AddContentMD5 has been specified for PutObject or UploadPart
-        $addContentMD5 = $command['AddContentMD5'] ?? null;
-
-        $op = $this->api->getOperation($command->getName());
+        // Trigger warning if AddContentMD5 is specified for PutObject or UploadPart
+        $this->handleDeprecatedAddContentMD5($command);
 
         $checksumInfo = $op['httpChecksum'] ?? [];
-        $checksumMemberName = array_key_exists('requestAlgorithmMember', $checksumInfo)
-            ? $checksumInfo['requestAlgorithmMember']
-            : "";
+        $checksumMemberName = $checksumInfo['requestAlgorithmMember'] ?? '';
+        $checksumMember = !empty($checksumMemberName)
+            ? $op->getInput()->getMember($checksumMemberName)
+            : null;
+        $checksumRequired = $checksumInfo['requestChecksumRequired'] ?? false;
         $requestedAlgorithm = $command[$checksumMemberName] ?? null;
-        if (!empty($checksumMemberName) && !empty($requestedAlgorithm)) {
-            $requestedAlgorithm = strtolower($requestedAlgorithm);
-            $checksumMember = $op->getInput()->getMember($checksumMemberName);
-            $supportedAlgorithms = isset($checksumMember['enum'])
-                ? array_map('strtolower', $checksumMember['enum'])
-                : null;
-            if (is_array($supportedAlgorithms)
-                && in_array($requestedAlgorithm, $supportedAlgorithms)
-            ) {
-                $request = $this->addAlgorithmHeader($requestedAlgorithm, $request, $body);
-            } else {
-                throw new InvalidArgumentException(
-                    "Unsupported algorithm supplied for input variable {$checksumMemberName}."
-                    . "  Supported checksums for this operation include: "
-                    . implode(", ", $supportedAlgorithms) . "."
-                );
-            }
-            return $next($command, $request);
-        }
 
-        if (!empty($checksumInfo)) {
-        //if the checksum member is absent, check if it's required
-        $checksumRequired = $checksumInfo['requestChecksumRequired'] ?? null;
-            if ((!empty($checksumRequired))
-                || (in_array($name, self::$sha256AndMd5) && $addContentMD5)
-            ) {
-                //S3Express doesn't support MD5; default to crc32 instead
-                if ($this->isS3Express($command)) {
-                    $request = $this->addAlgorithmHeader('crc32', $request, $body);
-                } elseif (!$request->hasHeader('Content-MD5')) {
-                    // Set the content MD5 header for operations that require it.
-                    $request = $request->withHeader(
-                        'Content-MD5',
-                        base64_encode(Psr7\Utils::hash($body, 'md5', true))
-                    );
-                }
-                return $next($command, $request);
-            }
+        $shouldAddChecksum = $this->shouldAddChecksum(
+            $mode,
+            $checksumMember,
+            $name,
+            $checksumRequired,
+            $requestedAlgorithm
+        );
+        if ($shouldAddChecksum) {
+            $algorithm = $this->determineChecksumAlgorithm(
+                $checksumMember,
+                $requestedAlgorithm,
+                $checksumMemberName
+            );
+            $request = $this->addAlgorithmHeader($algorithm, $request, $body);
         }
 
-        if (in_array($name, self::$sha256AndMd5) && $command['ContentSHA256']) {
-            // Set the content hash header if provided in the parameters.
+        // Set the content hash header if ContentSHA256 is provided
+        if (isset(self::$sha256[$name]) && $command['ContentSHA256']) {
             $request = $request->withHeader(
                 'X-Amz-Content-Sha256',
                 $command['ContentSHA256']
@@ -116,10 +115,83 @@ public function __invoke(
         return $next($command, $request);
     }
 
+    /**
+     * @param CommandInterface $command
+     *
+     * @return void
+     */
+    private function handleDeprecatedAddContentMD5(CommandInterface $command): void
+    {
+        if (!empty($command['AddContentMD5'])) {
+            trigger_error(
+                'S3 no longer supports MD5 checksums. ' .
+                'A CRC32 checksum will be computed and applied on your behalf.',
+                E_USER_DEPRECATED
+            );
+            $command['ChecksumAlgorithm'] = self::DEFAULT_ALGORITHM;
+        }
+    }
+
+    /**
+     * @param string $mode
+     * @param Shape|null $checksumMember
+     * @param string $name
+     * @param bool $checksumRequired
+     * @param string|null $requestedAlgorithm
+     *
+     * @return bool
+     */
+    private function shouldAddChecksum(
+        string $mode,
+        ?Shape $checksumMember,
+        string $name,
+        bool $checksumRequired,
+        ?string $requestedAlgorithm
+    ): bool
+    {
+        return ($mode === 'when_supported' && $checksumMember)
+            || ($mode === 'when_required'
+                && ($checksumRequired || ($checksumMember && $requestedAlgorithm)));
+    }
+
+    /**
+     * @param Shape|null $checksumMember
+     * @param string|null $requestedAlgorithm
+     * @param string|null $checksumMemberName
+     *
+     * @return string
+     */
+    private function determineChecksumAlgorithm(
+        ?Shape $checksumMember,
+        ?string $requestedAlgorithm,
+        ?string $checksumMemberName
+    ): string
+    {
+        $algorithm = self::DEFAULT_ALGORITHM;
+        $supportedAlgorithms = isset($checksumMember['enum'])
+            ? array_map('strtolower', $checksumMember['enum'])
+            : [];
+
+        if ($requestedAlgorithm) {
+            $requestedAlgorithm = strtolower($requestedAlgorithm);
+            if (!in_array($requestedAlgorithm, $supportedAlgorithms)) {
+                throw new InvalidArgumentException(
+                    "Unsupported algorithm supplied for input variable {$checksumMemberName}. " .
+                    "Supported checksums for this operation include: "
+                    . implode(", ", $supportedAlgorithms) . "."
+                );
+            }
+            $algorithm = $requestedAlgorithm;
+        }
+
+        return $algorithm;
+    }
+
     /**
      * @param string $requestedAlgorithm
      * @param RequestInterface $request
      * @param StreamInterface $body
+     *
      * @return RequestInterface
      */
     private function addAlgorithmHeader(
@@ -129,19 +201,10 @@ private function addAlgorithmHeader(
     ) {
         $headerName = "x-amz-checksum-{$requestedAlgorithm}";
         if (!$request->hasHeader($headerName)) {
-            $encoded = $this->getEncodedValue($requestedAlgorithm, $body);
+            $encoded = self::getEncodedValue($requestedAlgorithm, $body);
             $request = $request->withHeader($headerName, $encoded);
         }
-        return $request;
-    }
 
-    /**
-     * @param CommandInterface $command
-     * @return bool
-     */
-    private function isS3Express(CommandInterface $command): bool
-    {
-        return isset($command['@context']['signing_service'])
-            && $command['@context']['signing_service'] === 's3express';
+        return $request;
     }
 }
diff --git a/src/S3/CalculatesChecksumTrait.php b/src/S3/CalculatesChecksumTrait.php
@@ -8,6 +8,13 @@
 
 trait CalculatesChecksumTrait
 {
+    private static $supportedAlgorithms = [
+        'crc32c' => true,
+        'crc32' => true,
+        'sha256' => true,
+        'sha1' => true
+    ];
+
     /**
      * @param string $requestedAlgorithm  the algorithm to encode with
      * @param string $value               the value to be encoded
@@ -16,35 +23,36 @@ trait CalculatesChecksumTrait
     public static function getEncodedValue($requestedAlgorithm, $value) {
         $requestedAlgorithm = strtolower($requestedAlgorithm);
         $useCrt = extension_loaded('awscrt');
-        if ($useCrt) {
-            $crt = new Crt();
-            switch ($requestedAlgorithm) {
-                case 'crc32c':
-                    return base64_encode(pack('N*',($crt->crc32c($value))));
-                case 'crc32':
-                    return base64_encode(pack('N*',($crt->crc32($value))));
-                case 'sha256':
-                case 'sha1':
-                    return base64_encode(Psr7\Utils::hash($value, $requestedAlgorithm, true));
-                default:
-                    break;
-                throw new InvalidArgumentException(
-                    "Invalid checksum requested: {$requestedAlgorithm}."
-                    . "  Valid algorithms are CRC32C, CRC32, SHA256, and SHA1."
-                );
+
+        if (array_key_exists($requestedAlgorithm, self::$supportedAlgorithms)) {
+            if ($useCrt) {
+                $crt = new Crt();
+                switch ($requestedAlgorithm) {
+                    case 'crc32c':
+                        return base64_encode(pack('N*',($crt::crc32c($value))));
+                    case 'crc32':
+                        return base64_encode(pack('N*',($crt::crc32($value))));
+                    default:
+                        break;
+                }
             }
-        }  else {
-            if ($requestedAlgorithm == 'crc32c') {
+
+            if ($requestedAlgorithm === 'crc32c') {
                 throw new CommonRuntimeException("crc32c is not supported for checksums "
                     . "without use of the common runtime for php.  Please enable the CRT or choose "
                     . "a different algorithm."
                 );
             }
-            if ($requestedAlgorithm == "crc32") {
+
+            if ($requestedAlgorithm === "crc32") {
                 $requestedAlgorithm = "crc32b";
             }
             return base64_encode(Psr7\Utils::hash($value, $requestedAlgorithm, true));
         }
+        //TODO add tests for this
+        throw new InvalidArgumentException(
+            "Invalid checksum requested: {$requestedAlgorithm}."
+            . "  Valid algorithms are CRC32C, CRC32, SHA256, and SHA1."
+        );
     }
-
 }
diff --git a/src/S3/MultipartUploader.php b/src/S3/MultipartUploader.php
@@ -70,6 +70,8 @@ public function __construct(
             'key'    => null,
             'exception_class' => S3MultipartUploadException::class,
         ]);
+
+        //TODO move
     }
 
     protected function loadUploadWorkflowInfo()

diff --git a/src/S3/MultipartUploadingTrait.php b/src/S3/MultipartUploadingTrait.php
@@ -54,10 +54,17 @@ protected function handleResult(CommandInterface $command, ResultInterface $resu
         $partData = [];
         $partData['PartNumber'] = $command['PartNumber'];
         $partData['ETag'] = $this->extractETag($result);
+
+        $checksumResult = $this instanceof MultipartCopy
+            ? $result['CopyPartResult']
+            : $result;
         if (isset($command['ChecksumAlgorithm'])) {
             $checksumMemberName = 'Checksum' . strtoupper($command['ChecksumAlgorithm']);
-            $partData[$checksumMemberName] = $result[$checksumMemberName];
+            $partData[$checksumMemberName] = $checksumResult[$checksumMemberName];
+        } else {
+            $this->applyChecksumToResult($checksumResult, $partData);
         }
+
         $this->getState()->markPartAsUploaded($command['PartNumber'], $partData);
     }
 
@@ -114,6 +121,15 @@ protected function getInitiateParams()
         return $params;
     }
 
+    protected function applyChecksumToResult($result, array &$partData): void
+    {
+        foreach($result as $key => $value) {
+            if (!empty($value) && strpos($key, 'Checksum') === 0) {
+                $partData[$key] = $value;
+            }
+        }
+    }
+
     /**
      * @return UploadState
      */