fix: send checksum on s3 upload (#1926)

add `x-amz-checksum-sha256` header on s3 object put requests.

This is nice as we reuse our existing sha256 hash of the car as an integrity check. AWS will calculate the sha256 of the body it receives and send us a 400 BadDigest error if they dont match, to avoid storing CARs that had bits flipped in transit.

see: web3-storage/web3.storage#1068


License: (Apache-2.0 AND MIT)
Signed-off-by: Oli Evans <oli@tableflip.io>

Author: olizilla

packages/api/src/utils/s3-backup-client.js

@@ -51,12 +51,22 @@ export class S3BackupClient {
   }
 
   /**
-   * Gets a base32 encoded sha256 hash of the passed data.
-   * @param {Uint8Array} data
+   * Gets a sha256 multihash digest of a blob.
+   * @param {Blob} blob
    */
-  async _getDataHash(data) {
-    const hash = await sha256.digest(new Uint8Array(data))
-    return uint8ArrayToString(hash.bytes, 'base32')
+  async _getMultihash(blob) {
+    const buf = await blob.arrayBuffer()
+    return sha256.digest(new Uint8Array(buf))
+  }
+
+  /**
+   * Gets base64pad-encoded string from a multihash
+   * @param {import('multiformats').digest.MultihashDigest} multihash
+   */
+  _getAwsChecksum(multihash) {
+    // strip the multihash varint prefix to get the raw sha256 digest for aws upload integrity check
+    const rawSha256 = multihash.bytes.subarray(2)
+    return uint8ArrayToString(rawSha256, 'base64pad')
   }
 
   /**
@@ -67,19 +77,36 @@ export class S3BackupClient {
    * @param {import('../bindings').DagStructure} [structure]
    */
   async backupCar(userId, rootCid, car, structure = 'Unknown') {
-    const buf = await car.arrayBuffer()
-    const dataHash = await this._getDataHash(new Uint8Array(buf))
-    const key = `raw/${rootCid}/${this._appName}-${userId}/${dataHash}.car`
-    const bucket = this._bucketName
+    const multihash = await this._getMultihash(car)
+    const hashStr = uint8ArrayToString(multihash.bytes, 'base32')
+    const key = `raw/${rootCid}/${this._appName}-${userId}/${hashStr}.car`
     const cmdParams = {
-      Bucket: bucket,
+      Bucket: this._bucketName,
       Key: key,
       Body: car,
       Metadata: { structure },
+      // ChecksumSHA256 specifies the base64-encoded, 256-bit SHA-256 digest of the object, used as a data integrity check to verify that the data received is the same data that was originally sent.
+      // see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html#AmazonS3-PutObject-request-header-ChecksumSHA256
+      ChecksumSHA256: this._getAwsChecksum(multihash),
     }
     /** @type {import('@aws-sdk/client-s3').PutObjectCommand} */
-    const cmd = new PutObjectCommand(cmdParams)
-    await this._s3.send(cmd)
+
+    try {
+      await this._s3.send(new PutObjectCommand(cmdParams))
+    } catch (/** @type {any} */ err) {
+      if (err.name === 'BadDigest') {
+        // s3 returns a 400 Bad Request `BadDigest` error if the hash does not match their calculation.
+        // see: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#RESTErrorResponses
+        // see: https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/index.html#troubleshooting
+        console.log(
+          'BadDigest: sha256 of data recieved did not match what we sent. Maybe bits flipped in transit. Retrying once.'
+        )
+        await this._s3.send(new PutObjectCommand(cmdParams))
+      } else {
+        throw err
+      }
+    }
+
     return new URL(key, this._baseUrl.toString())
   }
 }