Add support for S3 express endpoints

Author: adam-fowler

Sources/SotoCore/HTTP/AWSHTTPBody.swift

@@ -74,6 +74,15 @@ public struct AWSHTTPBody: Sendable {
         }
     }
 
+    public var isEmpty: Bool {
+        switch self.storage {
+        case .byteBuffer(let buffer):
+            return buffer.readableBytes == 0
+        case .asyncSequence(_, let length):
+            return length == 0
+        }
+    }
+
     public var isStreaming: Bool {
         switch self.storage {
         case .byteBuffer:

Sources/SotoCore/Middleware/Middleware/S3Middleware.swift

@@ -68,60 +68,51 @@ public struct S3Middleware: AWSMiddlewareProtocol {
         if request.url.path.hasPrefix("/arn:") {
             return try await handleARNBucket(request, context: context, next: next)
         }
-        /// process URL into form ${bucket}.s3.amazon.com
+        /// split path into components. Drop first as it is an empty string
         let paths = request.url.path.split(separator: "/", maxSplits: 2, omittingEmptySubsequences: false).dropFirst()
         guard let bucket = paths.first, var host = request.url.host else { return try await next(request, context) }
+        let path = paths.dropFirst().first.flatMap { String($0) } ?? ""
 
         if let port = request.url.port {
             host = "\(host):\(port)"
         }
         var urlPath: String
         var urlHost: String
-        let isAmazonUrl = host.hasSuffix("amazonaws.com")
+        let isAmazonUrl = host.hasSuffix(context.serviceConfig.region.partition.dnsSuffix)
 
-        var hostComponents = host.split(separator: ".")
-        if isAmazonUrl, context.serviceConfig.options.contains(.s3UseTransferAcceleratedEndpoint) {
-            if let s3Index = hostComponents.firstIndex(where: { $0 == "s3" }) {
-                var s3 = "s3"
-                s3 += "-accelerate"
-                // assume next host component is region
-                let regionIndex = s3Index + 1
-                hostComponents.remove(at: regionIndex)
-                hostComponents[s3Index] = Substring(s3)
-                host = hostComponents.joined(separator: ".")
-            }
-        }
+        // if bucket has suffix "-x-s3" then it must be an s3 express directory bucket and the endpoint needs set up
+        // to use s3express
+        if bucket.hasSuffix("--x-s3"), let host = getS3ExpressBucketEndpoint(bucket: bucket, path: path, context: context) {
+            urlHost = host
+            urlPath = path
+        } else {
 
-        // Is bucket an ARN
-        if bucket.hasPrefix("arn:") {
-            guard let arn = ARN(string: bucket),
-                let resourceType = arn.resourceType,
-                let region = arn.region,
-                let accountId = arn.accountId
-            else {
-                throw AWSClient.ClientError.invalidARN
-            }
-            guard resourceType == "accesspoint", arn.service == "s3-object-lambda" || arn.service == "s3-outposts" else {
-                throw AWSClient.ClientError.invalidARN
+            // Should we use accelerated endpoint
+            var hostComponents = host.split(separator: ".")
+            if isAmazonUrl, context.serviceConfig.options.contains(.s3UseTransferAcceleratedEndpoint) {
+                if let s3Index = hostComponents.firstIndex(where: { $0 == "s3" }) {
+                    var s3 = "s3"
+                    s3 += "-accelerate"
+                    // assume next host component is region
+                    let regionIndex = s3Index + 1
+                    hostComponents.remove(at: regionIndex)
+                    hostComponents[s3Index] = Substring(s3)
+                    host = hostComponents.joined(separator: ".")
+                }
             }
-            urlPath = "/"
-            // https://tutorial-object-lambda-accesspoint-123456789012.s3-object-lambda.us-west-2.amazonaws.com:443
-            urlHost = "\(arn.resourceId)-\(resourceType)-\(accountId).\(arn.service).\(region).amazonaws.com"
 
-            // if host name contains amazonaws.com and bucket name doesn't contain a period do virtual address look up
-        } else if isAmazonUrl || context.serviceConfig.options.contains(.s3ForceVirtualHost), !bucket.contains(".") {
-            let pathsWithoutBucket = paths.dropFirst()  // bucket
-            urlPath = pathsWithoutBucket.first.flatMap { String($0) } ?? ""
-
-            if hostComponents.first == bucket {
-                // Bucket name is part of host. No need to append bucket
-                urlHost = host
+            if isAmazonUrl || context.serviceConfig.options.contains(.s3ForceVirtualHost), !bucket.contains(".") {
+                urlPath = path
+                if hostComponents.first == bucket {
+                    // Bucket name is part of host. No need to append bucket
+                    urlHost = host
+                } else {
+                    urlHost = "\(bucket).\(host)"
+                }
             } else {
-                urlHost = "\(bucket).\(host)"
+                urlPath = paths.joined(separator: "/")
+                urlHost = host
             }
-        } else {
-            urlPath = paths.joined(separator: "/")
-            urlHost = host
         }
         let request = Self.updateRequestURL(request, host: urlHost, path: urlPath)
         return try await next(request, context)
@@ -155,7 +146,7 @@ public struct S3Middleware: AWSMiddlewareProtocol {
         let path = String(resourceIDSplit.dropFirst().first ?? "")
         let service = String(arn.service)
         let serviceIdentifier = service != "s3" ? service : "s3-accesspoint"
-        let urlHost = "\(bucket)-\(accountId).\(serviceIdentifier).\(region).amazonaws.com"
+        let urlHost = "\(bucket)-\(accountId).\(serviceIdentifier).\(region).\(context.serviceConfig.region.partition.dnsSuffix)"
         let request = Self.updateRequestURL(request, host: urlHost, path: path)
 
         var context = context
@@ -177,11 +168,29 @@ public struct S3Middleware: AWSMiddlewareProtocol {
         return try await next(request, context)
     }
 
+    ///  Handle bucket names in the form of an ARN
+    /// - Parameters:
+    ///   - request: request
+    ///   - context: request context
+    ///   - next: next handler
+    /// - Returns: returns response from next handler
+    func getS3ExpressBucketEndpoint(
+        bucket: Substring,
+        path: String,
+        context: AWSMiddlewareContext
+    ) -> String? {
+        // Note this uses my own version of split (as the Swift one requires macOS 13)
+        let split = bucket.split(separator: "--")
+        guard split.count > 2, split.last == "x-s3" else { return nil }
+        let zone = split[split.count - 2]
+        return "\(bucket).s3express-\(zone).\(context.serviceConfig.region).\(context.serviceConfig.region.partition.dnsSuffix)"
+    }
+
     ///  Update request with new host and path
     /// - Parameters:
     ///   - request: request
     ///   - host: new host name
-    ///   - path: new path
+    ///   - path: new path (without forward slash prefix)
     /// - Returns: new request
     static func updateRequestURL(_ request: AWSHTTPRequest, host: some StringProtocol, path: String) -> AWSHTTPRequest {
         var path = path
@@ -210,16 +219,18 @@ public struct S3Middleware: AWSMiddlewareProtocol {
         switch context.operation {
         // fixup CreateBucket to include location
         case "CreateBucket":
-            var xml = ""
-            if context.serviceConfig.region != .useast1 {
-                xml += "<CreateBucketConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
-                xml += "<LocationConstraint>"
-                xml += context.serviceConfig.region.rawValue
-                xml += "</LocationConstraint>"
-                xml += "</CreateBucketConfiguration>"
+            if request.body.isEmpty {
+                var xml = ""
+                if context.serviceConfig.region != .useast1 {
+                    xml += "<CreateBucketConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
+                    xml += "<LocationConstraint>"
+                    xml += context.serviceConfig.region.rawValue
+                    xml += "</LocationConstraint>"
+                    xml += "</CreateBucketConfiguration>"
+                }
+                // TODO: pass service config down so we can use the ByteBufferAllocator
+                request.body = .init(string: xml)
             }
-            // TODO: pass service config down so we can use the ByteBufferAllocator
-            request.body = .init(string: xml)
 
         default:
             break
@@ -263,3 +274,35 @@ public struct S3Middleware: AWSMiddlewareProtocol {
         return error
     }
 }
+
+extension StringProtocol {
+    func split(separator: some StringProtocol) -> [Self.SubSequence] {
+        var split: [Self.SubSequence] = []
+        var index: Self.Index = self.startIndex
+        var startSplit: Self.Index = self.startIndex
+        while index != self.endIndex {
+            if let end = self[index...].prefixEnd(separator) {
+                split.append(self[startSplit..<index])
+                startSplit = end
+                index = end
+            } else {
+                index = self.index(after: index)
+            }
+        }
+        split.append(self[startSplit..<index])
+        return split
+    }
+
+    func prefixEnd(_ prefix: some StringProtocol) -> Self.Index? {
+        var prefixIndex = prefix.startIndex
+        var index = self.startIndex
+        while prefixIndex != prefix.endIndex {
+            if self[index] != prefix[prefixIndex] {
+                return nil
+            }
+            index = self.index(after: index)
+            prefixIndex = prefix.index(after: prefixIndex)
+        }
+        return index
+    }
+}

Tests/SotoCoreTests/MiddlewareTests.swift

@@ -218,6 +218,18 @@ class MiddlewareTests: XCTestCase {
         }
     }
 
+    func testS3MiddlewareS3ExpressEndpoint() async throws {
+        // Test virual address
+        try await self.testMiddleware(
+            S3Middleware(),
+            serviceName: "s3",
+            serviceOptions: .s3UseTransferAcceleratedEndpoint,
+            uri: "/s3express--bucket--use1-az6--x-s3/file"
+        ) { request, _ in
+            XCTAssertEqual(request.url.absoluteString, "https://s3express--bucket--use1-az6--x-s3.s3express-use1-az6.us-east-1.amazonaws.com/file")
+        }
+    }
+
     // create a buffer of random values. Will always create the same given you supply the same z and w values
     // Random number generator from https://www.codeproject.com/Articles/25172/Simple-Random-Number-Generation
     func createRandomBuffer(_ w: UInt, _ z: UInt, size: Int) -> [UInt8] {