Add support for S3 arn based access points (#627)

adam-fowler · web-flow · commit 52c639250274 · 2024-12-12T17:56:57.000Z
* Add parsing of ARNs as S3 buckets

* Add support for using ARN based S3 access points

* Explicitly set service config instead of using `.with`

as endpoints in original service are wrong point to `s3`.

* Fix 5.x compile error

* Changes requested in PR

Add comment to ARN type, removed commented out code

* More comments
diff --git a/Sources/SotoCore/AWSClient.swift b/Sources/SotoCore/AWSClient.swift
@@ -186,6 +186,7 @@ public final class AWSClient: Sendable {
             case waiterFailed
             case waiterTimeout
             case failedToAccessPayload
+            case invalidARN
         }
 
         let error: Error
@@ -202,6 +203,8 @@ public final class AWSClient: Sendable {
         public static var waiterTimeout: ClientError { .init(error: .waiterTimeout) }
         /// Failed to access payload while building request
         public static var failedToAccessPayload: ClientError { .init(error: .failedToAccessPayload) }
+        /// ARN provided to client is invalid
+        public static var invalidARN: ClientError { .init(error: .invalidARN) }
     }
 
     /// Additional options
@@ -550,6 +553,8 @@ extension AWSClient.ClientError: CustomStringConvertible {
             return "Waiter failed to complete in time allocated"
         case .failedToAccessPayload:
             return "Failed to access payload while building request for AWS"
+        case .invalidARN:
+            return "ARN provided to the client was invalid"
         }
     }
 }
diff --git a/Sources/SotoCore/AWSServiceConfig.swift b/Sources/SotoCore/AWSServiceConfig.swift
@@ -298,8 +298,10 @@ public final class AWSServiceConfig {
         service: AWSServiceConfig,
         with patch: Patch
     ) {
-        let region = patch.region ?? service.region
-        let options = patch.options ?? service.options
+        self.region = patch.region ?? service.region
+        self.options = patch.options ?? service.options
+        self.serviceIdentifier = service.serviceIdentifier
+        self.signingName = service.signingName
 
         if let endpoint = patch.endpoint {
             self.endpoint = endpoint
@@ -308,23 +310,19 @@ public final class AWSServiceConfig {
                 patch.endpoint
                 ?? Self.getEndpoint(
                     endpoint: service.providedEndpoint,
-                    region: region,
-                    serviceIdentifier: service.serviceIdentifier,
-                    options: options,
+                    region: self.region,
+                    serviceIdentifier: self.serviceIdentifier,
+                    options: self.options,
                     serviceEndpoints: service.serviceEndpoints,
                     partitionEndpoints: service.partitionEndpoints,
                     variantEndpoints: service.variantEndpoints
                 )
         } else {
             self.endpoint = service.endpoint
         }
-        self.region = region
-        self.options = options
 
         self.amzTarget = service.amzTarget
         self.serviceName = service.serviceName
-        self.serviceIdentifier = service.serviceIdentifier
-        self.signingName = service.signingName
         self.serviceProtocol = service.serviceProtocol
         self.apiVersion = service.apiVersion
         self.providedEndpoint = service.providedEndpoint
diff --git a/Sources/SotoCore/Doc/ARN.swift b/Sources/SotoCore/Doc/ARN.swift
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Soto for AWS open source project
+//
+// Copyright (c) 2017-2024 the Soto project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of Soto project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+/// Amazon Resource Name (ARN). A unique identifier assigned to AWS resource
+///
+/// Comes in one of the following forms
+/// - arn:partition:service:region:account-id:resource-id
+/// - arn:partition:service:region:account-id:resource-type/resource-id
+/// - arn:partition:service:region:account-id:resource-type:resource-id
+public struct ARN {
+    public init?<S: StringProtocol>(string: S) where S.SubSequence == Substring {
+        let split = string.split(separator: ":", omittingEmptySubsequences: false)
+        guard split.count >= 6 else { return nil }
+        guard split[0] == "arn" else { return nil }
+        guard let partition = AWSPartition(rawValue: String(split[1])) else { return nil }
+        self.partition = partition
+        self.service = split[2]
+        self.region = split[3].count > 0 ? Region(rawValue: String(split[3])) : nil
+        if let region {
+            guard region.partition == self.partition else { return nil }
+        }
+        self.accountId = split[4].count > 0 ? split[4] : nil
+        guard self.accountId?.first(where: { !$0.isNumber }) == nil else { return nil }
+        if split.count == 6 {
+            let resourceSplit = split[5].split(separator: "/", maxSplits: 1)
+            if resourceSplit.count == 1 {
+                self.resourceType = nil
+                self.resourceId = resourceSplit[0]
+            } else {
+                self.resourceType = resourceSplit[0]
+                self.resourceId = resourceSplit[1]
+            }
+        } else if split.count == 7 {
+            self.resourceType = split[5]
+            self.resourceId = split[6]
+        } else {
+            return nil
+        }
+    }
+
+    public let partition: AWSPartition
+    public let service: Substring
+    public let region: Region?
+    public let accountId: Substring?
+    public let resourceId: Substring
+    public let resourceType: Substring?
+}
diff --git a/Sources/SotoCore/Middleware/Middleware.swift b/Sources/SotoCore/Middleware/Middleware.swift
@@ -15,7 +15,7 @@
 import Logging
 
 /// Context object sent to `AWSMiddlewareProtocol` `handle` functions
-public struct AWSMiddlewareContext {
+public struct AWSMiddlewareContext: Sendable {
     public var operation: String
     public var serviceConfig: AWSServiceConfig
     public var logger: Logger
diff --git a/Sources/SotoCore/Middleware/Middleware/S3Middleware.swift b/Sources/SotoCore/Middleware/Middleware/S3Middleware.swift
@@ -37,87 +37,173 @@ internal import SotoXML
 /// - Creates error body for notFound responses to HEAD requests
 public struct S3Middleware: AWSMiddlewareProtocol {
     public func handle(_ request: AWSHTTPRequest, context: AWSMiddlewareContext, next: AWSMiddlewareNextHandler) async throws -> AWSHTTPResponse {
-        var request = request
 
-        self.virtualAddressFixup(request: &request, context: context)
-        self.createBucketFixup(request: &request, context: context)
-        if !context.serviceConfig.options.contains(.s3Disable100Continue) {
-            self.expect100Continue(request: &request)
-        }
+        try await self.handleVirtualAddressFixup(request, context: context) { request, context in
+            var request = request
+            self.createBucketFixup(request: &request, context: context)
+            if !context.serviceConfig.options.contains(.s3Disable100Continue) {
+                self.expect100Continue(request: &request)
+            }
 
-        do {
-            var response = try await next(request, context)
-            if context.operation == "GetBucketLocation" {
-                self.getBucketLocationResponseFixup(response: &response)
+            do {
+                var response = try await next(request, context)
+                if context.operation == "GetBucketLocation" {
+                    self.getBucketLocationResponseFixup(response: &response)
+                }
+                return response
+            } catch let error as AWSRawError {
+                let newError = self.fixupRawError(error: error, context: context)
+                throw newError
             }
-            return response
-        } catch let error as AWSRawError {
-            let newError = self.fixupRawError(error: error, context: context)
-            throw newError
         }
     }
 
     public init() {}
 
-    func virtualAddressFixup(request: inout AWSHTTPRequest, context: AWSMiddlewareContext) {
+    func handleVirtualAddressFixup(
+        _ request: AWSHTTPRequest,
+        context: AWSMiddlewareContext,
+        next: AWSMiddlewareNextHandler
+    ) async throws -> AWSHTTPResponse {
+        if request.url.path.hasPrefix("/arn:") {
+            return try await handleARNBucket(request, context: context, next: next)
+        }
         /// process URL into form ${bucket}.s3.amazon.com
-        let paths = request.url.path.split(separator: "/", omittingEmptySubsequences: true)
-        if paths.count > 0 {
-            guard var host = request.url.host else { return }
-            if let port = request.url.port {
-                host = "\(host):\(port)"
+        let paths = request.url.path.split(separator: "/", maxSplits: 2, omittingEmptySubsequences: false).dropFirst()
+        guard let bucket = paths.first, var host = request.url.host else { return try await next(request, context) }
+
+        if let port = request.url.port {
+            host = "\(host):\(port)"
+        }
+        var urlPath: String
+        var urlHost: String
+        let isAmazonUrl = host.hasSuffix("amazonaws.com")
+
+        var hostComponents = host.split(separator: ".")
+        if isAmazonUrl, context.serviceConfig.options.contains(.s3UseTransferAcceleratedEndpoint) {
+            if let s3Index = hostComponents.firstIndex(where: { $0 == "s3" }) {
+                var s3 = "s3"
+                s3 += "-accelerate"
+                // assume next host component is region
+                let regionIndex = s3Index + 1
+                hostComponents.remove(at: regionIndex)
+                hostComponents[s3Index] = Substring(s3)
+                host = hostComponents.joined(separator: ".")
             }
-            let bucket = paths[0]
-            var urlPath: String
-            var urlHost: String
-            let isAmazonUrl = host.hasSuffix("amazonaws.com")
-
-            var hostComponents = host.split(separator: ".")
-            if isAmazonUrl, context.serviceConfig.options.contains(.s3UseTransferAcceleratedEndpoint) {
-                if let s3Index = hostComponents.firstIndex(where: { $0 == "s3" }) {
-                    var s3 = "s3"
-                    s3 += "-accelerate"
-                    // assume next host component is region
-                    let regionIndex = s3Index + 1
-                    hostComponents.remove(at: regionIndex)
-                    hostComponents[s3Index] = Substring(s3)
-                    host = hostComponents.joined(separator: ".")
-                }
+        }
+
+        // Is bucket an ARN
+        if bucket.hasPrefix("arn:") {
+            guard let arn = ARN(string: bucket),
+                let resourceType = arn.resourceType,
+                let region = arn.region,
+                let accountId = arn.accountId
+            else {
+                throw AWSClient.ClientError.invalidARN
+            }
+            guard resourceType == "accesspoint", arn.service == "s3-object-lambda" || arn.service == "s3-outposts" else {
+                throw AWSClient.ClientError.invalidARN
             }
+            urlPath = "/"
+            // https://tutorial-object-lambda-accesspoint-123456789012.s3-object-lambda.us-west-2.amazonaws.com:443
+            urlHost = "\(arn.resourceId)-\(resourceType)-\(accountId).\(arn.service).\(region).amazonaws.com"
 
             // if host name contains amazonaws.com and bucket name doesn't contain a period do virtual address look up
-            if isAmazonUrl || context.serviceConfig.options.contains(.s3ForceVirtualHost), !bucket.contains(".") {
-                let pathsWithoutBucket = paths.dropFirst()  // bucket
-                urlPath = pathsWithoutBucket.joined(separator: "/")
-
-                if hostComponents.first == bucket {
-                    // Bucket name is part of host. No need to append bucket
-                    urlHost = host
-                } else {
-                    urlHost = "\(bucket).\(host)"
-                }
-            } else {
-                urlPath = paths.joined(separator: "/")
+        } else if isAmazonUrl || context.serviceConfig.options.contains(.s3ForceVirtualHost), !bucket.contains(".") {
+            let pathsWithoutBucket = paths.dropFirst()  // bucket
+            urlPath = pathsWithoutBucket.first.flatMap { String($0) } ?? ""
+
+            if hostComponents.first == bucket {
+                // Bucket name is part of host. No need to append bucket
                 urlHost = host
+            } else {
+                urlHost = "\(bucket).\(host)"
             }
-            // add trailing "/" back if it was present, no need to check for single slash path here
-            if request.url.pathWithSlash.hasSuffix("/") {
-                urlPath += "/"
-            }
-            // add percent encoding back into path as converting from URL to String has removed it
-            let percentEncodedUrlPath = Self.urlEncodePath(urlPath)
-            var urlString = "\(request.url.scheme ?? "https")://\(urlHost)/\(percentEncodedUrlPath)"
-            if let query = request.url.query {
-                urlString += "?\(query)"
-            }
-            request.url = URL(string: urlString)!
+        } else {
+            urlPath = paths.joined(separator: "/")
+            urlHost = host
+        }
+        let request = Self.updateRequestURL(request, host: urlHost, path: urlPath)
+        return try await next(request, context)
+    }
+
+    ///  Handle bucket names in the form of an ARN
+    /// - Parameters:
+    ///   - request: request
+    ///   - context: request context
+    ///   - next: next handler
+    /// - Returns: returns response from next handler
+    func handleARNBucket(
+        _ request: AWSHTTPRequest,
+        context: AWSMiddlewareContext,
+        next: AWSMiddlewareNextHandler
+    ) async throws -> AWSHTTPResponse {
+        guard let arn = ARN(string: request.url.path.dropFirst()),
+            let resourceType = arn.resourceType,
+            let accountId = arn.accountId
+        else {
+            throw AWSClient.ClientError.invalidARN
+        }
+        let region = arn.region ?? context.serviceConfig.region
+        guard resourceType == "accesspoint", arn.service == "s3-object-lambda" || arn.service == "s3-outposts" || arn.service == "s3" else {
+            throw AWSClient.ClientError.invalidARN
         }
+
+        // extract bucket and path from ARN
+        let resourceIDSplit = arn.resourceId.split(separator: "/", maxSplits: 1, omittingEmptySubsequences: false)
+        guard let bucket = resourceIDSplit.first else { throw AWSClient.ClientError.invalidARN }
+        let path = String(resourceIDSplit.dropFirst().first ?? "")
+        let service = String(arn.service)
+        let serviceIdentifier = service != "s3" ? service : "s3-accesspoint"
+        let urlHost = "\(bucket)-\(accountId).\(serviceIdentifier).\(region).amazonaws.com"
+        let request = Self.updateRequestURL(request, host: urlHost, path: path)
+
+        var context = context
+        context.serviceConfig = AWSServiceConfig(
+            region: region,
+            partition: region.partition,
+            serviceName: "S3",
+            serviceIdentifier: serviceIdentifier,
+            signingName: service,
+            serviceProtocol: context.serviceConfig.serviceProtocol,
+            apiVersion: context.serviceConfig.apiVersion,
+            errorType: context.serviceConfig.errorType,
+            xmlNamespace: context.serviceConfig.xmlNamespace,
+            middleware: context.serviceConfig.middleware,
+            timeout: context.serviceConfig.timeout,
+            byteBufferAllocator: context.serviceConfig.byteBufferAllocator,
+            options: context.serviceConfig.options
+        )
+        return try await next(request, context)
+    }
+
+    ///  Update request with new host and path
+    /// - Parameters:
+    ///   - request: request
+    ///   - host: new host name
+    ///   - path: new path
+    /// - Returns: new request
+    static func updateRequestURL(_ request: AWSHTTPRequest, host: some StringProtocol, path: String) -> AWSHTTPRequest {
+        var path = path
+        // add trailing "/" back if it was present, no need to check for single slash path here
+        if request.url.pathWithSlash.hasSuffix("/") {
+            path += "/"
+        }
+        // add percent encoding back into path as converting from URL to String has removed it
+        let percentEncodedUrlPath = Self.urlEncodePath(path)
+        var urlString = "\(request.url.scheme ?? "https")://\(host)/\(percentEncodedUrlPath)"
+        if let query = request.url.query {
+            urlString += "?\(query)"
+        }
+        var request = request
+        request.url = URL(string: urlString)!
+        return request
     }
 
     static let s3PathAllowedCharacters = CharacterSet.urlPathAllowed.subtracting(.init(charactersIn: "+@()&$=:,'!*"))
     /// percent encode path value.
-    private static func urlEncodePath(_ value: String) -> String {
-        value.addingPercentEncoding(withAllowedCharacters: Self.s3PathAllowedCharacters) ?? value
+    private static func urlEncodePath(_ value: some StringProtocol) -> String {
+        value.addingPercentEncoding(withAllowedCharacters: Self.s3PathAllowedCharacters) ?? String(value)
     }
 
     func createBucketFixup(request: inout AWSHTTPRequest, context: AWSMiddlewareContext) {
diff --git a/Tests/SotoCoreTests/Doc/ARNTests.swift b/Tests/SotoCoreTests/Doc/ARNTests.swift
diff --git a/Tests/SotoCoreTests/MiddlewareTests.swift b/Tests/SotoCoreTests/MiddlewareTests.swift

Original file line number	Diff line number	Diff line change
`@@ -186,6 +186,7 @@ public final class AWSClient: Sendable {`
`186`	`186`	`case waiterFailed`
`187`	`187`	`case waiterTimeout`
`188`	`188`	`case failedToAccessPayload`
	`189`	`+ case invalidARN`
`189`	`190`	`}`
`190`	`191`
`191`	`192`	`let error: Error`
`@@ -202,6 +203,8 @@ public final class AWSClient: Sendable {`
`202`	`203`	`public static var waiterTimeout: ClientError { .init(error: .waiterTimeout) }`
`203`	`204`	`/// Failed to access payload while building request`
`204`	`205`	`public static var failedToAccessPayload: ClientError { .init(error: .failedToAccessPayload) }`
	`206`	`+ /// ARN provided to client is invalid`
	`207`	`+ public static var invalidARN: ClientError { .init(error: .invalidARN) }`
`205`	`208`	`}`
`206`	`209`
`207`	`210`	`/// Additional options`
`@@ -550,6 +553,8 @@ extension AWSClient.ClientError: CustomStringConvertible {`
`550`	`553`	`return "Waiter failed to complete in time allocated"`
`551`	`554`	`case .failedToAccessPayload:`
`552`	`555`	`return "Failed to access payload while building request for AWS"`
	`556`	`+ case .invalidARN:`
	`557`	`+ return "ARN provided to the client was invalid"`
`553`	`558`	`}`
`554`	`559`	`}`
`555`	`560`	`}`