Smithy RPCv2 CBOR merge (#5621)

* Add RPCv2 module

* Add RPCv2 module (#5445)

* Comment out empty rpcv2 dependency

* Sync version to 2.26.31-SNAPSHOT

* Sugmanue/add byte support (#5477)

* Add support to serialize byte values

* Add tests for byte support

* Address PR comments

* Add rpcv2 protocol core (#5496)

* Add support to serialize byte values

* Add RPCv2 protocol core marshalling/unmarshalling

* Address PR comments

* Address PR comments 2

* Address PR comments 3

* Support for operation without input defined (#5512)

* Support for operation without input defined

* Fix a checkstyle issue

* Code clean up

* Code clean up 2

* Rewrite the condition to conjunctive normal form

* Add codgen tests (#5517)

* Add codgen tests

* Address PR comments

* Address PR comments 2

* Add missing class rename

* Add missing AWS_JSON protocol facts

* Account for null protocol case

* Add RPCv2 benchmark tests (#5526)

* Add RPCv2 benchmark tests

* Give the constants name a meaningful name

* Avoid parsing numbers when using RPCv2 protocol (#5539)

* Avoid parsing numbers when using RPCv2 protocol

* Refactor to avoid impacting JSON with RPCv2 logic (#5544)

* Refactor to avoid impacting JSON with RPCv2 logic

* Avoid making the unmarshallers depend on timestamp formats

* Avoid streams while unmarshalling

* Fix build failures

* Fix build failures 2

* Avoid growing copies of collections of known size (#5551)

* Add the new Smithy RPCv2 package

* Sugmanue/rpcv2 improve cbor performance 04 (#5564)

* Improve lookup by marshalling type

* Improve trait lookup using TraitType

* Add support for Smithy RPCv2 to the new service scripts (#5613)

* Add changelog for the release

* Fix typo in changelog

* Update to next SNAPSHOT version

.brazil.json

@@ -14,6 +14,7 @@
         "aws-json-protocol": { "packageName": "AwsJavaSdk-Core-AwsJsonProtocol" },
         "aws-query-protocol": { "packageName": "AwsJavaSdk-Core-AwsQueryProtocol" },
         "aws-xml-protocol": { "packageName": "AwsJavaSdk-Core-AwsXmlProtocol" },
+        "smithy-rpcv2-protocol": { "packageName": "AwsJavaSdk-Core-SmithyRpcV2Protocol" },
         "cloudwatch-metric-publisher": { "packageName": "AwsJavaSdk-MetricPublisher-CloudWatch" },
         "codegen": { "packageName": "AwsJavaSdk-Codegen" },
         "dynamodb-enhanced": { "packageName": "AwsJavaSdk-DynamoDb-Enhanced" },

.changes/next-release/feature-AWSSDKforJavav2-fea638f.json

@@ -0,0 +1,6 @@
+{
+    "category": "AWS SDK for Java v2", 
+    "contributor": "sugmanue", 
+    "type": "feature", 
+    "description": "Added support for the Smithy RPCv2 CBOR protocol, a new RPC protocol with better performance characteristics than AWS Json."
+}

bom/pom.xml

@@ -112,6 +112,11 @@
                 <artifactId>aws-xml-protocol</artifactId>
                 <version>${awsjavasdk.version}</version>
             </dependency>
+            <dependency>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>smithy-rpcv2-protocol</artifactId>
+                <version>${awsjavasdk.version}</version>
+            </dependency>
             <dependency>
                 <groupId>software.amazon.awssdk</groupId>
                 <artifactId>protocol-core</artifactId>

codegen/pom.xml

@@ -142,6 +142,11 @@
             <artifactId>aws-xml-protocol</artifactId>
             <version>${awsjavasdk.version}</version>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>smithy-rpcv2-protocol</artifactId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>protocol-core</artifactId>

codegen/src/main/java/software/amazon/awssdk/codegen/AddEmptyInputShape.java

@@ -15,7 +15,7 @@
 
 package software.amazon.awssdk.codegen;
 
-import static software.amazon.awssdk.codegen.internal.Utils.createInputShapeMarshaller;
+import static software.amazon.awssdk.codegen.internal.Utils.createSyntheticInputShapeMarshaller;
 import static software.amazon.awssdk.codegen.internal.Utils.unCapitalize;
 
 import java.util.HashMap;
@@ -75,7 +75,7 @@ private Map<String, ShapeModel> addEmptyInputShapes(
                 shape.setVariable(inputVariable);
 
                 shape.setMarshaller(
-                        createInputShapeMarshaller(serviceModel.getMetadata(), operation));
+                        createSyntheticInputShapeMarshaller(serviceModel.getMetadata(), operation));
 
                 emptyInputShapes.put(inputShape, shape);
 

codegen/src/main/java/software/amazon/awssdk/codegen/customization/processors/DefaultCustomizationProcessor.java

@@ -34,6 +34,7 @@ public static CodegenCustomizationProcessor getProcessorFor(
                 new ShapeSubstitutionsProcessor(config.getShapeSubstitutions()),
                 new CustomSdkShapesProcessor(config.getCustomSdkShapes()),
                 new OperationModifiersProcessor(config.getOperationModifiers()),
+                new SmithyRpcV2CborProtocolProcessor(),
                 new RemoveExceptionMessagePropertyProcessor(),
                 new UseLegacyEventGenerationSchemeProcessor(),
                 new NewAndLegacyEventStreamProcessor(),

codegen/src/main/java/software/amazon/awssdk/codegen/customization/processors/SmithyRpcV2CborProtocolProcessor.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.codegen.customization.processors;
+
+import software.amazon.awssdk.codegen.customization.CodegenCustomizationProcessor;
+import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
+import software.amazon.awssdk.codegen.model.service.Http;
+import software.amazon.awssdk.codegen.model.service.Operation;
+import software.amazon.awssdk.codegen.model.service.ServiceModel;
+import software.amazon.awssdk.utils.StringUtils;
+
+/**
+ * This processor only runs for services using the <code>smithy-rpc-v2-cbor</code> protocol.
+ *
+ * Adds a request URI that conform to the Smithy RPCv2 protocol to each operation in the model, if there's no URI already
+ * defined.
+ */
+public class SmithyRpcV2CborProtocolProcessor implements CodegenCustomizationProcessor {
+    @Override
+    public void preprocess(ServiceModel serviceModel) {
+        if (!"smithy-rpc-v2-cbor".equals(serviceModel.getMetadata().getProtocol())) {
+            return;
+        }
+        serviceModel.getOperations().forEach((name, op) -> setRequestUri(serviceModel, name, op));
+    }
+
+    private void setRequestUri(ServiceModel service, String name, Operation op) {
+        Http http = op.getHttp();
+        String requestUri = http.getRequestUri();
+        if (StringUtils.isNotBlank(requestUri) && !"/".equals(requestUri)) {
+            return;
+        }
+        String uri = String.format("/service/%s/operation/%s", service.getMetadata().getTargetPrefix(), op.getName());
+        op.getHttp().setRequestUri(uri);
+    }
+
+    @Override
+    public void postprocess(IntermediateModel intermediateModel) {
+    }
+}

codegen/src/main/java/software/amazon/awssdk/codegen/internal/DefaultProtocolMetadataConstants.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.codegen.internal;
+
+import java.util.AbstractMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import software.amazon.awssdk.protocols.core.OperationMetadataAttribute;
+import software.amazon.awssdk.utils.AttributeMap;
+
+/**
+ * Default implementation of {@link ProtocolMetadataConstants}.
+ */
+public final class DefaultProtocolMetadataConstants implements ProtocolMetadataConstants {
+    private final Set<Map.Entry<Class<?>, OperationMetadataAttribute<?>>> knownKeys = new LinkedHashSet<>();
+    private final AttributeMap.Builder map = AttributeMap.builder();
+
+    @Override
+    public List<Map.Entry<Class<?>, OperationMetadataAttribute<?>>> keys() {
+        return knownKeys.stream().filter(x -> map.get(x.getValue()) != null).collect(Collectors.toList());
+    }
+
+    @Override
+    public <T> T put(Class<?> containingClass, OperationMetadataAttribute<T> key, T value) {
+        knownKeys.add(new AbstractMap.SimpleEntry<>(containingClass, key));
+        T oldValue = map.get(key);
+        map.put(key, value);
+        return oldValue;
+    }
+
+    @Override
+    public <T> T get(OperationMetadataAttribute<T> key) {
+        return map.get(key);
+    }
+}

codegen/src/main/java/software/amazon/awssdk/codegen/internal/ProtocolMetadataConstants.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.codegen.internal;
+
+import java.util.List;
+import java.util.Map;
+import software.amazon.awssdk.protocols.core.OperationMetadataAttribute;
+
+/**
+ * Keeps the set of {@link OperationMetadataAttribute} constants attributes per operation/protocol. This is used to codegen
+ * those constant values.
+ */
+public interface ProtocolMetadataConstants {
+
+    /**
+     * Returns the list of keys sets. The {@link Map.Entry} contains as key the class containing the key field and the value
+     * contains the key constant itself. The class is needed to properly codegen a reference to the key.
+     * @return
+     */
+    List<Map.Entry<Class<?>, OperationMetadataAttribute<?>>> keys();
+
+    /**
+     * Adds an operation metadata to the set of constants.
+     */
+    <T> T put(Class<?> containingClass, OperationMetadataAttribute<T> key, T value);
+
+    /**
+     * Adds an operation metadata to the set of constants.
+     */
+    default <T> T put(OperationMetadataAttribute<T> key, T value) {
+        return put(key.getClass(), key, value);
+    }
+
+    /**
+     * Gets the constant value for the operation metadata key.
+     */
+    <T> T get(OperationMetadataAttribute<T> key);
+}

codegen/src/main/java/software/amazon/awssdk/codegen/internal/ProtocolMetadataDefault.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.codegen.internal;
+
+import software.amazon.awssdk.codegen.model.intermediate.Protocol;
+import software.amazon.awssdk.codegen.model.intermediate.ShapeMarshaller;
+import software.amazon.awssdk.protocols.json.BaseAwsJsonProtocolFactory;
+
+/**
+ * Enum that maps protocol to metadata attribute constants for a given operation.
+ */
+public enum ProtocolMetadataDefault {
+
+    SMITHY_RPC_V2_CBOR(Protocol.SMITHY_RPC_V2_CBOR) {
+        public ProtocolMetadataConstants protocolMetadata(ShapeMarshaller shapeMarshaller) {
+            ProtocolMetadataConstants attributes = new DefaultProtocolMetadataConstants();
+
+            // If the shape is synthetic that means that no-input was defined in the model. For this
+            // case the protocol requires to send an empty body with no content-type. See
+            // https://smithy.io/2.0/additional-specs/protocols/smithy-rpc-v2.html#requests.
+            // To accomplish this we use a no-op JSON generator. Otherwise, we serialize the input
+            // even when no members are defined.
+            Boolean isSynthetic = shapeMarshaller.getIsSynthetic();
+            if (Boolean.TRUE.equals(isSynthetic)) {
+                attributes.put(BaseAwsJsonProtocolFactory.class,
+                               BaseAwsJsonProtocolFactory.GENERATES_BODY,
+                               Boolean.FALSE);
+            }
+            return attributes;
+        }
+    },
+    DEFAULT(null);
+
+    private final Protocol protocol;
+
+    ProtocolMetadataDefault(Protocol protocol) {
+        this.protocol = protocol;
+    }
+
+    /**
+     * Returns a function that maps from a {@link ShapeMarshaller} to a set of protocol metadata constants that we codegen.
+     */
+    public ProtocolMetadataConstants protocolMetadata(ShapeMarshaller shapeMarshaller) {
+        return new DefaultProtocolMetadataConstants();
+    }
+
+    public static ProtocolMetadataDefault from(Protocol protocol) {
+        for (ProtocolMetadataDefault value : values()) {
+            if (value.protocol == protocol) {
+                return value;
+            }
+        }
+        return DEFAULT;
+    }
+}

codegen/src/main/java/software/amazon/awssdk/codegen/internal/TypeUtils.java

@@ -85,6 +85,7 @@ public static final class TypeKey {
         MARSHALLING_TYPE_MAPPINGS.put("BigDecimal", "BIG_DECIMAL");
         MARSHALLING_TYPE_MAPPINGS.put("InputStream", "STREAM");
         MARSHALLING_TYPE_MAPPINGS.put("Short", "SHORT");
+        MARSHALLING_TYPE_MAPPINGS.put("Byte", "BYTE");
         MARSHALLING_TYPE_MAPPINGS.put(null, "NULL");
         MARSHALLING_TYPE_MAPPINGS.put("Document", "DOCUMENT");
     }

codegen/src/main/java/software/amazon/awssdk/codegen/internal/Utils.java

@@ -334,7 +334,8 @@ public static ShapeMarshaller createInputShapeMarshaller(ServiceMetadata service
         ShapeMarshaller marshaller = new ShapeMarshaller()
                 .withAction(operation.getName())
                 .withVerb(operation.getHttp().getMethod())
-                .withRequestUri(operation.getHttp().getRequestUri());
+                .withRequestUri(operation.getHttp().getRequestUri())
+                .withProtocol(service.getProtocol());
         Input input = operation.getInput();
         if (input != null) {
             marshaller.setLocationName(input.getLocationName());
@@ -344,12 +345,22 @@ public static ShapeMarshaller createInputShapeMarshaller(ServiceMetadata service
                 marshaller.setXmlNameSpaceUri(xmlNamespace.getUri());
             }
         }
-        if (Metadata.isNotRestProtocol(service.getProtocol())) {
+        if (Metadata.usesOperationIdentifier(service.getProtocol())) {
             marshaller.setTarget(StringUtils.isEmpty(service.getTargetPrefix()) ?
                                  operation.getName() :
                                  service.getTargetPrefix() + "." + operation.getName());
         }
         return marshaller;
 
     }
+
+    /**
+     * Create the ShapeMarshaller to the input shape from the specified Operation.
+     * The input shape in the operation could be empty.
+     */
+    public static ShapeMarshaller createSyntheticInputShapeMarshaller(ServiceMetadata service, Operation operation) {
+        ShapeMarshaller result = createInputShapeMarshaller(service, operation);
+        result.withIsSynthetic(true);
+        return result;
+    }
 }

codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/Metadata.java

@@ -547,8 +547,13 @@ public boolean isCborProtocol() {
         return protocol == Protocol.CBOR;
     }
 
+    public boolean isRpcV2CborProtocol() {
+        return protocol == Protocol.SMITHY_RPC_V2_CBOR;
+    }
+
     public boolean isJsonProtocol() {
-        return protocol == Protocol.CBOR ||
+        return protocol == Protocol.SMITHY_RPC_V2_CBOR ||
+               protocol == Protocol.CBOR ||
                protocol == Protocol.AWS_JSON ||
                protocol == Protocol.REST_JSON;
     }
@@ -565,12 +570,13 @@ public boolean isQueryProtocol() {
     }
 
     /**
-     * @return True for RESTful protocols. False for all other protocols (RPC, Query, etc).
+     * @return True for protocols that require an operation identifier to be sent in the `X-Amz-Target` header.
      */
-    public static boolean isNotRestProtocol(String protocol) {
+    public static boolean usesOperationIdentifier(String protocol) {
         switch (Protocol.fromValue(protocol)) {
             case REST_JSON:
             case REST_XML:
+            case SMITHY_RPC_V2_CBOR:
                 return false;
             default:
                 return true;

codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/Protocol.java

@@ -24,7 +24,8 @@ public enum Protocol {
     REST_JSON("rest-json"),
     CBOR("cbor"),
     QUERY("query"),
-    REST_XML("rest-xml");
+    REST_XML("rest-xml"),
+    SMITHY_RPC_V2_CBOR("smithy-rpc-v2-cbor");
 
     private String protocol;