Merge branch 'main' into enable-pkcs12-test

.builder/actions/build_gg_samples.py

@@ -0,0 +1,37 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+
+import Builder
+import os
+
+
+class BuildGGSamples(Builder.Action):
+    """
+    Defines a custom build step for Greengrass samples.
+    It's used via builder.json config file in the project root directory.
+    """
+
+    def run(self, env):
+        # parse extra cmake configs
+        cmd_args = env.args
+
+        steps = []
+        samples = [
+            'samples/greengrass/ipc',
+            'samples/greengrass/basic_discovery',
+        ]
+
+        for sample_path in samples:
+            build_path = os.path.join('build', sample_path)
+            steps.append(['cmake',
+                          f'-B{build_path}',
+                          f'-H{sample_path}',
+                          f'-DCMAKE_PREFIX_PATH={env.install_dir}',
+                          '-DCMAKE_BUILD_TYPE=RelWithDebInfo'])
+            # append extra cmake configs
+            steps[-1].extend(cmd_args.cmake_extra)
+            steps.append(['cmake',
+                          '--build', build_path,
+                          '--config', 'RelWithDebInfo'])
+
+        return Builder.Script(steps)

.builder/actions/build_samples.py

@@ -1,10 +1,17 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+
 import Builder
 import os
 import sys
-import argparse
 
 
 class BuildSamples(Builder.Action):
+    """
+    Defines a custom build step for samples and tests.
+    It's used via builder.json config file in the project root directory.
+    """
+
     def run(self, env):
         # parse extra cmake configs
         cmd_args = env.args

.github/workflows/ci.yml

@@ -37,6 +37,7 @@ env:
   CI_X509_ROLE: arn:aws:iam::180635532705:role/CI_X509_Role
   CI_MQTT5_ROLE: arn:aws:iam::180635532705:role/CI_MQTT5_Role
   CI_GREENGRASS_ROLE: arn:aws:iam::180635532705:role/CI_Greengrass_Role
+  CI_GREENGRASS_INSTALLER_ROLE: arn:aws:iam::180635532705:role/CI_GreengrassInstaller_Role
   CI_JOBS_SERVICE_CLIENT_ROLE: arn:aws:iam::180635532705:role/CI_JobsServiceClient_Role
   CI_SHADOW_SERVICE_CLIENT_ROLE: arn:aws:iam::180635532705:role/CI_ShadowServiceClient_Role
 
@@ -94,7 +95,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        version:
+        compiler:
           - clang-3
           - clang-6
           - clang-8
@@ -325,7 +326,12 @@ jobs:
           python builder.pyz build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DBUILD_SHARED_LIBS=ON
 
   osx:
-    runs-on: macos-latest
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        runner:
+          - macos-13
+          - macos-latest
     permissions:
       id-token: write # This is required for requesting the JWT
       security-events: write # This is required for pkcs12 sample to sign the key
@@ -337,6 +343,8 @@ jobs:
           ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream
       - name: Running samples in CI setup
         run: |
+          python3 -m venv .venv
+          source .venv/bin/activate
           python3 -m pip install boto3
       - name: configure AWS credentials (PubSub)
         uses: aws-actions/configure-aws-credentials@v2
@@ -345,6 +353,7 @@ jobs:
           aws-region: ${{ env.AWS_DEFAULT_REGION }}
       - name: run MQTT3 PubSub sample
         run: |
+          source .venv/bin/activate
           python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_pubsub_cfg.json
      - name: run PKCS12 sample
        run: |
@@ -370,6 +379,7 @@ jobs:
           aws-region: ${{ env.AWS_DEFAULT_REGION }}
       - name: run MQTT5 PubSub sample
         run: |
+          source .venv/bin/activate
           python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_mqtt5_pubsub_cfg.json
       - name: configure AWS credentials (Device Advisor)
         uses: aws-actions/configure-aws-credentials@v2
@@ -378,6 +388,7 @@ jobs:
           aws-region: ${{ env.AWS_DEFAULT_REGION }}
       - name: run DeviceAdvisor
         run: |
+          source .venv/bin/activate
           cd ./aws-iot-device-sdk-cpp-v2
           python3 ./deviceadvisor/script/DATestRun.py
   # Not strictly needed, but allows us to run Device Advisor and PubSub on Linux without needing to run all samples
@@ -631,14 +642,66 @@ jobs:
       - name: run X509 sample
         run: |
           python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_x509_connect_cfg.json
-      - name: configure AWS credentials (Greengrass)
-        uses: aws-actions/configure-aws-credentials@v2
-        with:
-          role-to-assume: ${{ env.CI_GREENGRASS_ROLE }}
-          aws-region: ${{ env.AWS_DEFAULT_REGION }}
-      - name: run Greengrass Discovery sample
+
+  # Runs the Greengrass samples
+  linux-greengrass-tests:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write # This is required for requesting the JWT
+    steps:
+      - name: Setup C++
         run: |
-          python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_greengrass_discovery_cfg.json
+          sudo apt-get -qq update -y
+          sudo apt-get -qq install -y build-essential
+          sudo apt install cmake
+          gcc --version
+          cmake --version
+      - name: Build ${{ env.PACKAGE_NAME }} + consumers
+        run: |
+          echo "Downloading source"
+          git clone --recursive https://github.com/aws/aws-iot-device-sdk-cpp-v2.git --branch ${{ env.HEAD_REF || github.ref_name }}
+          echo "Running builder"
+          python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
+          python builder.pyz build -p ${{ env.PACKAGE_NAME }} --variant build_gg_samples_only
+      - name: Install Greengrass Development Kit
+        run: |
+          python3 -m pip install awsiotsdk
+          python3 -m pip install -U git+https://github.com/aws-greengrass/aws-greengrass-gdk-cli.git@v1.6.2
+      - name: Configure AWS credentials (Greengrass)
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ env.CI_GREENGRASS_INSTALLER_ROLE }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+      - name: Build and run Greengrass basic discovery sample
+        working-directory: ./aws-iot-device-sdk-cpp-v2/tests/greengrass/basic_discovery
+        run: |
+          gdk component build
+          gdk test-e2e build
+          gdk test-e2e run
+      - name: Show logs
+        working-directory: ./aws-iot-device-sdk-cpp-v2/tests/greengrass/basic_discovery
+        # Print logs unconditionally to provide more details on Greengrass run even if the test failed.
+        if: always()
+        run: |
+          echo "=== greengrass.log"
+          cat testResults/gg*/greengrass.log
+          echo "=== software.amazon.awssdk.sdk-gg-test-discovery.log"
+          cat testResults/gg*/software.amazon.awssdk.sdk-gg-test-discovery.log
+      - name: Build and run Greengrass IPC sample
+        working-directory: ./aws-iot-device-sdk-cpp-v2/tests/greengrass/ipc
+        run: |
+          gdk component build
+          gdk test-e2e build
+          gdk test-e2e run
+      - name: Show logs
+        working-directory: ./aws-iot-device-sdk-cpp-v2/tests/greengrass/ipc
+        # Print logs unconditionally to provide more details on Greengrass run even if the test failed.
+        if: always()
+        run: |
+          echo "=== greengrass.log"
+          cat testResults/gg*/greengrass.log
+          echo "=== software.amazon.awssdk.sdk-gg-ipc.log"
+          cat testResults/gg*/software.amazon.awssdk.sdk-gg-ipc.log
 
   # check that docs can still build
   check-docs:

.github/workflows/ci_run_greengrass_discovery_cfg.json

@@ -1,39 +1,35 @@
 {
     "language": "CPP",
-    "sample_file": "./aws-iot-device-sdk-cpp-v2/build/samples/greengrass/basic_discovery/basic-discovery",
-    "sample_region": "us-east-1",
-    "sample_main_class": "",
+    "runnable_file": "basic-discovery",
+    "runnable_region": "us-east-1",
+    "runnable_main_class": "",
     "arguments": [
         {
             "name": "--cert",
-            "secret": "ci/Greengrass/cert",
+            "secret": "ci/GreengrassDiscovery/cert",
             "filename": "tmp_certificate.pem"
         },
         {
             "name": "--key",
-            "secret": "ci/Greengrass/key",
+            "secret": "ci/GreengrassDiscovery/key",
             "filename": "tmp_key.pem"
         },
         {
-            "name": "--ca_file",
-            "secret": "ci/Greengrass/ca",
-            "filename": "tmp_ca.pem"
+            "name": "--thing_name",
+            "data": "CI_Greengrass_Discovery_Thing"
         },
         {
             "name": "--region",
             "data": "us-east-1"
         },
         {
-            "name": "--thing_name",
-            "data": "CI_GreenGrass_Thing"
-        },
-        {
-            "name": "--is_ci",
-            "data": "true"
+            "name": "--topic",
+            "data": "clients/CI_Greengrass_Discovery_Thing/hello/world/$INPUT_UUID"
         },
         {
-            "name": "--print_discover_resp_only",
-            "data": ""
+            "name": "--mode",
+            "data": "publish"
         }
-    ]
+    ],
+    "stdin_file": "messages.txt"
 }

.github/workflows/ci_run_greengrass_ipc_cfg.json

@@ -0,0 +1,21 @@
+{
+    "language": "CPP",
+    "runnable_file": "greengrass-ipc",
+    "runnable_region": "us-east-1",
+    "runnable_main_class": "",
+    "arguments": [
+        {
+            "name": "--topic",
+            "data": "test/gg-ipc-topic"
+        },
+        {
+            "name": "--message",
+            "data": "hello"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
+        }
+
+    ]
+}

CMakeLists.txt

@@ -11,6 +11,7 @@ project(aws-iot-device-sdk-cpp-v2
 include(CTest)
 
 option(BUILD_DEPS "Builds aws common runtime dependencies as part of build. Turn off if you want to control your dependency chain." ON)
+option(USE_EXTERNAL_DEPS_SOURCES "Use dependencies provided by add_subdirectory command" OFF)
 option(BUILD_SAMPLES "(DEPRECATED) Build samples as part of the build" OFF)
 
 if (DEFINED CMAKE_PREFIX_PATH)
@@ -57,6 +58,10 @@ else()
 endif()
 
 if (BUILD_DEPS)
+    if (USE_EXTERNAL_DEPS_SOURCES)
+        message(FATAL_ERROR "USE_EXTERNAL_DEPS_SOURCES option should be used with BUILD_DEPS set to OFF")
+    endif()
+
     list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/crt/aws-crt-cpp/crt/aws-c-common/cmake")
 
     include(AwsFindPackage)
@@ -70,8 +75,27 @@ if (BUILD_DEPS)
     endif ()
     set(BUILD_TESTING ${BUILD_TESTING_PREV})
 else()
+    if (USE_EXTERNAL_DEPS_SOURCES)
+        function(validate_deps_sources dep_name)
+            if (NOT ${dep_name}_SOURCE_DIR)
+                message(FATAL_ERROR "USE_EXTERNAL_DEPS_SOURCES option is set, but ${dep_name} project is not configured."
+                    " ${dep_name} must be added using add_subdirectory command (or one of the higher-level commands that"
+                    " uses add_subdirectory, like FetchContent)")
+            endif()
+        endfunction()
+
+        validate_deps_sources(aws-crt-cpp)
+        validate_deps_sources(aws-c-iot)
+        validate_deps_sources(aws-c-common)
+
+        list(APPEND CMAKE_MODULE_PATH "${aws-c-common_SOURCE_DIR}/cmake")
+
+        set(IN_SOURCE_BUILD ON)
+    else()
+        set(IN_SOURCE_BUILD OFF)
+    endif()
+
     include(AwsFindPackage)
-    set(IN_SOURCE_BUILD OFF)
 endif()
 
 aws_use_package(aws-crt-cpp)

README.md

@@ -131,4 +131,4 @@ is provided by code that been generated from a model of the service.
 
 This library is licensed under the [Apache 2.0 License](./documents/LICENSE).
 
-Latest released version: v1.32.3
+Latest released version: v1.33.0

VERSION

@@ -1 +1 @@
-1.32.3
+1.33.0

builder.json

@@ -28,6 +28,12 @@
         "!build_steps": [
           "build"
         ]
+    },
+    "build_gg_samples_only": {
+        "!build_steps": [
+          "build",
+          "build-gg-samples"
+        ]
     }
   }
 }

eventstream_rpc/tests/include/awstest/EchoTestRpcClient.h

@@ -40,18 +40,36 @@ namespace Awstest
         void Close() noexcept;
         void WithLaunchMode(std::launch mode) noexcept;
 
+        /**
+         * Fetches all products, indexed by SKU
+         */
         std::shared_ptr<GetAllProductsOperation> NewGetAllProducts() noexcept;
 
+        /**
+         * Throws a ServiceError instead of returning a response.
+         */
         std::shared_ptr<CauseServiceErrorOperation> NewCauseServiceError() noexcept;
 
+        /**
+         * Responds to initial request normally then throws a ServiceError on stream response
+         */
         std::shared_ptr<CauseStreamServiceToErrorOperation> NewCauseStreamServiceToError(
             std::shared_ptr<CauseStreamServiceToErrorStreamHandler> streamHandler) noexcept;
 
+        /**
+         * Initial request and response are empty, but echos streaming messages sent by client
+         */
         std::shared_ptr<EchoStreamMessagesOperation> NewEchoStreamMessages(
             std::shared_ptr<EchoStreamMessagesStreamHandler> streamHandler) noexcept;
 
+        /**
+         * Returns the same data sent in the request to the response
+         */
         std::shared_ptr<EchoMessageOperation> NewEchoMessage() noexcept;
 
+        /**
+         * Fetches all customers
+         */
         std::shared_ptr<GetAllCustomersOperation> NewGetAllCustomers() noexcept;
 
         ~EchoTestRpcClient() noexcept;