producer c release candidate (#435)

* Added 1-20% jitter to expiration value stored by AwsCredentials struct. This is part of a larger effort to help customers with large fleets distribute the timings of their attempts to reconnect to our servers. * reduced clumped distribution between 1-2% * fix auth query template, used by signAwsRequestInfoQueryParam * aws service configurable for presigned urls, eg s3 * Update PIC to include modified threadpool teardown * use git tag commit from pic-develop * Update PIC to include modified threadpool teardown * use git tag commit from pic-develop * Set newly added timeouts to default value * Expose CONSTRAINED_DEVICE option * use develop tag * Add logging section (#417) * Introduce M1 build in the CI (#419) * m1 build * cmake update * run tests in m1 * Add setting of fragment meta to the sample * Formatting + style * More style * Small optimization * Update readme * Clang format * Clang format, again * Add gcc paths (#420) * spotaws PR #226 (#423) * proper versioned shared libs * Fix version * Update to latest version * Exclude shared builds from windows --------- Co-authored-by: Tom spot Callaway <spot@fedoraproject.org> Co-authored-by: Divya Sampath Kumar <disa6302@colorado.edu> Co-authored-by: Hassan Sahibzada <hsahibza@amazon.com> * Revert "Pr 323" * Added Iot credential usage to producer c samples (#220) * Added Iot credential usage to producer c samples * Refactor samples to include IoT usage and update readme --------- Co-authored-by: delaplan <delaplan@u48757f25b2c250.ant.amazon.com> Co-authored-by: Divya Sampath Kumar <disa6302@colorado.edu> * Update thing name length to be 128 as per AWS docs (#426) * Release build for producer C (#428) * Release build for producer C * Fix assert failure * Handle aws special regions (#430) * run windows tests with debug logging (#431) * run windows tests with debug logging * zero out frame buffer to avoid invalid mkv * fix tsan related issues (#427) fix tsan related issues * update pic hash to release candidate * add gcc m1 build * bump version for release * Add PR description lint (#438) * update pic to release tag --------- Co-authored-by: James Delaplane <delaplan@amazon.com> Co-authored-by: Michael Dietz <michael.dietz@waya.ai> Co-authored-by: Niyati Maheshwari <niyatim@amazon.com> Co-authored-by: Divya Sampath Kumar <disa6302@colorado.edu> Co-authored-by: Dave Johansen <davejohansen@gmail.com> Co-authored-by: Niyati Maheshwari <niyatim23@gmail.com> Co-authored-by: Jeremy Gunawan <jggunawa@amazon.com> Co-authored-by: Stefan Kieszkowski <85728496+stefankiesz@users.noreply.github.com> Co-authored-by: Tom spot Callaway <spot@fedoraproject.org> Co-authored-by: Jeremy Gunawan <sirknightj@gmail.com> Co-authored-by: delaplan <delaplan@u48757f25b2c250.ant.amazon.com>
awslabs · Mar 11, 2024 · 3e519b7 · 3e519b7
1 parent 178109a
commit 3e519b7
Show file tree

Hide file tree

Showing 34 changed files with 773 additions and 215 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,11 @@
+*Issue #, if available:*
+
+*What was changed?*
+
+*Why was it changed?*
+
+*How was it changed?*
+
+*What testing was done for the changes?*
+
+By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -29,8 +29,8 @@ jobs:
       id-token: write
       contents: read
     env:
-      CC: gcc
-      CXX: g++
+      CC: /usr/local/bin/gcc-13
+      CXX: /usr/local/bin/g++-13
       AWS_KVS_LOG_LEVEL: 2
     steps:
       - name: Clone repository
@@ -58,8 +58,6 @@ jobs:
     runs-on: macos-11
     env:
       AWS_KVS_LOG_LEVEL: 2
-      LDFLAGS: -L/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib
-      CPATH: /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/
     permissions:
       id-token: write
       contents: read
@@ -85,14 +83,78 @@ jobs:
           cd build
           ./tst/producer_test
 
+  mac-os-m1-build-clang:
+    runs-on: macos-13-xlarge
+    env:
+      AWS_KVS_LOG_LEVEL: 2
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v3
+      - name: Build repository
+        run: |
+          brew install pkgconfig
+          brew unlink openssl # it seems the libcurl is trying to access this openssl despite explicitly setting it to our build
+          mkdir build && cd build
+          cmake .. -DBUILD_TEST=TRUE -DCOMPILER_WARNINGS=TRUE -DCMAKE_C_COMPILER=$(brew --prefix llvm@15)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm@15)/bin/clang++
+          make
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1-node16
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          role-session-name: ${{ secrets.AWS_ROLE_SESSION_NAME }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          role-duration-seconds: 10800
+      - name: Run tests
+        run: |
+          cd build
+          ./tst/producer_test
+
+  mac-os-m1-build-gcc:
+    runs-on: macos-13-xlarge
+    env:
+      CC: /opt/homebrew/bin/gcc-13
+      CXX: /opt/homebrew/bin/g++-13
+      AWS_KVS_LOG_LEVEL: 2
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Setup XCode version 15.2
+        uses: maxim-lobanov/setup-xcode@v1
+        with:
+          xcode-version: '15.2'
+      - name: Clone repository
+        uses: actions/checkout@v3
+      - name: Build repository
+        run: |
+          brew install pkgconfig
+          brew unlink openssl # it seems the libcurl is trying to access this openssl despite explicitly setting it to our build
+          mkdir build && cd build
+          cmake .. -DBUILD_TEST=TRUE -DCOMPILER_WARNINGS=TRUE
+          make
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1-node16
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          role-session-name: ${{ secrets.AWS_ROLE_SESSION_NAME }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          role-duration-seconds: 10800
+      - name: Run tests
+        run: |
+          cd build
+          ./tst/producer_test
+
   mac-os-build-gcc-local-openssl:
     runs-on: macos-11
     permissions:
       id-token: write
       contents: read
     env:
-      CC: gcc
-      CXX: g++
+      CC: /usr/local/bin/gcc-13
+      CXX: /usr/local/bin/g++-13
       AWS_KVS_LOG_LEVEL: 2
       LDFLAGS: -L/usr/local/opt/openssl@3/lib
       CPPFLAGS: -I/usr/local/opt/openssl@3/include
@@ -268,31 +330,34 @@ jobs:
   #         ulimit -c unlimited -S
   #         timeout --signal=SIGABRT 150m ./tst/producer_test --gtest_break_on_failure
 
-  # thread-sanitizer:
-  #   runs-on: ubuntu-20.04
-  #   permissions:
-  #     id-token: write
-  #     contents: read
-  #   env:
-  #     CC: clang
-  #     CXX: clang++
-  #     AWS_KVS_LOG_LEVEL: 2
-  #   steps:
-  #     - name: Clone repository
-  #       uses: actions/checkout@v3
-  #     - name: Configure AWS Credentials
-  #       uses: aws-actions/configure-aws-credentials@v1-node16
-  #       with:
-  #         role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-  #         role-session-name: ${{ secrets.AWS_ROLE_SESSION_NAME }}
-  #         aws-region: ${{ secrets.AWS_REGION }}
-  #     - name: Build repository
-  #       run: |
-  #         mkdir build && cd build
-  #         cmake .. -DBUILD_TEST=TRUE -DTHREAD_SANITIZER=TRUE -DBUILD_COMMON_LWS=TRUE
-  #         make
-  #         ulimit -c unlimited -S
-  #         timeout --signal=SIGABRT 150m ./tst/producer_test --gtest_break_on_failure
+  #thread-sanitizer:
+  #  runs-on: ubuntu-20.04
+  #  permissions:
+  #    id-token: write
+  #    contents: read
+  #  env:
+  #    CC: clang
+  #    CXX: clang++
+  #    AWS_KVS_LOG_LEVEL: 2
+  #  steps:
+  #    - name: Clone repository
+  #      uses: actions/checkout@v3
+  #    - name: Configure AWS Credentials
+  #      uses: aws-actions/configure-aws-credentials@v1-node16
+  #      with:
+  #        role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+  #        role-session-name: ${{ secrets.AWS_ROLE_SESSION_NAME }}
+  #        aws-region: ${{ secrets.AWS_REGION }}
+  #    - name: Build repository
+  #      run: |
+  #        mkdir build && cd build
+  #        cmake .. -DBUILD_TEST=TRUE -DTHREAD_SANITIZER=TRUE -DBUILD_COMMON_LWS=TRUE
+  #        make
+  #    - name: Run tests
+  #      run: |
+  #        cd build
+  #        ulimit -c unlimited -S
+  #        timeout --signal=SIGABRT 150m ./tst/producer_test --gtest_break_on_failure
 
   ubuntu-gcc:
     runs-on: ubuntu-20.04
@@ -331,7 +396,7 @@ jobs:
   windows-msvc:
     runs-on: windows-2022
     env:
-      AWS_KVS_LOG_LEVEL: 7
+      AWS_KVS_LOG_LEVEL: 2
     permissions:
       id-token: write
       contents: read

diff --git a/.github/workflows/pr-desc-lint.yml b/.github/workflows/pr-desc-lint.yml
@@ -0,0 +1,55 @@
+name: PR Description Check
+
+on:
+  pull_request:
+    branches:
+      - develop
+      - master
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - edited
+
+jobs:
+  check-description:
+    runs-on: macos-latest
+    steps:
+      - name: Install GitHub CLI
+        run: |
+          brew install gh
+
+      - name: Check PR Description
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          pr_description=$(gh pr view https://github.com/${GITHUB_REPOSITORY}/pull/${{ github.event.pull_request.number }} --json body -q ".body")
+          error_occurred=0
+          # Define minimum character count for each section
+          MIN_CHARS=25
+
+          # Extract contents
+          what_changed=$(echo "$pr_description" | sed -n -e '/\*What was changed?\*/,/\*/p' | sed '$d' | sed '1d')
+          why_changed=$(echo "$pr_description" | sed -n -e '/\*Why was it changed?\*/,/\*/p' | sed '$d' | sed '1d')
+          how_changed=$(echo "$pr_description" | sed -n -e '/\*How was it changed?\*/,/\*/p' | sed '$d' | sed '1d')
+          testing_done=$(echo "$pr_description" | sed -n -e '/\*What testing was done for the changes?\*/,/\*/p' | sed '$d' | sed '1d')
+
+          if [[ ${#what_changed} -lt $MIN_CHARS ]]; then
+            echo "PR description for what changed section is either missing or too short."
+            error_occurred=1
+          fi
+          if [[ ${#why_changed} -lt $MIN_CHARS ]]; then
+            echo "PR description for why it changed section is either missing or too short."
+            error_occurred=1
+          fi
+          if [[ ${#how_changed} -lt $MIN_CHARS ]]; then
+            echo "PR description for how was it changed section is either missing or too short."
+            error_occurred=1
+          fi
+          if [[ ${#testing_done} -lt $MIN_CHARS ]]; then
+            echo "PR description for testing section are either missing or too short."
+            error_occurred=1
+          fi
+          if [[ $error_occurred -eq 1 ]]; then
+          exit 1
+          fi
diff --git a/CMake/Dependencies/libcurl-CMakeLists.txt b/CMake/Dependencies/libcurl-CMakeLists.txt
@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.6.3)
 
-project(libcurl-download NONE)
+project(libcurl-download LANGUAGES C)
 
 find_program(MAKE_EXE NAMES make)
 
@@ -17,6 +17,7 @@ endif()
 
 set(CMAKE_ARGS
         -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}
+        -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
         -DCMAKE_INSTALL_PREFIX=${OPEN_SRC_INSTALL_PREFIX}
         -DBUILD_CURL_EXE=0
         -DHTTP_ONLY=1)

diff --git a/CMake/Dependencies/libgtest-CMakeLists.txt b/CMake/Dependencies/libgtest-CMakeLists.txt
@@ -3,7 +3,6 @@ cmake_minimum_required(VERSION 3.6.3)
 project(libgtest-download NONE)
 
 include(ExternalProject)
-
 ExternalProject_Add(libgtest-download
     GIT_REPOSITORY    https://github.com/google/googletest.git
     GIT_TAG           release-1.12.1

diff --git a/CMake/Dependencies/libjsmn-CMakeLists.txt b/CMake/Dependencies/libjsmn-CMakeLists.txt
diff --git a/CMake/Dependencies/libjsmn-add-cmakelists.patch b/CMake/Dependencies/libjsmn-add-cmakelists.patch
diff --git a/CMake/Dependencies/libkvspic-CMakeLists.txt b/CMake/Dependencies/libkvspic-CMakeLists.txt
@@ -1,13 +1,13 @@
 cmake_minimum_required(VERSION 3.6.3)
 
-project(libkvspic-download NONE)
+project(libkvspic-download LANGUAGES C)
 
 include(ExternalProject)
 
 # clone repo only
 ExternalProject_Add(libkvspic-download
 	GIT_REPOSITORY    https://github.com/awslabs/amazon-kinesis-video-streams-pic.git
-	GIT_TAG           57637ea593f4b43c509413a44d993ed08d7f2616
+	GIT_TAG           v1.1.0
    	SOURCE_DIR        "${CMAKE_CURRENT_BINARY_DIR}/kvspic-src"
 	BINARY_DIR        "${CMAKE_CURRENT_BINARY_DIR}/kvspic-build"
 	CMAKE_ARGS

diff --git a/CMake/Dependencies/libmbedtls-CMakeLists.txt b/CMake/Dependencies/libmbedtls-CMakeLists.txt
@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.6.3)
 
-project(libmbedtls-download NONE)
+project(libmbedtls-download LANGUAGES C)
 
 include(ExternalProject)
 
@@ -22,7 +22,7 @@ ExternalProject_Add(
   CMAKE_ARGS
   	-DCMAKE_INSTALL_PREFIX=${OPEN_SRC_INSTALL_PREFIX}
     -DUSE_SHARED_MBEDTLS_LIBRARY=${BUILD_SHARED}
-    -DCMAKE_BUILD_TYPE=Release
+    -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
     -DCMAKE_MACOSX_RPATH=${CMAKE_MACOSX_RPATH}
   BUILD_ALWAYS 		TRUE
   TEST_COMMAND 		"")
diff --git a/CMake/Dependencies/libopenssl-CMakeLists.txt b/CMake/Dependencies/libopenssl-CMakeLists.txt
@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.6.3)
 
-project(libopenssl-download NONE)
+project(libopenssl-download LANGUAGES C)
 
 if (WIN32)
   find_program(MAKE_EXE NAMES nmake)
@@ -11,6 +11,12 @@ else()
     SET(OPENSSL_EXTRA ${OPENSSL_EXTRA} no-shared no-dso)
   endif()
 
+  if (DEFINED CMAKE_OSX_SYSROOT AND NOT CMAKE_OSX_SYSROOT STREQUAL "")
+      if ("${CMAKE_C_COMPILER_ID}" MATCHES "Clang")
+        SET(OPENSSL_EXTRA ${OPENSSL_EXTRA} -I${CMAKE_OSX_SYSROOT}/usr/include -L${CMAKE_OSX_SYSROOT}/usr/lib)
+      endif()
+  endif()
+
   if (DEFINED BUILD_OPENSSL_PLATFORM AND NOT BUILD_OPENSSL_PLATFORM STREQUAL OFF)
     SET(CONFIGURE_COMMAND ${CMAKE_CURRENT_BINARY_DIR}/build/src/project_libopenssl/Configure ${OPENSSL_EXTRA} --prefix=${OPEN_SRC_INSTALL_PREFIX} --openssldir=${OPEN_SRC_INSTALL_PREFIX} ${BUILD_OPENSSL_PLATFORM} -Wno-nullability-completeness -Wno-expansion-to-defined)
   else()

diff --git a/CMake/Dependencies/libwebsockets-CMakeLists.txt b/CMake/Dependencies/libwebsockets-CMakeLists.txt
@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.6.3)
 
-project(libwebsocket-download NONE)
+project(libwebsocket-download LANGUAGES C)
 
 include(ExternalProject)
 
@@ -25,6 +25,7 @@ ExternalProject_Add(project_libwebsockets
     CMAKE_ARGS
         -DCMAKE_INSTALL_PREFIX=${OPEN_SRC_INSTALL_PREFIX}
         -DLWS_WITH_HTTP2=1
+        -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
         -DLWS_HAVE_HMAC_CTX_new=1
         -DLWS_HAVE_SSL_EXTRA_CHAIN_CERTS=1
         -DLWS_HAVE_OPENSSL_ECDH_H=1