Merge branch 'ci/cron_optimization'

lucasssvaz · Jun 27, 2024 · 7891ec2 · 7891ec2
2 parents 6683a0d + 8b9b9c4
commit 7891ec2
Show file tree

Hide file tree

Showing 7 changed files with 252 additions and 151 deletions.
diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml
@@ -1,31 +1,49 @@
 name: Cron Build
 
-on: 
+on:
   schedule:
 #             ┌───────────── minute (0 - 59)
 #             │ ┌───────────── hour (0 - 23)
 #             │ │ ┌───────────── day of the month (1 - 31)
 #             │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
 #             │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
-#             │ │ │ │ │                                   
+#             │ │ │ │ │
 #             │ │ │ │ │
 #             │ │ │ │ │
 #             * * * * *
     - cron:  '0 */6 * * *'
+  workflow_dispatch:
+    inputs:
+      idf_branch:
+        description: 'IDF branch to build'
+        required: true
+        default: 'release/v5.1'
+      target:
+        description: 'Target to build'
+        required: true
+        default: 'all'
 
 defaults:
   run:
     shell: bash
 
 jobs:
-  run:
-    name: Build with IDF ${{ matrix.idf_branch }}
+  build-libs:
+    name: Build with IDF ${{ matrix.idf_branch }} for ${{ matrix.target }}
     runs-on: ubuntu-latest
-
+    if: github.event_name == 'schedule'
     strategy:
       fail-fast: false
       matrix:
         idf_branch: [release/v5.1, release/v4.4] #, release/v3.3]
+        target: [esp32, esp32s2, esp32s3, esp32c3]
+        include:
+          - idf_branch: release/v5.1
+            target: esp32c2
+          - idf_branch: release/v5.1
+            target: esp32c6
+          - idf_branch: release/v5.1
+            target: esp32h2
     steps:
     - uses: actions/checkout@v4
       with:
@@ -39,8 +57,94 @@ jobs:
         GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
         IDF_BRANCH: ${{ matrix.idf_branch }}
       run: |
-        git checkout ${{ matrix.idf_branch }} || echo "Using master branch"
-        bash ./tools/cron.sh
+        git checkout ${{ env.IDF_BRANCH }} || echo "Using master branch"
+        if [ ! "$GITHUB_EVENT_NAME" == "schedule" ]; then
+          echo "Wrong event '$GITHUB_EVENT_NAME'!"
+          exit 1
+        fi
+
+        bash ./build.sh -e -t ${{ matrix.target }}
+
+    - name: Upload build
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: build-${{ matrix.idf_branch }}-${{ matrix.target }}
+        path: build
+
+    - name: Upload library files
+      uses: actions/upload-artifact@v4
+      with:
+        name: libs-${{ matrix.idf_branch }}-${{ matrix.target }}
+        path: dist
+
+  combine-artifacts:
+    name: Combine artifacts for IDF ${{ matrix.idf_branch }}
+    runs-on: ubuntu-latest
+    if: github.event_name == 'schedule'
+    needs: build-libs
+    strategy:
+      fail-fast: false
+      matrix:
+        idf_branch: [release/v5.1, release/v4.4]
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist
+          pattern: libs-${{ matrix.idf_branch }}-*
+          merge-multiple: true
+
+      - name: Combine artifacts
+        shell: bash
+        run: |
+          set -e
+          mkdir -p out
+          find dist -name 'arduino-esp32-libs-esp*.tar.gz' -exec tar zxvf {} -C out \;
+          for file in $files; do
+            tar zxvf $file -C out
+            cat out/tools/esp32-arduino-libs/versions.txt >> out/tools/esp32-arduino-libs/versions_full.txt
+          done
+          awk -i inplace '!seen[$0]++' out/tools/esp32-arduino-libs/versions_full.txt
+          mv -f out/tools/esp32-arduino-libs/versions_full.txt out/tools/esp32-arduino-libs/versions.txt
+          cd out/tools/esp32-arduino-libs && tar zcf ../../../dist/esp32-arduino-libs.tar.gz * && cd ../../..
+          cp out/package_esp32_index.template.json dist/package_esp32_index.template.json
+
+      - name: Push changes
+        if: ${{ github.repository == 'espressif/esp32-arduino-lib-builder' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
+          GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
+          GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
+          IDF_BRANCH: ${{ matrix.idf_branch }}
+        run: |
+          ./tools/push-to-arduino.sh
+          if [ $? -ne 0 ]; then exit 1; fi
+
+  build-libs-dispatch:
+    name: Dispatch build with IDF ${{ inputs.idf_branch }}
+    runs-on: ubuntu-latest
+    if: github.event_name == 'workflow_dispatch'
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Install dependencies
+      run: bash ./tools/prepare-ci.sh
+    - name: Build
+      env:
+        GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
+        GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
+        GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
+        IDF_BRANCH: ${{ inputs.idf_branch }}
+      run: |
+        git checkout ${{ env.IDF_BRANCH }} || echo "Using master branch"
+        if [ ! "$GITHUB_EVENT_NAME" == "schedule" ]; then
+          echo "Wrong event '$GITHUB_EVENT_NAME'!"
+          exit 1
+        fi
+
+        bash ./build.sh -d -t ${{ inputs.target }}
     - name: Upload build
       if: failure()
       uses: actions/upload-artifact@v4

diff --git a/build.sh b/build.sh
@@ -142,7 +142,7 @@ if [ "$BUILD_TYPE" != "all" ]; then
             # Skip building for targets that are not in the $TARGET array
             continue
         fi
-                
+
         configs="configs/defconfig.common;configs/defconfig.$target;configs/defconfig.debug_$BUILD_DEBUG"
         for defconf in `echo "$target_json" | jq -c '.features[]' | tr -d '"'`; do
             configs="$configs;configs/defconfig.$defconf"
@@ -187,7 +187,7 @@ for target_json in `jq -c '.targets[]' configs/builds.json`; do
             continue
         fi
     fi
-    
+
     # Skip chips that should not be a part of the final libs
     # WARNING!!! this logic needs to be updated when cron builds are split into jobs
     if [ "$TARGET" = "all" ] && [ $target_skip -eq 1 ]; then
@@ -299,13 +299,15 @@ done
 
 # update package_esp32_index.template.json
 if [ "$BUILD_TYPE" = "all" ]; then
+    echo "* Generating package_esp32_index.template.json..."
     python3 ./tools/gen_tools_json.py -i "$IDF_PATH" -j "$AR_COMPS/arduino/package/package_esp32_index.template.json" -o "$AR_OUT/"
     python3 ./tools/gen_tools_json.py -i "$IDF_PATH" -o "$TOOLS_JSON_OUT/"
     if [ $? -ne 0 ]; then exit 1; fi
 fi
 
 # Generate PlatformIO manifest file
 if [ "$BUILD_TYPE" = "all" ]; then
+    echo "* Generating PlatformIO manifest file..."
     pushd $IDF_PATH
     ibr=$(git describe --all 2>/dev/null)
     ic=$(git -C "$IDF_PATH" rev-parse --short HEAD)
@@ -316,18 +318,21 @@ fi
 
 # copy everything to arduino-esp32 installation
 if [ $COPY_OUT -eq 1 ] && [ -d "$ESP32_ARDUINO" ]; then
+    echo "* Copying to Arduino..."
     ./tools/copy-to-arduino.sh
     if [ $? -ne 0 ]; then exit 1; fi
 fi
 
 # push changes to esp32-arduino-libs and create pull request into arduino-esp32
 if [ $DEPLOY_OUT -eq 1 ]; then
+    echo "* Pushing to Arduino..."
     ./tools/push-to-arduino.sh
     if [ $? -ne 0 ]; then exit 1; fi
 fi
 
 # archive the build
 if [ $ARCHIVE_OUT -eq 1 ]; then
+    echo "* Archiving build..."
     ./tools/archive-build.sh "$TARGET"
     if [ $? -ne 0 ]; then exit 1; fi
 fi
diff --git a/tools/archive-build.sh b/tools/archive-build.sh
@@ -8,5 +8,5 @@ archive_path="dist/arduino-esp32-libs-$1-$idf_version_string.tar.gz"
 
 mkdir -p dist && rm -rf "$archive_path"
 if [ -d "out" ]; then
-	cd out && tar zcf "../$archive_path" * && cd ..
+  cd out && tar zcf "../$archive_path" * && cd ..
 fi
diff --git a/tools/copy-to-arduino.sh b/tools/copy-to-arduino.sh
@@ -2,16 +2,16 @@
 source ./tools/config.sh
 
 if [ -z $ESP32_ARDUINO ]; then
-    if [[ "$AR_OS" == "macos" ]]; then
-    	ESP32_ARDUINO="$HOME/Documents/Arduino/hardware/espressif/esp32"
-    else
-    	ESP32_ARDUINO="$HOME/Arduino/hardware/espressif/esp32"
-    fi
+  if [[ "$AR_OS" == "macos" ]]; then
+    ESP32_ARDUINO="$HOME/Documents/Arduino/hardware/espressif/esp32"
+  else
+    ESP32_ARDUINO="$HOME/Arduino/hardware/espressif/esp32"
+  fi
 fi
 
 if ! [ -d "$ESP32_ARDUINO" ]; then
-	echo "ERROR: Target arduino folder does not exist!"
-	exit 1
+  echo "ERROR: Target arduino folder does not exist!"
+  exit 1
 fi
 
 echo "Installing new libraries to $ESP32_ARDUINO"

diff --git a/tools/cron.sh b/tools/cron.sh
diff --git a/tools/install-esp-idf.sh b/tools/install-esp-idf.sh
@@ -51,7 +51,7 @@ source $IDF_PATH/export.sh
 # SETUP ARDUINO DEPLOY
 #
 
-if [ "$GITHUB_EVENT_NAME" == "schedule" ] || [ "$GITHUB_EVENT_NAME" == "repository_dispatch" -a "$GITHUB_EVENT_ACTION" == "deploy" ]; then
+if [ "$GITHUB_EVENT_NAME" == "schedule" ] || [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ] || [ "$GITHUB_EVENT_NAME" == "repository_dispatch" -a "$GITHUB_EVENT_ACTION" == "deploy" ]; then
 	# format new branch name and pr title
 	if [ -x $commit_predefined ]; then #commit was not specified at build time
 		AR_NEW_BRANCH_NAME="idf-$IDF_BRANCH"