Skip to content

Commit 4c0e6fc

Browse files
lucasssvazlucasssvaz
committed
Merge branch 'ci/cron_optimization'
2 parents 5584eac + f7540da commit 4c0e6fc

File tree

3 files changed

+170
-180
lines changed

3 files changed

+170
-180
lines changed

‎.github/workflows/cron.yml

Lines changed: 11 additions & 180 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Cron Build
1+
name: Cron Deploy
22

33
on:
44
schedule:
@@ -18,187 +18,18 @@ defaults:
1818
run:
1919
shell: bash
2020

21-
env:
22-
LATEST_RELEASE_BRANCH: "release/v5.1" # Change this to the latest release branch so the checkouts are done correctly
23-
2421
jobs:
25-
gen-matrix:
26-
name: Generate matrix
27-
runs-on: ubuntu-latest
28-
outputs:
29-
matrix: ${{ steps.gen-matrix.outputs.matrix }}
30-
branches: ${{ steps.gen-matrix.outputs.branches }}
31-
steps:
32-
- uses: actions/checkout@v4
33-
with:
34-
fetch-depth: 0
35-
36-
- name: Generate matrix
37-
id: gen-matrix
38-
env:
39-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
40-
run: |
41-
set -e
42-
43-
# Change this based on the IDF branches we want to build. Don't forget to update env.LATEST_RELEASE_BRANCH
44-
all_branches=("release/v5.1")
45-
46-
# Change this based on the COMMON targets for all branches we want to build.
47-
common_targets="[\"esp32\", \"esp32s2\", \"esp32s3\", \"esp32c2\", \"esp32c3\", \"esp32c6\", \"esp32h2\"]"
48-
49-
# For additional targets per branch, add them here
50-
additional_targets="[{\"idf_branch\": \"release/v5.3\", \"target\": \"esp32p4\"}]"
51-
52-
branches="["
53-
matrix="{"
54-
55-
for branch in ${all_branches[@]}; do
56-
if [ "$branch" == "$LATEST_RELEASE_BRANCH" ]; then
57-
git checkout master
58-
else
59-
git checkout $branch
60-
fi
61-
export IDF_BRANCH=$branch
62-
source ./tools/check-deploy-needed.sh
63-
#if [ "$DEPLOY_NEEDED" == "1" ]; then
64-
branches+="\"$branch\","
65-
#fi
66-
done
67-
68-
branches="${branches%,}]"
69-
matrix+="\"idf_branch\": $branches,"
70-
matrix+="\"target\": $common_targets,"
71-
72-
matrix+="\"include\": "
73-
# Add all additional targets that are in the selected branches
74-
matrix+=$(echo $additional_targets | jq --argjson branches "$branches" '[.[] | select(.idf_branch as $branch | $branches | index($branch))]')
75-
76-
matrix+="}"
77-
78-
echo "Branches: $branches"
79-
80-
echo "Matrix:"
81-
echo "$matrix" | jq .
82-
83-
if [ ! -x $GITHUB_OUTPUT ]; then
84-
echo "matrix=$matrix" >> $GITHUB_OUTPUT
85-
echo "branches=$branches" >> $GITHUB_OUTPUT
86-
fi
87-
8822
build-libs:
89-
name: Build with IDF ${{ matrix.idf_branch }} for ${{ matrix.target }}
90-
runs-on: ubuntu-latest
91-
if: needs.gen-matrix.outputs.branches != '[]'
92-
needs: gen-matrix
93-
strategy:
94-
fail-fast: false
95-
matrix: ${{ fromJson(needs.gen-matrix.outputs.matrix) }}
96-
steps:
97-
- uses: actions/checkout@v4
98-
with:
99-
# Useful workaround for the checkout action to work with the matrix
100-
# https://github.com/actions/runner/issues/409#issuecomment-1013325196
101-
ref: ${{ matrix.idf_branch == env.LATEST_RELEASE_BRANCH && 'master' || matrix.idf_branch }}
102-
103-
- name: Install dependencies
104-
run: bash ./tools/prepare-ci.sh
105-
106-
- name: Build
107-
env:
108-
GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN || secrets.GITHUB_TOKEN }}
109-
GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
110-
GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
111-
IDF_BRANCH: ${{ matrix.idf_branch }}
112-
TARGET: ${{ matrix.target }}
113-
run: |
114-
bash ./tools/cron.sh
115-
116-
- name: Replace invalid characters in the artifact name
117-
run: |
118-
branch=${{ matrix.idf_branch }}
119-
echo "libs_branch=${branch//\//_}" >> $GITHUB_ENV
120-
121-
- name: Upload build
122-
if: failure()
123-
uses: actions/upload-artifact@v4
124-
with:
125-
name: build-${{ env.libs_branch }}-${{ matrix.target }}
126-
path: build
127-
128-
- name: Upload library files
129-
uses: actions/upload-artifact@v4
130-
with:
131-
name: libs-${{ env.libs_branch }}-${{ matrix.target }}
132-
path: dist
133-
134-
combine-artifacts:
135-
name: Combine artifacts for IDF ${{ matrix.idf_branch }}
136-
runs-on: ubuntu-latest
137-
needs: [gen-matrix, build-libs]
138-
# Condition is evaluated before the job is run so it won't cause a failure
139-
if: needs.gen-matrix.outputs.branches != '[]'
23+
name: Build with IDF ${{ matrix.idf_branch }}
24+
uses: ./.github/workflows/cron_build.yml
25+
with:
26+
idf_branch: ${{ matrix.idf_branch }}
27+
lib_builder_branch: ${{ matrix.lib_builder_branch }}
28+
targets: ${{ matrix.targets }}
14029
strategy:
14130
fail-fast: false
14231
matrix:
143-
idf_branch: ${{ fromJson(needs.gen-matrix.outputs.branches) }}
144-
steps:
145-
- uses: actions/checkout@v4
146-
with:
147-
# Useful workaround for the checkout action to work with the matrix
148-
# https://github.com/actions/runner/issues/409#issuecomment-1013325196
149-
ref: ${{ matrix.idf_branch == env.LATEST_RELEASE_BRANCH && 'master' || matrix.idf_branch }}
150-
151-
- name: Replace invalid characters in the artifact name
152-
run: |
153-
branch=${{ matrix.idf_branch }}
154-
echo "libs_branch=${branch//\//_}" >> $GITHUB_ENV
155-
156-
- name: Download artifacts
157-
uses: actions/download-artifact@v4
158-
with:
159-
path: dist
160-
pattern: libs-${{ env.libs_branch }}-*
161-
merge-multiple: true
162-
163-
- name: Combine artifacts
164-
run: |
165-
set -e
166-
mkdir -p out
167-
168-
libs_folder="out/tools/esp32-arduino-libs"
169-
170-
files=$(find dist -name 'arduino-esp32-libs-esp*.tar.gz')
171-
for file in $files; do
172-
echo "Extracting $file"
173-
tar zxvf $file -C out
174-
cat $libs_folder/versions.txt >> $libs_folder/versions_full.txt
175-
done
176-
177-
# Merge versions.txt files
178-
awk -i inplace '!seen[$0]++' $libs_folder/versions_full.txt
179-
mv -f $libs_folder/versions_full.txt $libs_folder/versions.txt
180-
181-
cd $libs_folder && tar zcf ../../../dist/esp32-arduino-libs.tar.gz * && cd ../../..
182-
cp out/package_esp32_index.template.json dist/package_esp32_index.template.json
183-
184-
- name: Upload full esp32-arduino-libs archive
185-
uses: actions/upload-artifact@v4
186-
with:
187-
name: esp32-arduino-libs
188-
path: dist/esp32-arduino-libs.tar.gz
189-
190-
- name: Upload package_esp32_index.template.json
191-
uses: actions/upload-artifact@v4
192-
with:
193-
name: package-esp32-index-json
194-
path: dist/package_esp32_index.template.json
195-
196-
- name: Push changes
197-
if: github.repository == 'espressif/esp32-arduino-lib-builder'
198-
env:
199-
GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
200-
GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
201-
GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
202-
IDF_BRANCH: ${{ matrix.idf_branch }}
203-
run: |
204-
bash ./tools/push-to-arduino.sh
32+
include:
33+
- idf_branch: "release/v5.1"
34+
lib_builder_branch: "master"
35+
targets: "esp32,esp32s2,esp32s3,esp32c3,esp32c6,esp32h2"

‎.github/workflows/cron_build.yml

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
name: Cron Build Matrix
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
idf_branch:
7+
type: string
8+
required: true
9+
description: 'IDF branch to build'
10+
lib_builder_branch:
11+
type: string
12+
required: true
13+
description: 'Branch of the lib-builder to use'
14+
targets:
15+
type: string
16+
required: true
17+
description: 'Targets to build'
18+
19+
env:
20+
IDF_BRANCH: ${{ inputs.idf_branch }}
21+
22+
jobs:
23+
check-if-needed:
24+
name: Check if deploy is needed for ${{ inputs.idf_branch }}
25+
runs-on: ubuntu-latest
26+
outputs:
27+
deploy_needed: ${{ steps.gen-matrix.outputs.deploy_needed }}
28+
targets_list: ${{ steps.gen-matrix.outputs.targets_list }}
29+
steps:
30+
- uses: actions/checkout@v4
31+
with:
32+
ref: ${{ inputs.lib_builder_branch }}
33+
34+
- name: Generate matrix
35+
id: gen-matrix
36+
env:
37+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
38+
run: |
39+
source ./tools/check-deploy-needed.sh
40+
if [ "$DEPLOY_NEEDED" == "1" ]; then
41+
echo "deploy_needed=true" >> $GITHUB_OUTPUT
42+
targets_list=$(echo "${{ inputs.targets }}" | sed 's/ *, */,/g' | sed 's/^/["/' | sed 's/$/"]/' | sed 's/,/","/g')
43+
else
44+
echo "deploy_needed=false" >> $GITHUB_OUTPUT
45+
targets_list='[]'
46+
fi
47+
48+
echo "Targets list: $targets_list"
49+
echo "targets_list=$targets_list" >> $GITHUB_OUTPUT
50+
51+
build-libs:
52+
name: Build with IDF ${{ inputs.idf_branch }} for ${{ matrix.target }}
53+
runs-on: ubuntu-latest
54+
if: needs.check-if-needed.outputs.deploy_needed == 'true'
55+
needs: check-if-needed
56+
strategy:
57+
fail-fast: false
58+
matrix:
59+
target: ${{ fromJson(needs.check-if-needed.outputs.targets_list) }}
60+
steps:
61+
- uses: actions/checkout@v4
62+
with:
63+
ref: ${{ inputs.idf_branch }}
64+
65+
- name: Install dependencies
66+
run: bash ./tools/prepare-ci.sh
67+
68+
- name: Build
69+
env:
70+
GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN || secrets.GITHUB_TOKEN }}
71+
GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
72+
GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
73+
TARGET: ${{ matrix.target }}
74+
run: |
75+
bash ./tools/cron.sh
76+
77+
- name: Replace invalid characters in the artifact name
78+
run: |
79+
branch=${{ inputs.idf_branch }}
80+
echo "libs_branch=${branch//\//_}" >> $GITHUB_ENV
81+
82+
- name: Upload build
83+
if: failure()
84+
uses: actions/upload-artifact@v4
85+
with:
86+
name: build-${{ env.libs_branch }}-${{ matrix.target }}
87+
path: build
88+
89+
- name: Upload library files
90+
uses: actions/upload-artifact@v4
91+
with:
92+
name: libs-${{ env.libs_branch }}-${{ matrix.target }}
93+
path: dist
94+
95+
combine-artifacts:
96+
name: Combine artifacts for IDF ${{ inputs.idf_branch }}
97+
runs-on: ubuntu-latest
98+
needs: [check-if-needed, build-libs]
99+
if: needs.check-if-needed.outputs.deploy_needed == 'true'
100+
steps:
101+
- uses: actions/checkout@v4
102+
with:
103+
ref: ${{ inputs.lib_builder_branch }}
104+
105+
- name: Replace invalid characters in the artifact name
106+
run: |
107+
branch=${{ inputs.lib_builder_branch }}
108+
echo "libs_branch=${branch//\//_}" >> $GITHUB_ENV
109+
110+
- name: Download artifacts
111+
uses: actions/download-artifact@v4
112+
with:
113+
path: dist
114+
pattern: libs-${{ env.libs_branch }}-*
115+
merge-multiple: true
116+
117+
- name: Combine artifacts
118+
run: bash ./tools/combine-artifacts.sh
119+
120+
- name: Upload full esp32-arduino-libs archive
121+
uses: actions/upload-artifact@v4
122+
with:
123+
name: esp32-arduino-libs-${{ env.libs_branch }}
124+
path: dist/esp32-arduino-libs.tar.gz
125+
126+
- name: Upload package_esp32_index.template.json
127+
uses: actions/upload-artifact@v4
128+
with:
129+
name: package-esp32-index-json-${{ env.libs_branch }}
130+
path: dist/package_esp32_index.template.json
131+
132+
- name: Push changes
133+
if: github.repository == 'espressif/esp32-arduino-lib-builder'
134+
env:
135+
GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
136+
GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
137+
GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
138+
run: |
139+
bash ./tools/push-to-arduino.sh

‎tools/cron-combine.sh

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/bash
2+
3+
set -e
4+
mkdir -p out
5+
6+
libs_folder="out/tools/esp32-arduino-libs"
7+
8+
files=$(find dist -name 'arduino-esp32-libs-esp*.tar.gz')
9+
for file in $files; do
10+
echo "Extracting $file"
11+
tar zxvf $file -C out
12+
cat $libs_folder/versions.txt >> $libs_folder/versions_full.txt
13+
done
14+
15+
# Merge versions.txt files
16+
awk -i inplace '!seen[$0]++' $libs_folder/versions_full.txt
17+
mv -f $libs_folder/versions_full.txt $libs_folder/versions.txt
18+
19+
cd $libs_folder && tar zcf ../../../dist/esp32-arduino-libs.tar.gz * && cd ../../..
20+
cp out/package_esp32_index.template.json dist/package_esp32_index.template.json

0 commit comments

Comments
 (0)