ci: optimize ci framework

espressif · Dec 23, 2024 · 57ad725 · 57ad725
1 parent 354f742
commit 57ad725
Show file tree

Hide file tree

Showing 6 changed files with 244 additions and 253 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,107 +1,4 @@
-stages:
-  - build
-
-variables:
-  CI_DOCKER_REGISTRY: "ciregistry.espressif.cn:8443"
-  ESP_ZIGBEE_SDK_PATH: "$CI_PROJECT_DIR/esp-zigbee-sdk"
-  ESP_ZBOSS_LIB_PATH: "$CI_PROJECT_DIR"
-
-before_script:
-  - mkdir -p ~/.ssh
-  - chmod 700 ~/.ssh
-  - echo -n "${GITLAB_KEY}" >~/.ssh/id_rsa_base64
-  - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa
-  - chmod 600 ~/.ssh/id_rsa
-  - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >>~/.ssh/config
-
-.setup_idf: &setup_idf
-  - cd $CI_PROJECT_DIR/..
-  - rm -rf esp-idf
-  - git clone --recursive -b v5.3.2 ${ESP_IDF_URL}
-  - cd esp-idf
-  - git checkout --track origin/$CI_COMMIT_REF_NAME || git branch
-  - git submodule sync --recursive
-  - git submodule update --recursive --init
-  - ./install.sh
-  - . ./export.sh
-
-.setup_esp_sdk: &setup_esp_sdk
-  - cd $CI_PROJECT_DIR
-  - git clone --recursive -b main ${ESP_ZIGBEE_SDK_URL}
-  - cd esp-zigbee-sdk
-  - git checkout --track origin/$CI_COMMIT_REF_NAME || git branch
-  - git submodule update --recursive --init
-  - cp -r ${ESP_ZIGBEE_SDK_PATH}/components/esp-zigbee-lib ${IDF_PATH}/components/espressif__esp-zigbee-lib
-
-.update_zboss_lib_script: &update_zboss_lib_script
-  - *setup_idf
-  - *setup_esp_sdk
-  - cd ${ESP_ZBOSS_LIB_PATH}
-  - cp -r ${ESP_ZBOSS_LIB_PATH} ${IDF_PATH}/components/espressif__esp-zboss-lib
-  - cd ${ESP_ZIGBEE_SDK_PATH}
-  - pip install -r tools/ci/requirements-build.txt
-
-build_non_pytest_examples:
-  stage: build
-  image: ${CI_DOCKER_REGISTRY}/esp-env-v5.3:1
-  artifacts:
-    paths:
-      - "**/build*/*.bin"
-      - "**/build*/flasher_args.json"
-      - "**/build*/config/sdkconfig.json"
-      - "**/build*/bootloader/*.bin"
-      - "**/build*/partition_table/*.bin"
-    when: always
-    expire_in: 4 days
-  script:
-    - *update_zboss_lib_script
-    - python tools/ci/build_apps.py ./examples --no_pytest
-      --parallel-count ${CI_NODE_TOTAL:-1}
-      --parallel-index ${CI_NODE_INDEX:-1}
-  parallel: 2
-  tags:
-    - build
-
-build_pytest_examples:
-  stage: build
-  image: ${CI_DOCKER_REGISTRY}/esp-env-v5.3:1
-  artifacts:
-    paths:
-      - "**/build*/*.bin"
-      - "**/build*/*.map"
-      - "**/build*/*.elf"
-      - "**/build*/flasher_args.json"
-      - "**/build*/flash_project_args"
-      - "**/build*/config/sdkconfig.json"
-      - "**/build*/bootloader/*.bin"
-      - "**/build*/partition_table/*.bin"
-    when: always
-    expire_in: 4 days
-  script:
-    - *update_zboss_lib_script
-    - python tools/ci/build_apps.py ./examples --pytest
-  tags:
-    - build
-
-build_idf_examples:
-  stage: build
-  image: ${CI_DOCKER_REGISTRY}/esp-env-v5.3:1
-  artifacts:
-    paths:
-      - "**/build*/*.bin"
-      - "**/build*/*.map"
-      - "**/build*/*.elf"
-      - "**/build*/flasher_args.json"
-      - "**/build*/flash_project_args"
-      - "**/build*/config/sdkconfig.json"
-      - "**/build*/bootloader/*.bin"
-      - "**/build*/partition_table/*.bin"
-    when: always
-    expire_in: 4 days
-  script:
-      - *update_zboss_lib_script
-      - cp -r ${ESP_ZBOSS_LIB_PATH}/tools/ci/build_apps.py ${IDF_PATH}/examples/zigbee/build_apps.py
-      - cd ${IDF_PATH}
-      - python examples/zigbee/build_apps.py ./examples/zigbee --idf_examples
-  tags:
-    - build
+include:
+  - '.gitlab/ci/common.yml'
+  - '.gitlab/ci/generate_build_test_jobs.yml'
+  - '.gitlab/ci/build_idf_examples.yml'
diff --git a/.gitlab/ci/build_idf_examples.yml b/.gitlab/ci/build_idf_examples.yml
@@ -0,0 +1,24 @@
+build_idf_examples:
+  stage: build
+  image: ${CI_DOCKER_REGISTRY}/esp-env-${DEFAULT_BUILD_DOCKER_ENV}
+  extends:
+    - .setup_build_environment_script
+  artifacts:
+    paths:
+      - "**/build*/*.bin"
+      - "**/build*/*.map"
+      - "**/build*/*.elf"
+      - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+    when: always
+    expire_in: 4 days
+  variables:
+    IDF_VERSION: ${DEFAULT_IDF_VERSION}
+  script:
+      - cd ${IDF_PATH}
+      - python ${CI_PROJECT_DIR}/esp-zigbee-sdk/tools/ci/build_apps.py ./examples/zigbee
+  tags:
+    - build
diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml
@@ -0,0 +1,81 @@
+stages:
+  - generate
+  - build
+  - target_test
+
+variables:
+    GIT_SUBMODULE_STRATEGY: none
+    DEFAULT_IDF_VERSION: "v5.3.2"
+    DEFAULT_BUILD_DOCKER_ENV: "v5.3:1"
+    # tell build system do not check submodule update as we download archive instead of clone
+    IDF_SKIP_CHECK_SUBMODULES: 1
+
+.init_ssh: &init_ssh
+  - mkdir -p ~/.ssh
+  - chmod 700 ~/.ssh
+  - echo -n "${GITLAB_KEY}" >~/.ssh/id_rsa_base64
+  - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa
+  - chmod 600 ~/.ssh/id_rsa
+  - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >>~/.ssh/config
+
+.setup_idf: &setup_idf
+  - cd $CI_PROJECT_DIR
+  - git clone --recursive ${ESP_IDF_URL}
+  - cd esp-idf
+  - git checkout --track origin/$CI_COMMIT_REF_NAME || git checkout ${IDF_VERSION}
+  - git submodule sync --recursive
+  - git submodule update --recursive --init
+  - ./install.sh
+  - . ./export.sh
+
+.setup_esp_sdk: &setup_esp_sdk
+  - cd $CI_PROJECT_DIR
+  - git clone --recursive -b main ${ZIGBEE_SDK_REPO_URL}
+  - cd esp-zigbee-sdk
+  - git checkout --track origin/$CI_COMMIT_REF_NAME || git branch
+
+.update_cmake_dependency: &update_cmake_dependency
+  - echo ${IDF_PATH}
+  - cp -r $CI_PROJECT_DIR/esp-zigbee-sdk/components/esp-zigbee-lib ${IDF_PATH}/components/espressif__esp-zigbee-lib
+  # copy esp-zboss-lib to esp-idf
+  - mkdir ${IDF_PATH}/components/espressif__esp-zboss-lib
+  - cp -r $CI_PROJECT_DIR/include ${IDF_PATH}/components/espressif__esp-zboss-lib
+  - cp -r $CI_PROJECT_DIR/lib ${IDF_PATH}/components/espressif__esp-zboss-lib
+  - cp $CI_PROJECT_DIR/idf_component.yml ${IDF_PATH}/components/espressif__esp-zboss-lib
+  - cp $CI_PROJECT_DIR/CMakeLists.txt ${IDF_PATH}/components/espressif__esp-zboss-lib
+  - ls -lf ${IDF_PATH}/components/espressif__esp-zboss-lib
+  # end copy
+  - sed -i "/\besp-zboss-lib\b/d;/\besp-zigbee-lib\b/d" ${IDF_PATH}/examples/zigbee/**/main/idf_component.yml
+  - sed -i "/\besp-zboss-lib\b/d;/\besp-zigbee-lib\b/d" $CI_PROJECT_DIR/esp-zigbee-sdk/examples/**/main/idf_component.yml
+  - sed -i "/\besp-zboss-lib\b/d;/\besp-zigbee-lib\b/d" $CI_PROJECT_DIR/esp-zigbee-sdk/components/*/idf_component.yml
+
+.build_rcp_gateway: &build_rcp_gateway
+  - mv $CI_PROJECT_DIR/esp-zigbee-sdk/examples/esp_zigbee_gateway/sdkconfig.defaults $CI_PROJECT_DIR/esp-zigbee-sdk/examples/esp_zigbee_gateway/sdkconfig.defaults.ori
+  - cp $CI_PROJECT_DIR/esp-zigbee-sdk/tools/ci/sdkconfig_for_gateway_ci.defaults $CI_PROJECT_DIR/esp-zigbee-sdk/examples/esp_zigbee_gateway/sdkconfig.defaults
+  - cd $CI_PROJECT_DIR/esp-idf/examples/openthread/ot_rcp
+  - echo "CONFIG_OPENTHREAD_NCP_VENDOR_HOOK=y" >> sdkconfig.defaults
+  - idf.py set-target esp32h2
+  - idf.py build
+  - cd $CI_PROJECT_DIR/esp-zigbee-sdk/examples/esp_zigbee_gateway
+  - idf.py set-target esp32s3
+  - idf.py build
+  - mv $CI_PROJECT_DIR/esp-zigbee-sdk/examples/esp_zigbee_gateway/sdkconfig.defaults.ori $CI_PROJECT_DIR/esp-zigbee-sdk/examples/esp_zigbee_gateway/sdkconfig.defaults
+  - cd ${CI_PROJECT_DIR}
+
+.setup_build_environment_script:
+  before_script:
+    - *init_ssh
+    - *setup_idf
+    - *setup_esp_sdk
+    - *update_cmake_dependency
+    - pip install -r tools/ci/requirements-build.txt
+
+.setup_esp_sdk_script:
+  before_script:
+    - *init_ssh
+    - *setup_esp_sdk
+
+.build_gateway_script:
+  script:
+    - python tools/ci/build_apps.py ./examples --rcp_gateway --ignore_warning
+    - *build_rcp_gateway
diff --git a/.gitlab/ci/generate_build_test_jobs.yml b/.gitlab/ci/generate_build_test_jobs.yml
@@ -0,0 +1,43 @@
+generate_child_pipeline:
+  stage: generate
+  extends:
+    - .setup_esp_sdk_script
+  artifacts:
+    paths:
+      - child_pipeline_default.yml
+      - child_pipeline_lts.yml
+    expire_in: 1 week
+    when: always
+  script:
+    - cd $CI_PROJECT_DIR
+    - python3 $CI_PROJECT_DIR/esp-zigbee-sdk/tools/ci/dynamic_pipelines/generate_child_pipeline.py -p $CI_PROJECT_DIR -t .gitlab/ci/templates.yml -i ${DEFAULT_IDF_VERSION} -d ${DEFAULT_BUILD_DOCKER_ENV}
+
+build_and_test_pipeline_default:
+  stage: build
+  needs:
+    - generate_child_pipeline
+  inherit:
+    variables: true
+  trigger:
+    include:
+      - artifact: child_pipeline_default.yml
+        job: generate_child_pipeline
+    forward:
+      pipeline_variables: true
+    strategy: depend
+
+
+build_and_test_pipeline_lts:
+  stage: build
+  needs:
+    - generate_child_pipeline
+  inherit:
+    variables: true
+  when: manual
+  trigger:
+    include:
+      - artifact: child_pipeline_lts.yml
+        job: generate_child_pipeline
+    forward:
+      pipeline_variables: true
+    strategy: depend
diff --git a/.gitlab/ci/templates.yml b/.gitlab/ci/templates.yml
@@ -0,0 +1,92 @@
+include:
+  - local: '.gitlab/ci/common.yml'
+
+.build_non_pytest_example_template:
+  stage: build
+  image: ${CI_DOCKER_REGISTRY}/esp-env-${DOCKER_ENV_VERSION}
+  extends:
+    - .setup_build_environment_script
+  artifacts:
+    paths:
+      - "**/build*/*.bin"
+      - "**/build*/flasher_args.json"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+    when: always
+    expire_in: 3 days
+  script:
+    - cd $CI_PROJECT_DIR/esp-zigbee-sdk
+    - python tools/ci/build_apps.py ./examples --no_pytest --ignore_warning
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+  tags:
+    - build
+
+.build_pytest_example_template:
+  stage: build
+  image: ${CI_DOCKER_REGISTRY}/esp-env-${DOCKER_ENV_VERSION}
+  extends:
+    - .setup_build_environment_script
+  artifacts:
+    paths:
+      - "**/build*/*.bin"
+      - "**/build*/flasher_args.json"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+      - "**/build*/*.map"
+      - "**/build*/*.elf"
+      - "**/build*/flash_project_args"
+      - "**/esp-zigbee-sdk/conftest.py"
+      - "**/esp-zigbee-sdk/pytest.ini"
+      - "**/examples/*.py"
+      - "**/ci/requirements-pytest.txt"
+    when: always
+    expire_in: 3 days
+  script:
+    - cd $CI_PROJECT_DIR/esp-zigbee-sdk
+    - python tools/ci/build_apps.py ./examples --pytest --ignore_warning
+  tags:
+    - build
+
+.build_pytest_gateway_template:
+  stage: build
+  image: ${CI_DOCKER_REGISTRY}/esp-env-${DOCKER_ENV_VERSION}
+  extends:
+    - .setup_build_environment_script
+    - .build_gateway_script
+  artifacts:
+    paths:
+      - "**/build*/*.bin"
+      - "**/build*/flasher_args.json"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+      - "**/build*/*.map"
+      - "**/build*/*.elf"
+      - "**/build*/flash_project_args"
+      - "**/esp-zigbee-sdk/conftest.py"
+      - "**/esp-zigbee-sdk/pytest.ini"
+      - "**/examples/*.py"
+      - "**/ci/requirements-pytest.txt"
+    when: always
+    expire_in: 3 days
+  tags:
+    - build
+
+.pytest_template:
+  stage: target_test
+  image: ${CI_DOCKER_REGISTRY}/target-test-env-${DOCKER_ENV_VERSION}
+  artifacts:
+    paths:
+      - "$CI_PROJECT_DIR/esp-zigbee-sdk/pytest_embedded_log/"
+    when: always
+    expire_in: 3 days
+  script:
+    - cd $CI_PROJECT_DIR/esp-zigbee-sdk
+    - pip install -r tools/ci/requirements-pytest.txt
+    - find examples/ -name "pytest_*.py" | xargs pytest --target ${CHIP} -m zigbee_multi_dut
+  tags:
+    - ${CHIP}
+    - zigbee_multi_dut