From 57ad7257e8fdb5589e4425c47e51c3b69993690b Mon Sep 17 00:00:00 2001 From: liuqiang Date: Tue, 10 Dec 2024 16:49:51 +0800 Subject: [PATCH] ci: optimize ci framework --- .gitlab-ci.yml | 111 +----------------- .gitlab/ci/build_idf_examples.yml | 24 ++++ .gitlab/ci/common.yml | 81 +++++++++++++ .gitlab/ci/generate_build_test_jobs.yml | 43 +++++++ .gitlab/ci/templates.yml | 92 +++++++++++++++ tools/ci/build_apps.py | 146 ------------------------ 6 files changed, 244 insertions(+), 253 deletions(-) create mode 100644 .gitlab/ci/build_idf_examples.yml create mode 100644 .gitlab/ci/common.yml create mode 100644 .gitlab/ci/generate_build_test_jobs.yml create mode 100644 .gitlab/ci/templates.yml delete mode 100644 tools/ci/build_apps.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d1206ec..d8556b9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,107 +1,4 @@ -stages: - - build - -variables: - CI_DOCKER_REGISTRY: "ciregistry.espressif.cn:8443" - ESP_ZIGBEE_SDK_PATH: "$CI_PROJECT_DIR/esp-zigbee-sdk" - ESP_ZBOSS_LIB_PATH: "$CI_PROJECT_DIR" - -before_script: - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - echo -n "${GITLAB_KEY}" >~/.ssh/id_rsa_base64 - - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa - - chmod 600 ~/.ssh/id_rsa - - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >>~/.ssh/config - -.setup_idf: &setup_idf - - cd $CI_PROJECT_DIR/.. - - rm -rf esp-idf - - git clone --recursive -b v5.3.2 ${ESP_IDF_URL} - - cd esp-idf - - git checkout --track origin/$CI_COMMIT_REF_NAME || git branch - - git submodule sync --recursive - - git submodule update --recursive --init - - ./install.sh - - . ./export.sh - -.setup_esp_sdk: &setup_esp_sdk - - cd $CI_PROJECT_DIR - - git clone --recursive -b main ${ESP_ZIGBEE_SDK_URL} - - cd esp-zigbee-sdk - - git checkout --track origin/$CI_COMMIT_REF_NAME || git branch - - git submodule update --recursive --init - - cp -r ${ESP_ZIGBEE_SDK_PATH}/components/esp-zigbee-lib ${IDF_PATH}/components/espressif__esp-zigbee-lib - -.update_zboss_lib_script: &update_zboss_lib_script - - *setup_idf - - *setup_esp_sdk - - cd ${ESP_ZBOSS_LIB_PATH} - - cp -r ${ESP_ZBOSS_LIB_PATH} ${IDF_PATH}/components/espressif__esp-zboss-lib - - cd ${ESP_ZIGBEE_SDK_PATH} - - pip install -r tools/ci/requirements-build.txt - -build_non_pytest_examples: - stage: build - image: ${CI_DOCKER_REGISTRY}/esp-env-v5.3:1 - artifacts: - paths: - - "**/build*/*.bin" - - "**/build*/flasher_args.json" - - "**/build*/config/sdkconfig.json" - - "**/build*/bootloader/*.bin" - - "**/build*/partition_table/*.bin" - when: always - expire_in: 4 days - script: - - *update_zboss_lib_script - - python tools/ci/build_apps.py ./examples --no_pytest - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - parallel: 2 - tags: - - build - -build_pytest_examples: - stage: build - image: ${CI_DOCKER_REGISTRY}/esp-env-v5.3:1 - artifacts: - paths: - - "**/build*/*.bin" - - "**/build*/*.map" - - "**/build*/*.elf" - - "**/build*/flasher_args.json" - - "**/build*/flash_project_args" - - "**/build*/config/sdkconfig.json" - - "**/build*/bootloader/*.bin" - - "**/build*/partition_table/*.bin" - when: always - expire_in: 4 days - script: - - *update_zboss_lib_script - - python tools/ci/build_apps.py ./examples --pytest - tags: - - build - -build_idf_examples: - stage: build - image: ${CI_DOCKER_REGISTRY}/esp-env-v5.3:1 - artifacts: - paths: - - "**/build*/*.bin" - - "**/build*/*.map" - - "**/build*/*.elf" - - "**/build*/flasher_args.json" - - "**/build*/flash_project_args" - - "**/build*/config/sdkconfig.json" - - "**/build*/bootloader/*.bin" - - "**/build*/partition_table/*.bin" - when: always - expire_in: 4 days - script: - - *update_zboss_lib_script - - cp -r ${ESP_ZBOSS_LIB_PATH}/tools/ci/build_apps.py ${IDF_PATH}/examples/zigbee/build_apps.py - - cd ${IDF_PATH} - - python examples/zigbee/build_apps.py ./examples/zigbee --idf_examples - tags: - - build +include: + - '.gitlab/ci/common.yml' + - '.gitlab/ci/generate_build_test_jobs.yml' + - '.gitlab/ci/build_idf_examples.yml' diff --git a/.gitlab/ci/build_idf_examples.yml b/.gitlab/ci/build_idf_examples.yml new file mode 100644 index 0000000..5e42a93 --- /dev/null +++ b/.gitlab/ci/build_idf_examples.yml @@ -0,0 +1,24 @@ +build_idf_examples: + stage: build + image: ${CI_DOCKER_REGISTRY}/esp-env-${DEFAULT_BUILD_DOCKER_ENV} + extends: + - .setup_build_environment_script + artifacts: + paths: + - "**/build*/*.bin" + - "**/build*/*.map" + - "**/build*/*.elf" + - "**/build*/flasher_args.json" + - "**/build*/flash_project_args" + - "**/build*/config/sdkconfig.json" + - "**/build*/bootloader/*.bin" + - "**/build*/partition_table/*.bin" + when: always + expire_in: 4 days + variables: + IDF_VERSION: ${DEFAULT_IDF_VERSION} + script: + - cd ${IDF_PATH} + - python ${CI_PROJECT_DIR}/esp-zigbee-sdk/tools/ci/build_apps.py ./examples/zigbee + tags: + - build diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml new file mode 100644 index 0000000..ff55e1d --- /dev/null +++ b/.gitlab/ci/common.yml @@ -0,0 +1,81 @@ +stages: + - generate + - build + - target_test + +variables: + GIT_SUBMODULE_STRATEGY: none + DEFAULT_IDF_VERSION: "v5.3.2" + DEFAULT_BUILD_DOCKER_ENV: "v5.3:1" + # tell build system do not check submodule update as we download archive instead of clone + IDF_SKIP_CHECK_SUBMODULES: 1 + +.init_ssh: &init_ssh + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - echo -n "${GITLAB_KEY}" >~/.ssh/id_rsa_base64 + - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa + - chmod 600 ~/.ssh/id_rsa + - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >>~/.ssh/config + +.setup_idf: &setup_idf + - cd $CI_PROJECT_DIR + - git clone --recursive ${ESP_IDF_URL} + - cd esp-idf + - git checkout --track origin/$CI_COMMIT_REF_NAME || git checkout ${IDF_VERSION} + - git submodule sync --recursive + - git submodule update --recursive --init + - ./install.sh + - . ./export.sh + +.setup_esp_sdk: &setup_esp_sdk + - cd $CI_PROJECT_DIR + - git clone --recursive -b main ${ZIGBEE_SDK_REPO_URL} + - cd esp-zigbee-sdk + - git checkout --track origin/$CI_COMMIT_REF_NAME || git branch + +.update_cmake_dependency: &update_cmake_dependency + - echo ${IDF_PATH} + - cp -r $CI_PROJECT_DIR/esp-zigbee-sdk/components/esp-zigbee-lib ${IDF_PATH}/components/espressif__esp-zigbee-lib + # copy esp-zboss-lib to esp-idf + - mkdir ${IDF_PATH}/components/espressif__esp-zboss-lib + - cp -r $CI_PROJECT_DIR/include ${IDF_PATH}/components/espressif__esp-zboss-lib + - cp -r $CI_PROJECT_DIR/lib ${IDF_PATH}/components/espressif__esp-zboss-lib + - cp $CI_PROJECT_DIR/idf_component.yml ${IDF_PATH}/components/espressif__esp-zboss-lib + - cp $CI_PROJECT_DIR/CMakeLists.txt ${IDF_PATH}/components/espressif__esp-zboss-lib + - ls -lf ${IDF_PATH}/components/espressif__esp-zboss-lib + # end copy + - sed -i "/\besp-zboss-lib\b/d;/\besp-zigbee-lib\b/d" ${IDF_PATH}/examples/zigbee/**/main/idf_component.yml + - sed -i "/\besp-zboss-lib\b/d;/\besp-zigbee-lib\b/d" $CI_PROJECT_DIR/esp-zigbee-sdk/examples/**/main/idf_component.yml + - sed -i "/\besp-zboss-lib\b/d;/\besp-zigbee-lib\b/d" $CI_PROJECT_DIR/esp-zigbee-sdk/components/*/idf_component.yml + +.build_rcp_gateway: &build_rcp_gateway + - mv $CI_PROJECT_DIR/esp-zigbee-sdk/examples/esp_zigbee_gateway/sdkconfig.defaults $CI_PROJECT_DIR/esp-zigbee-sdk/examples/esp_zigbee_gateway/sdkconfig.defaults.ori + - cp $CI_PROJECT_DIR/esp-zigbee-sdk/tools/ci/sdkconfig_for_gateway_ci.defaults $CI_PROJECT_DIR/esp-zigbee-sdk/examples/esp_zigbee_gateway/sdkconfig.defaults + - cd $CI_PROJECT_DIR/esp-idf/examples/openthread/ot_rcp + - echo "CONFIG_OPENTHREAD_NCP_VENDOR_HOOK=y" >> sdkconfig.defaults + - idf.py set-target esp32h2 + - idf.py build + - cd $CI_PROJECT_DIR/esp-zigbee-sdk/examples/esp_zigbee_gateway + - idf.py set-target esp32s3 + - idf.py build + - mv $CI_PROJECT_DIR/esp-zigbee-sdk/examples/esp_zigbee_gateway/sdkconfig.defaults.ori $CI_PROJECT_DIR/esp-zigbee-sdk/examples/esp_zigbee_gateway/sdkconfig.defaults + - cd ${CI_PROJECT_DIR} + +.setup_build_environment_script: + before_script: + - *init_ssh + - *setup_idf + - *setup_esp_sdk + - *update_cmake_dependency + - pip install -r tools/ci/requirements-build.txt + +.setup_esp_sdk_script: + before_script: + - *init_ssh + - *setup_esp_sdk + +.build_gateway_script: + script: + - python tools/ci/build_apps.py ./examples --rcp_gateway --ignore_warning + - *build_rcp_gateway diff --git a/.gitlab/ci/generate_build_test_jobs.yml b/.gitlab/ci/generate_build_test_jobs.yml new file mode 100644 index 0000000..6721cd4 --- /dev/null +++ b/.gitlab/ci/generate_build_test_jobs.yml @@ -0,0 +1,43 @@ +generate_child_pipeline: + stage: generate + extends: + - .setup_esp_sdk_script + artifacts: + paths: + - child_pipeline_default.yml + - child_pipeline_lts.yml + expire_in: 1 week + when: always + script: + - cd $CI_PROJECT_DIR + - python3 $CI_PROJECT_DIR/esp-zigbee-sdk/tools/ci/dynamic_pipelines/generate_child_pipeline.py -p $CI_PROJECT_DIR -t .gitlab/ci/templates.yml -i ${DEFAULT_IDF_VERSION} -d ${DEFAULT_BUILD_DOCKER_ENV} + +build_and_test_pipeline_default: + stage: build + needs: + - generate_child_pipeline + inherit: + variables: true + trigger: + include: + - artifact: child_pipeline_default.yml + job: generate_child_pipeline + forward: + pipeline_variables: true + strategy: depend + + +build_and_test_pipeline_lts: + stage: build + needs: + - generate_child_pipeline + inherit: + variables: true + when: manual + trigger: + include: + - artifact: child_pipeline_lts.yml + job: generate_child_pipeline + forward: + pipeline_variables: true + strategy: depend diff --git a/.gitlab/ci/templates.yml b/.gitlab/ci/templates.yml new file mode 100644 index 0000000..4f97424 --- /dev/null +++ b/.gitlab/ci/templates.yml @@ -0,0 +1,92 @@ +include: + - local: '.gitlab/ci/common.yml' + +.build_non_pytest_example_template: + stage: build + image: ${CI_DOCKER_REGISTRY}/esp-env-${DOCKER_ENV_VERSION} + extends: + - .setup_build_environment_script + artifacts: + paths: + - "**/build*/*.bin" + - "**/build*/flasher_args.json" + - "**/build*/config/sdkconfig.json" + - "**/build*/bootloader/*.bin" + - "**/build*/partition_table/*.bin" + when: always + expire_in: 3 days + script: + - cd $CI_PROJECT_DIR/esp-zigbee-sdk + - python tools/ci/build_apps.py ./examples --no_pytest --ignore_warning + --parallel-count ${CI_NODE_TOTAL:-1} + --parallel-index ${CI_NODE_INDEX:-1} + tags: + - build + +.build_pytest_example_template: + stage: build + image: ${CI_DOCKER_REGISTRY}/esp-env-${DOCKER_ENV_VERSION} + extends: + - .setup_build_environment_script + artifacts: + paths: + - "**/build*/*.bin" + - "**/build*/flasher_args.json" + - "**/build*/config/sdkconfig.json" + - "**/build*/bootloader/*.bin" + - "**/build*/partition_table/*.bin" + - "**/build*/*.map" + - "**/build*/*.elf" + - "**/build*/flash_project_args" + - "**/esp-zigbee-sdk/conftest.py" + - "**/esp-zigbee-sdk/pytest.ini" + - "**/examples/*.py" + - "**/ci/requirements-pytest.txt" + when: always + expire_in: 3 days + script: + - cd $CI_PROJECT_DIR/esp-zigbee-sdk + - python tools/ci/build_apps.py ./examples --pytest --ignore_warning + tags: + - build + +.build_pytest_gateway_template: + stage: build + image: ${CI_DOCKER_REGISTRY}/esp-env-${DOCKER_ENV_VERSION} + extends: + - .setup_build_environment_script + - .build_gateway_script + artifacts: + paths: + - "**/build*/*.bin" + - "**/build*/flasher_args.json" + - "**/build*/config/sdkconfig.json" + - "**/build*/bootloader/*.bin" + - "**/build*/partition_table/*.bin" + - "**/build*/*.map" + - "**/build*/*.elf" + - "**/build*/flash_project_args" + - "**/esp-zigbee-sdk/conftest.py" + - "**/esp-zigbee-sdk/pytest.ini" + - "**/examples/*.py" + - "**/ci/requirements-pytest.txt" + when: always + expire_in: 3 days + tags: + - build + +.pytest_template: + stage: target_test + image: ${CI_DOCKER_REGISTRY}/target-test-env-${DOCKER_ENV_VERSION} + artifacts: + paths: + - "$CI_PROJECT_DIR/esp-zigbee-sdk/pytest_embedded_log/" + when: always + expire_in: 3 days + script: + - cd $CI_PROJECT_DIR/esp-zigbee-sdk + - pip install -r tools/ci/requirements-pytest.txt + - find examples/ -name "pytest_*.py" | xargs pytest --target ${CHIP} -m zigbee_multi_dut + tags: + - ${CHIP} + - zigbee_multi_dut diff --git a/tools/ci/build_apps.py b/tools/ci/build_apps.py deleted file mode 100644 index 77bab9d..0000000 --- a/tools/ci/build_apps.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD - -# SPDX-License-Identifier: CC0-1.0 - -# This file is used in CI generate binary files for different kinds of apps - -import argparse -import sys -import logging -import os -import subprocess -from pathlib import Path -from typing import List - -idf_build_apps_logger = logging.getLogger('idf_build_apps') -from idf_build_apps import App, build_apps, find_apps, setup_logging - -# from idf_ci_utils import IDF_PATH, get_pytest_app_paths, get_pytest_cases, get_ttfw_app_paths - -PROJECT_ROOT = Path(__file__).parent.parent.parent.absolute() -DEF_APP_PATH = PROJECT_ROOT / 'examples' / 'zigbee' -APPS_BUILD_PER_JOB = 30 -BUILD_APPS_IDF = [ - -] -IGNORE_WARNINGS = [ - r"warning: 'init_spiffs' defined but not used", - r"warning: 'esp_zb_gateway_board_try_update' defined but not used", -] -MAINFEST_FILES = [ - str(PROJECT_ROOT / 'examples' / 'zigbee' / '.build-test-rules.yml'), -] - -def _is_build_app_idf(app: App) -> bool: - for build_app in BUILD_APPS_IDF: - if app.name == build_app["name"] and app.target == build_app["target"]: - return True - return False - -def get_cmake_apps( - paths: List[str], - target: str, - config_rules_str: List[str], -) -> List[App]: - apps = find_apps( - paths, - recursive=True, - target=target, - build_dir='build_@t_@w', - config_rules_str=config_rules_str, - check_warnings=True, - preserve=True, - manifest_files=MAINFEST_FILES, - ) - return apps - -def update_component_yml_files(): - idf_path = "${IDF_PATH}" - command = f'sed -i "/\\besp-zboss-lib\\b/d;/\\besp-zigbee-lib\\b/d" "{idf_path}/examples/zigbee/esp_zigbee_gateway/main/idf_component.yml"' - subprocess.run(command, shell=True) - command = f'sed -i "/\\besp-zboss-lib\\b/d;/\\besp-zigbee-lib\\b/d" "{idf_path}/examples/zigbee/light_sample/HA_on_off_light/main/idf_component.yml"' - subprocess.run(command, shell=True) - command = f'sed -i "/\\besp-zboss-lib\\b/d;/\\besp-zigbee-lib\\b/d" "{idf_path}/examples/zigbee/light_sample/HA_on_off_switch/main/idf_component.yml"' - subprocess.run(command, shell=True) - -def main(args: argparse.Namespace) -> None: - current_dir = os.getcwd() - update_component_yml_files() - os.chdir(current_dir) - apps = get_cmake_apps(args.paths, args.target, args.config) - - # no_pytest and only_pytest can not be both True - if args.idf_examples: - apps_for_build = [app for app in apps if not _is_build_app_idf(app)] - else: - apps_for_build = apps[:] - - logging.info('Found %d apps after filtering', len(apps_for_build)) - logging.info( - 'Suggest setting the parallel count to %d for this build job', - len(apps_for_build) // APPS_BUILD_PER_JOB + 1, - ) - - ret_code = build_apps( - apps_for_build, - parallel_count=args.parallel_count, - parallel_index=args.parallel_index, - dry_run=False, - collect_size_info=args.collect_size_info, - # build_verbose=0, - keep_going=True, - ignore_warning_strs=IGNORE_WARNINGS, - copy_sdkconfig=True, - ) - - sys.exit(ret_code) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description='Build all the apps for different test types. Will auto remove those non-test apps binaries', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument('paths', nargs='*', help='Paths to the apps to build.') - parser.add_argument( - '-t', - '--target', - default='all', - help='Build apps for given target. could pass "all" to get apps for all targets', - ) - parser.add_argument( - '--config', - default=['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'], - action='append', - help='Adds configurations (sdkconfig file names) to build. This can either be ' - 'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, ' - 'relative to the project directory, to be used. Optional NAME can be specified, ' - 'which can be used as a name of this configuration. FILEPATTERN is the name of ' - 'the sdkconfig file, relative to the project directory, with at most one wildcard. ' - 'The part captured by the wildcard is used as the name of the configuration.', - ) - parser.add_argument( - '--parallel-count', default=1, type=int, help='Number of parallel build jobs.' - ) - parser.add_argument( - '--parallel-index', - default=1, - type=int, - help='Index (1-based) of the job, out of the number specified by --parallel-count.', - ) - parser.add_argument( - '--idf_examples', - action="store_true", - help='Only build pytest apps, definded in PYTEST_APPS_IDF', - ) - parser.add_argument( - '--collect-size-info', - type=argparse.FileType('w'), - help='If specified, the test case name and size info json will be written to this file', - ) - - arguments = parser.parse_args() - if not arguments.paths: - arguments.paths = [DEF_APP_PATH] - setup_logging(verbose=1) # Info - main(arguments) \ No newline at end of file