From cde29eeea2d77024770a50819cfb961604ed2c31 Mon Sep 17 00:00:00 2001 From: Tomas Baca Date: Wed, 15 Nov 2023 14:07:48 +0100 Subject: [PATCH] towards rostests --- .ci/get_test_matrix.sh | 37 ++++ .ci/parse_yaml.py | 16 +- .github/workflows/rostest.yml | 78 ++++++++ mrs.yaml | 336 +++++++++++++++++++++------------- nonbloom.yaml | 35 ++-- scripts/clone_all.sh | 2 +- thirdparty.yaml | 70 ++++--- 7 files changed, 398 insertions(+), 176 deletions(-) create mode 100755 .ci/get_test_matrix.sh create mode 100644 .github/workflows/rostest.yml diff --git a/.ci/get_test_matrix.sh b/.ci/get_test_matrix.sh new file mode 100755 index 0000000000..cb16191dba --- /dev/null +++ b/.ci/get_test_matrix.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -e + +trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG +trap 'echo "$0: \"${last_command}\" command failed with exit code $?, log:" && cat /tmp/log.txt' ERR + +DEBUG=false + +LIST=mrs +ARCH=amd64 + +YAML_FILE=$LIST.yaml + +REPOS=$(./.ci/parse_yaml.py $YAML_FILE $ARCH) + +FIRST=true + +echo "[" + +echo "$REPOS" | while IFS= read -r REPO; do + + $DEBUG && echo "Cloning $REPO" + + PACKAGE=$(echo "$REPO" | awk '{print $1}') + URL=$(echo "$REPO" | awk '{print $2}') + TEST=$(echo "$REPO" | awk '{print $6}') + + if [[ "$TEST" != "True" ]]; then + continue + fi + + echo "$PACKAGE," + +done + +echo "]" diff --git a/.ci/parse_yaml.py b/.ci/parse_yaml.py index c7dbd39d7f..6a831b91c3 100755 --- a/.ci/parse_yaml.py +++ b/.ci/parse_yaml.py @@ -31,23 +31,31 @@ def main(): stable_ref = "none" testing_ref = "none" unstable_ref = "none" + ros_test = 0 + + refs = properties['git_refs'] + + try: + stable_ref = refs['stable'] + except: + pass try: - stable_ref = properties['stable_ref'] + testing_ref = refs['testing'] except: pass try: - testing_ref = properties['testing_ref'] + unstable_ref = refs['unstable'] except: pass try: - unstable_ref = properties['unstable_ref'] + ros_test = bool(properties['ros_test']) except: pass - print("{} {} {} {} {}".format(package, url, stable_ref, testing_ref, unstable_ref)) + print("{} {} {} {} {} {}".format(package, url, stable_ref, testing_ref, unstable_ref, ros_test)) if __name__ == '__main__': main() diff --git a/.github/workflows/rostest.yml b/.github/workflows/rostest.yml new file mode 100644 index 0000000000..fdcd434b3d --- /dev/null +++ b/.github/workflows/rostest.yml @@ -0,0 +1,78 @@ +name: rostest + +on: + workflow_dispatch: + + # schedule: + # - cron: '0 20 * * *' # every day at 10pm UTC+2 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + +jobs: + + generate-jobs: + runs-on: ubuntu-20.04 + outputs: + packages: ${{ steps.generate.outputs.packages }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'recursive' + - name: Checkout CI scripts + uses: actions/checkout@v3 + with: + repository: ctu-mrs/ci_scripts + ref: master + path: .ci_scripts + token: ${{ secrets.PUSH_TOKEN }} + - id: generate + run: | + JOB_STRATEGY_MATRIX=$(./.ci/get_test_matrix.sh) + cat /tmp/log.txt + echo "packages=$JOB_STRATEGY_MATRIX" >> "$GITHUB_OUTPUT" + + build-job: + needs: generate-jobs + runs-on: ubuntu-20.04 + timeout-minutes: 360 # 6 hour timeout + strategy: + matrix: + job: ${{ fromJson(needs.generate-jobs.outputs.packages) }} + max-parallel: 1 # Run jobs serially + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'recursive' + - name: Checkout CI scripts + uses: actions/checkout@v3 + with: + repository: ctu-mrs/ci_scripts + ref: master + path: .ci_scripts + token: ${{ secrets.PUSH_TOKEN }} + - id: build + run: | + echo "testing..." + + merge: + runs-on: ubuntu-20.04 + needs: build-job + env: + PUSH_TOKEN: ${{ secrets.PUSH_TOKEN }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'recursive' + - name: Checkout CI scripts + uses: actions/checkout@v3 + with: + repository: ctu-mrs/ci_scripts + ref: master + path: .ci_scripts + token: ${{ secrets.PUSH_TOKEN }} + - name: Merge + run: echo "Merging release_candidate to release" diff --git a/mrs.yaml b/mrs.yaml index c6d77fa60f..c17e3fdacd 100644 --- a/mrs.yaml +++ b/mrs.yaml @@ -1,293 +1,377 @@ mrs_msgs: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_msgs - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_modules_msgs: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_modules_msgs - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_lib: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_lib - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: true mrs_uav_managers: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_managers - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: true mrs_bumper: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_bumper - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_multirotor_simulator: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_multirotor_simulator - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_controllers: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_controllers - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_autostart: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_autostart - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_hw_api: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_hw_api - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_state_estimators: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_state_estimators - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_deployment: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_deployment - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_status: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_status - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_trackers: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_trackers - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_px4_api: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_px4_api - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_dji_tello_api: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_dji_tello_api - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_rviz_plugins: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_rviz_plugins - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_trajectory_generation: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_trajectory_generation - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_serial: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_serial - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_path_loader: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_path_loader - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_trajectory_loader: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_trajectory_loader - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_utils: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_utils - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_gazebo_common_resources: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_gazebo_common_resources - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_pcl_tools: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_pcl_tools - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_subt_planning_lib: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_subt_planning_lib - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_octomap_tools: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_octomap_tools - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_octomap_server: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_octomap_server - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_octomap_planner: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_octomap_planner - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_octomap_mapping_planning: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_octomap_mapping_planning - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_coppelia_simulation: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_coppelia_simulation - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_gazebo_simulation: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_gazebo_simulation - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false aloam: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/aloam - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_modules: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_modules - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_core: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_core - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_uav_system: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_uav_system - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_hector_estimator_plugin: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_hector_estimator_plugin - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_hector_core: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_hector_core - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_open_vins_estimator_plugin: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_open_vins_estimator_plugin - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_vins_republisher: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_vins_republisher - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_vins_imu_filter: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_vins_imu_filter - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_open_vins_core: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_open_vins_core - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_aloam_estimator_plugin: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_aloam_estimator_plugin - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false mrs_aloam_core: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mrs_aloam_core - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release + ros_test: false diff --git a/nonbloom.yaml b/nonbloom.yaml index e67c5151b6..298bed5166 100644 --- a/nonbloom.yaml +++ b/nonbloom.yaml @@ -1,34 +1,39 @@ mrs_uav_shell_additions: architecture: [amd64] # this package is for all architectures source: https://github.com/ctu-mrs/mrs_uav_shell_additions - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release mrs_uav_usb_configurator: architecture: [amd64] # this package is for all architectures source: https://github.com/ctu-mrs/mrs_uav_usb_configurator - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release mrs_geographiclib_datasets: architecture: [amd64] # this package is for all architectures source: https://github.com/ctu-mrs/mrs-geographiclib-datasets - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release coppelia_sim_edu: architecture: [amd64] # this package is for all architectures source: https://github.com/ctu-mrs/coppelia_sim_edu - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release px4_firmware: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/px4_firmware - unstable_ref: 1.13.2-dev - testing_ref: release_candidate - stable_ref: mrs_release + git_refs: + unstable: 1.13.2-dev + testing: release_candidate + stable: mrs_release diff --git a/scripts/clone_all.sh b/scripts/clone_all.sh index 47cf6fdf91..80a936acd2 100755 --- a/scripts/clone_all.sh +++ b/scripts/clone_all.sh @@ -31,7 +31,7 @@ do PACKAGE=$(echo "$REPO" | awk '{print $1}') URL=$(echo "$REPO" | awk '{print $2}') STABLE_BRANCH=$(echo "$REPO" | awk '{print $3}') - UNSTABLE_BRANCH=$(echo "$REPO" | awk '{print $4}') + UNSTABLE_BRANCH=$(echo "$REPO" | awk '{print $5}') if [ -e ./$PACKAGE ]; then diff --git a/thirdparty.yaml b/thirdparty.yaml index 163843285b..cdfd733804 100644 --- a/thirdparty.yaml +++ b/thirdparty.yaml @@ -1,69 +1,79 @@ px4_sitl_gazebo: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/px4_sitl_gazebo - unstable_ref: firmware_1.13.2-deb - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: firmware_1.13.2-deb + testing: release_candidate + stable: release nlopt_ros: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/nlopt_ros - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release ouster-ros: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/ouster-ros - unstable_ref: mrs - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: mrs + testing: release_candidate + stable: release mavros: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mavros - unstable_ref: 1.13.0-deb - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: 1.13.0-deb + testing: release_candidate + stable: release mav_comm: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/mav_comm - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release ros_comm: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/ros_comm - unstable_ref: rosbag-patch - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: rosbag-patch + testing: release_candidate + stable: release geometry2: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/geometry2 - unstable_ref: warning-fix - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: warning-fix + testing: release_candidate + stable: release nimbro_network: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/nimbro_network - unstable_ref: develop-deb - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: develop-deb + testing: release_candidate + stable: release hector_slam: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/hector_slam - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release open_vins: architecture: [amd64, arm64] source: https://github.com/ctu-mrs/open_vins - unstable_ref: master - testing_ref: release_candidate - stable_ref: release + git_refs: + unstable: master + testing: release_candidate + stable: release