integ-test #797
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: integ-test | |
| on: | |
| schedule: | |
| - cron: "0 4 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| integ_tests_include: | |
| type: string | |
| description: |- | |
| List integration tests to include. | |
| Use "*" to run all tests. | |
| Use regex like 'node|^git_issue' to run only a subset. | |
| default: "*" | |
| # dns_config|time_server - NTP cannot be reconfigured if DNS is invalid | |
| # git_issues - slow, do not run on each push. TODO - run them only once a day | |
| # oidc_config - during reconfiguration API returns 500/502 errors for other requests | |
| # utils_login - it uses OIDC user to login | |
| # smtp - email_alert test requires a configured SMTP | |
| # role_cluster_config - role cluster_config reconfigures DNS, SMTP, OIDC. And it is slow. | |
| # version_update_single_node - role would change version, VSNS system cannot be updated. | |
| # vm_replication vm_clone__replicated - VSNS were not configured with replication | |
| integ_tests_exclude: | |
| type: string | |
| description: |- | |
| List integration tests to exclude. | |
| Use "*" to exclude all tests. | |
| Use regex like 'node|^git_issue|^dns_config$' to exclude only a subset. | |
| default: "^dns_config$|^cluster_shutdown$|^version_update$|^oidc_config$|^smtp$|^role_cluster_config$|^role_version_update_single_node$|^version_update__shutdown_restart_vms$|^utils_login$|^certificate$|^vm_replication$|^vm_replication_info$|^vm_clone__replicated$|^support_tunnel$" | |
| integ_seq_tests_include: | |
| type: string | |
| description: |- | |
| List seq integration tests to include. | |
| # We want to run sequentially all tests that cannot run in parallel (e.g. all that listed in integ_tests_exclude). | |
| # Some tests can be tested on VSNS - those are run on VSNS. | |
| # Some cannot, those are run on test cluster 10.5.11.50. | |
| # Some cannot be tested at all - shutdown and version update. | |
| default: "^dns_config$|^oidc_config$|^smtp$|^role_cluster_config$|^version_update__shutdown_restart_vms$|^utils_login$|^certificate$" | |
| integ_seq_tests_exclude: | |
| type: string | |
| description: |- | |
| List seq integration tests to exclude. | |
| default: "" | |
| examples_tests_include: | |
| type: string | |
| description: |- | |
| List examples to include into testing. | |
| Use "*" to test all examples. | |
| default: "iso_info" | |
| examples_tests_exclude: | |
| type: string | |
| description: |- | |
| List examples to exclude from testing. | |
| default: "^version_update_single_node.yml$|^vm_os_upgrade$|^vm_snapshot_attach_disk.yml$|^url2template$" | |
| env: | |
| INTEG_TESTS_INCLUDE_SCHEDULE: "*" | |
| INTEG_TESTS_EXCLUDE_SCHEDULE: "^dns_config$|^cluster_shutdown$|^version_update$|^oidc_config$|^smtp$|^role_cluster_config$|^role_version_update_single_node$|^version_update__shutdown_restart_vms$|^utils_login$|^certificate$|^vm_replication$|^vm_replication_info$|^vm_clone__replicated$|^support_tunnel$" | |
| INTEG_SEQ_TESTS_INCLUDE_SCHEDULE: "^dns_config$|^oidc_config$|^smtp$|^role_cluster_config$|^version_update__shutdown_restart_vms$|^utils_login$|^certificate$" | |
| INTEG_SEQ_TESTS_EXCLUDE_SCHEDULE: "" | |
| EXAMPLES_TESTS_INCLUDE_SCHEDULE: "*" | |
| EXAMPLES_TESTS_EXCLUDE_SCHEDULE: "^version_update_single_node.yml$|^vm_os_upgrade$|^vm_snapshot_attach_disk.yml$|^url2template$" | |
| # ansible-test needs special directory structure. | |
| # WORKDIR is a subdir of GITHUB_WORKSPACE | |
| WORKDIR: work-dir/ansible_collections/scale_computing/hypercore | |
| LANG: C.UTF-8 | |
| # Run only one workflow for specific branch. | |
| concurrency: | |
| group: ${{ github.ref_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| # This should be run at least once before first integration tests run. | |
| # As long as prepared HC3 host remains prepared, we do not need (except on very first run) | |
| # to delay integ-test until integration-prepare-env finishes. | |
| integration-prepare-env: | |
| runs-on: [self-hosted2] | |
| container: quay.io/justinc1_github/scale_ci_integ:10 | |
| env: | |
| ANSIBLE_COLLECTIONS_PATH: $GITHUB_WORKSPACE/work-dir | |
| defaults: | |
| run: | |
| working-directory: ${{ env.WORKDIR }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| sc_host: | |
| - https://10.5.11.200 # HC 9.1.x | |
| - https://10.5.11.201 # HC 9.2.x | |
| - https://10.5.11.203 # HC 9.3.x | |
| - https://10.5.11.205 # HC 9.4.x | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| path: ${{ env.WORKDIR }} | |
| - run: ansible-galaxy collection install community.crypto community.general | |
| - run: ansible-galaxy collection list | |
| - uses: ./work-dir/ansible_collections/scale_computing/hypercore/.github/actions/make-integ-config | |
| with: | |
| sc_host: ${{ matrix.sc_host }} | |
| sc_password_50: ${{ secrets.CI_CONFIG_HC_IP50_SC_PASSWORD }} | |
| smb_password: ${{ secrets.CI_CONFIG_HC_IP50_SMB_PASSWORD }} | |
| oidc_client_secret: ${{ secrets.OIDC_CLIENT_SECRET }} | |
| oidc_users_0_password: ${{ secrets.OIDC_USERS_0_PASSWORD }} | |
| working_directory: ${{ env.WORKDIR }} | |
| - run: ansible-playbook tests/integration/prepare/prepare_iso.yml | |
| - run: ansible-playbook tests/integration/prepare/prepare_vm.yml | |
| - run: ansible-playbook tests/integration/prepare/prepare_examples.yml | |
| if: "${{ github.event.inputs.integ_tests_include || github.event.schedule }}" | |
| integ-matrix: | |
| runs-on: [ubuntu-latest] | |
| container: quay.io/justinc1_github/scale_ci_integ:10 | |
| defaults: | |
| run: | |
| working-directory: ${{ env.WORKDIR }} | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| path: ${{ env.WORKDIR }} | |
| # We want to run all integ tests periodically - "integ_tests_include || '*'", the '*' is used. | |
| # When running with workflow-dispatch, user is required to put some non-empty string into integ_tests_include. | |
| - run: echo 'INTEG_TESTS_INCLUDE=${{ github.event.inputs.integ_tests_include || env.INTEG_TESTS_INCLUDE_SCHEDULE }}' >> $GITHUB_ENV | |
| - run: echo 'INTEG_TESTS_EXCLUDE=${{ github.event.inputs.integ_tests_exclude || env.INTEG_TESTS_EXCLUDE_SCHEDULE }}' >> $GITHUB_ENV | |
| - id: set-matrix | |
| shell: bash | |
| run: |- | |
| echo "matrix=$( | |
| ls -r tests/integration/targets | | |
| grep -v -E "${{ env.INTEG_TESTS_EXCLUDE || '^dummy-no-such-test$' }}" | | |
| grep -E "${{ env.INTEG_TESTS_INCLUDE }}" | | |
| jq -R -s -c 'split("\n")[:-1]' | |
| )" >> $GITHUB_OUTPUT | |
| if: "${{ github.event.inputs.integ_tests_include || github.event.schedule }}" | |
| integ-seq-matrix: | |
| needs: | |
| - integ-matrix | |
| runs-on: [ubuntu-latest] | |
| container: quay.io/justinc1_github/scale_ci_integ:10 | |
| defaults: | |
| run: | |
| working-directory: ${{ env.WORKDIR }} | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| path: ${{ env.WORKDIR }} | |
| - run: echo 'INTEG_SEQ_TESTS_INCLUDE=${{ github.event.inputs.integ_seq_tests_include || env.INTEG_SEQ_TESTS_INCLUDE_SCHEDULE }}' >> $GITHUB_ENV | |
| - run: echo 'INTEG_SEQ_TESTS_EXCLUDE=${{ github.event.inputs.integ_seq_tests_exclude || env.INTEG_SEQ_TESTS_EXCLUDE_SCHEDULE }}' >> $GITHUB_ENV | |
| - id: set-matrix | |
| shell: bash | |
| run: |- | |
| echo "matrix=$( | |
| ls -r tests/integration/targets | | |
| grep -v -E "${{ env.INTEG_SEQ_TESTS_EXCLUDE || '^dummy-no-such-test$' }}" | | |
| grep -E "${{ env.INTEG_SEQ_TESTS_INCLUDE }}" | | |
| jq -R -s -c 'split("\n")[:-1]' | |
| )" >> $GITHUB_OUTPUT | |
| if: "${{ github.event.inputs.integ_seq_tests_include || github.event.schedule }}" | |
| examples-matrix: | |
| needs: | |
| - integ-seq-matrix | |
| runs-on: [ ubuntu-latest ] | |
| container: quay.io/justinc1_github/scale_ci_integ:10 | |
| defaults: | |
| run: | |
| working-directory: ${{ env.WORKDIR }} | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| path: ${{ env.WORKDIR }} | |
| - run: echo 'EXAMPLES_TESTS_INCLUDE=${{ github.event.inputs.examples_tests_include || env.EXAMPLES_TESTS_INCLUDE_SCHEDULE }}' >> $GITHUB_ENV | |
| - run: echo 'EXAMPLES_TESTS_EXCLUDE=${{ github.event.inputs.examples_tests_exclude || env.EXAMPLES_TESTS_EXCLUDE_SCHEDULE }}' >> $GITHUB_ENV | |
| # A few files are not an example playbook: | |
| # hypercore_inventory.yml - inventory example | |
| # cloud-init-user-data-example.yml - cloud-init user-data example | |
| - id: set-matrix | |
| shell: bash | |
| run: |- | |
| echo "matrix=$( | |
| ls -r examples | | |
| grep -v -E "^README.md$|^hypercore_inventory.yml$|^cloud-init-user-data-example.yml$" | | |
| grep -v -E "${{ env.EXAMPLES_TESTS_EXCLUDE || '^dummy-no-such-example$' }}" | | |
| grep -E "${{ env.EXAMPLES_TESTS_INCLUDE }}" | | |
| jq -R -s -c 'split("\n")[:-1]' | |
| )" >> $GITHUB_OUTPUT | |
| if: "${{ github.event.inputs || github.event.schedule }}" | |
| examples-run: | |
| needs: | |
| - examples-matrix | |
| # run examples after integ - they will both try to modify say syslog_server etc | |
| - integ-seq-run | |
| if: "(!cancelled()) && (needs.examples-matrix.result=='success')" | |
| runs-on: [ self-hosted2 ] | |
| container: quay.io/justinc1_github/scale_ci_integ:10 | |
| env: | |
| ANSIBLE_COLLECTIONS_PATH: $GITHUB_WORKSPACE/work-dir | |
| DEBIAN_FRONTEND: noninteractive | |
| defaults: | |
| run: | |
| working-directory: ${{ env.WORKDIR }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ansible: [ 2.16.0 ] | |
| example_name: ${{ fromJson(needs.examples-matrix.outputs.matrix) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| path: ${{ env.WORKDIR }} | |
| - name: SKIP test if it succeeded before | |
| shell: bash | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| X_GITHUB_JOB_NAME: "examples-run (${{ matrix.ansible }}, ${{ matrix.example_name }})" | |
| run: | | |
| if ! .github/job_retry_needed.py | |
| then | |
| echo SKIP test, job retry not needed | |
| echo 'X_GITHUB_SKIP_TEST=1' >> $GITHUB_ENV | |
| else | |
| echo 'X_GITHUB_SKIP_TEST=0' >> $GITHUB_ENV | |
| fi | |
| - name: Install prerequisites | |
| shell: bash | |
| run: | | |
| [ "$X_GITHUB_SKIP_TEST" == "1" ] && exit 0 | |
| pip install ansible-core~=${{ matrix.ansible }} | |
| ansible-galaxy collection install community.crypto community.general | |
| ansible-galaxy collection list | |
| # ${{ env.WORKDIR }} cannot be used in "uses:" | |
| - uses: ./work-dir/ansible_collections/scale_computing/hypercore/.github/actions/make-integ-config | |
| with: | |
| sc_host: https://10.5.11.205 | |
| sc_password_50: ${{ secrets.CI_CONFIG_HC_IP50_SC_PASSWORD }} | |
| smb_password: ${{ secrets.CI_CONFIG_HC_IP50_SMB_PASSWORD }} | |
| oidc_client_secret: ${{ secrets.OIDC_CLIENT_SECRET }} | |
| oidc_users_0_password: ${{ secrets.OIDC_USERS_0_PASSWORD }} | |
| working_directory: ${{ env.WORKDIR }} | |
| - name: Debug test config | |
| shell: bash | |
| run: | | |
| pwd | |
| ls -al tests/integration/integration_config.yml | |
| head tests/integration/integration_config.yml | |
| - name: Run test | |
| shell: bash | |
| run: | | |
| [ "$X_GITHUB_SKIP_TEST" == "1" ] && exit 0 | |
| eval export SC_HOST=$(cat tests/integration/integration_config.yml | yq '.sc_host') | |
| eval export SC_USERNAME=$(cat tests/integration/integration_config.yml | yq '.sc_config."https://10.5.11.200".sc_username') | |
| eval export SC_PASSWORD=$(cat tests/integration/integration_config.yml | yq '.sc_config."https://10.5.11.200".sc_password') | |
| eval export SC_TIMEOUT=$(cat tests/integration/integration_config.yml | yq '.sc_timeout') | |
| ansible-playbook -i localhost, -e hypercore_example_check_mode=true examples/${{ matrix.example_name }} | |
| integ-run: | |
| needs: | |
| - integ-matrix | |
| if: "(!cancelled()) && (needs.integ-matrix.result=='success')" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| sc_host: | |
| - https://10.5.11.200 | |
| - https://10.5.11.201 | |
| - https://10.5.11.203 | |
| - https://10.5.11.205 | |
| secrets: inherit | |
| uses: ./.github/workflows/z_ansible-test.yml | |
| with: | |
| sc_host: ${{ matrix.sc_host }} | |
| test_names: ${{ needs.integ-matrix.outputs.matrix }} | |
| outer_job_name: integ-run | |
| integ-seq-run: | |
| needs: | |
| - integ-seq-matrix | |
| - integ-run | |
| if: "(!cancelled()) && (needs.integ-seq-matrix.result=='success')" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| sc_host: | |
| - https://10.5.11.200 | |
| - https://10.5.11.201 | |
| - https://10.5.11.203 | |
| - https://10.5.11.205 | |
| secrets: inherit | |
| uses: ./.github/workflows/z_ansible-test.yml | |
| with: | |
| sc_host: ${{ matrix.sc_host }} | |
| test_names: ${{ needs.integ-seq-matrix.outputs.matrix }} | |
| outer_job_name: integ-seq-run | |
| # On physical HyperCore | |
| integ-physical: | |
| secrets: inherit | |
| uses: ./.github/workflows/z_ansible-test.yml | |
| with: | |
| sc_host: https://10.5.11.50 | |
| test_names: >- | |
| [ | |
| "vm_replication", | |
| 'vm_replication_info", | |
| "vm_clone__replicated", | |
| "support_tunnel" | |
| ] | |
| outer_job_name: integ-physical | |
| replica_cleanup: | |
| needs: [] | |
| runs-on: [self-hosted2] | |
| container: quay.io/justinc1_github/scale_ci_integ:10 | |
| env: | |
| ANSIBLE_COLLECTIONS_PATH: $GITHUB_WORKSPACE/work-dir | |
| defaults: | |
| run: | |
| working-directory: ${{ env.WORKDIR }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| path: ${{ env.WORKDIR }} | |
| - run: pip install ansible-core~=2.16.0 | |
| # ${{ env.WORKDIR }} cannot be used | |
| - uses: ./work-dir/ansible_collections/scale_computing/hypercore/.github/actions/make-integ-config | |
| with: | |
| sc_host: https://10.5.11.50 | |
| sc_password_50: ${{ secrets.CI_CONFIG_HC_IP50_SC_PASSWORD }} | |
| smb_password: ${{ secrets.CI_CONFIG_HC_IP50_SMB_PASSWORD }} | |
| oidc_client_secret: ${{ secrets.OIDC_CLIENT_SECRET }} | |
| oidc_users_0_password: ${{ secrets.OIDC_USERS_0_PASSWORD }} | |
| working_directory: ${{ env.WORKDIR }} | |
| - run: ansible-galaxy collection install community.general | |
| - run: ansible-playbook tests/integration/cleanup/ci_replica_cleanup.yml | |
| smb_cleanup: | |
| needs: [] | |
| runs-on: [self-hosted2] | |
| container: quay.io/justinc1_github/scale_ci_integ:10 | |
| defaults: | |
| run: | |
| working-directory: ${{ env.WORKDIR }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| path: ${{ env.WORKDIR }} | |
| # ${{ env.WORKDIR }} cannot be used | |
| - uses: ./work-dir/ansible_collections/scale_computing/hypercore/.github/actions/make-integ-config | |
| with: | |
| sc_host: https://10.5.11.200 | |
| sc_password_50: ${{ secrets.CI_CONFIG_HC_IP50_SC_PASSWORD }} | |
| smb_password: ${{ secrets.CI_CONFIG_HC_IP50_SMB_PASSWORD }} | |
| oidc_client_secret: ${{ secrets.OIDC_CLIENT_SECRET }} | |
| oidc_users_0_password: ${{ secrets.OIDC_USERS_0_PASSWORD }} | |
| working_directory: ${{ env.WORKDIR }} | |
| - run: | | |
| cd tests/integration/cleanup && ./smb_cleanup.sh \ | |
| "$(cat ../integration_config.yml | yq '.smb_server' | sed -e 's/^"//' -e 's/"$//')" \ | |
| "$(cat ../integration_config.yml | yq '.smb_share' | sed -e 's/^"//' -e 's/"$//')" \ | |
| "$(cat ../integration_config.yml | yq '.smb_username' | sed -e 's/^"//' -e 's/"$//')" \ | |
| "$(cat ../integration_config.yml | yq '.smb_password' | sed -e 's/^"//' -e 's/"$//')" |