integ-test.yml

name: integ-test
on:
  schedule:
    - cron: "0 4 * * *"
  workflow_dispatch:
    inputs:
      integ_tests_include:
        type: string
        description: |-
          List integration tests to include.
          Use "*" to run all tests.
          Use regex like 'node|^git_issue' to run only a subset.
        default: "*"
      # dns_config|time_server - NTP cannot be reconfigured if DNS is invalid
      # git_issues - slow, do not run on each push. TODO - run them only once a day
      # oidc_config - during reconfiguration API returns 500/502 errors for other requests
      # utils_login - it uses OIDC user to login
      # smtp - email_alert test requires a configured SMTP
      # role_cluster_config - role cluster_config reconfigures DNS, SMTP, OIDC. And it is slow.
      # version_update_single_node - role would change version, VSNS system cannot be updated.
      # vm_replication vm_clone__replicated - VSNS were not configured with replication
      integ_tests_exclude:
        type: string
        description: |-
          List integration tests to exclude.
          Use "*" to exclude all tests.
          Use regex like 'node|^git_issue|^dns_config$' to exclude only a subset.
        default: "^dns_config$|^cluster_shutdown$|^version_update$|^oidc_config$|^smtp$|^role_cluster_config$|^role_version_update_single_node$|^version_update__shutdown_restart_vms$|^utils_login$|^certificate$|^vm_replication$|^vm_replication_info$|^vm_clone__replicated$|^support_tunnel$"
      integ_seq_tests_include:
        type: string
        description: |-
          List seq integration tests to include.
        # We want to run sequentially all tests that cannot run in parallel (e.g. all that listed in integ_tests_exclude).
        # Some tests can be tested on VSNS - those are run on VSNS.
        # Some cannot, those are run on test cluster 10.5.11.50.
        # Some cannot be tested at all - shutdown and version update.
        default: "^dns_config$|^oidc_config$|^smtp$|^role_cluster_config$|^version_update__shutdown_restart_vms$|^utils_login$|^certificate$"
      integ_seq_tests_exclude:
        type: string
        description: |-
          List seq integration tests to exclude.
        default: ""
      examples_tests_include:
        type: string
        description: |-
          List examples to include into testing.
          Use "*" to test all examples.
        default: "iso_info"
      examples_tests_exclude:
        type: string
        description: |-
          List examples to exclude from testing.
        default: "^version_update_single_node.yml$|^vm_os_upgrade$|^vm_snapshot_attach_disk.yml$|^url2template$"
env:
  INTEG_TESTS_INCLUDE_SCHEDULE: "*"
  INTEG_TESTS_EXCLUDE_SCHEDULE: "^dns_config$|^cluster_shutdown$|^version_update$|^oidc_config$|^smtp$|^role_cluster_config$|^role_version_update_single_node$|^version_update__shutdown_restart_vms$|^utils_login$|^certificate$|^vm_replication$|^vm_replication_info$|^vm_clone__replicated$|^support_tunnel$"
  INTEG_SEQ_TESTS_INCLUDE_SCHEDULE: "^dns_config$|^oidc_config$|^smtp$|^role_cluster_config$|^version_update__shutdown_restart_vms$|^utils_login$|^certificate$"
  INTEG_SEQ_TESTS_EXCLUDE_SCHEDULE: ""
  EXAMPLES_TESTS_INCLUDE_SCHEDULE: "*"
  EXAMPLES_TESTS_EXCLUDE_SCHEDULE: "^version_update_single_node.yml$|^vm_os_upgrade$|^vm_snapshot_attach_disk.yml$|^url2template$"
  # ansible-test needs special directory structure.
  # WORKDIR is a subdir of GITHUB_WORKSPACE
  WORKDIR: work-dir/ansible_collections/scale_computing/hypercore
  LANG: C.UTF-8
# Run only one workflow for specific branch.
concurrency:
  group: ${{ github.ref_name }}
  cancel-in-progress: true

jobs:
  # This should be run at least once before first integration tests run.
  # As long as prepared HC3 host remains prepared, we do not need (except on very first run)
  # to delay integ-test until integration-prepare-env finishes.
  integration-prepare-env:
    runs-on: [self-hosted2]
    container: quay.io/justinc1_github/scale_ci_integ:10
    env:
      ANSIBLE_COLLECTIONS_PATH: $GITHUB_WORKSPACE/work-dir
    defaults:
      run:
        working-directory: ${{ env.WORKDIR }}
    strategy:
      fail-fast: false
      matrix:
        sc_host:
          - https://10.5.11.200  # HC 9.1.x
          - https://10.5.11.201  # HC 9.2.x
          - https://10.5.11.203  # HC 9.3.x
          - https://10.5.11.205  # HC 9.4.x
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          path: ${{ env.WORKDIR }}
      - run: ansible-galaxy collection install community.crypto community.general
      - run: ansible-galaxy collection list
      - uses: ./work-dir/ansible_collections/scale_computing/hypercore/.github/actions/make-integ-config
        with:
          sc_host: ${{ matrix.sc_host }}
          sc_password_50: ${{ secrets.CI_CONFIG_HC_IP50_SC_PASSWORD }}
          smb_password: ${{ secrets.CI_CONFIG_HC_IP50_SMB_PASSWORD }}
          oidc_client_secret: ${{ secrets.OIDC_CLIENT_SECRET }}
          oidc_users_0_password: ${{ secrets.OIDC_USERS_0_PASSWORD }}
          working_directory: ${{ env.WORKDIR }}
      - run: ansible-playbook tests/integration/prepare/prepare_iso.yml
      - run: ansible-playbook tests/integration/prepare/prepare_vm.yml
      - run: ansible-playbook tests/integration/prepare/prepare_examples.yml
    if: "${{ github.event.inputs.integ_tests_include || github.event.schedule }}"

  integ-matrix:
    runs-on: [ubuntu-latest]
    container: quay.io/justinc1_github/scale_ci_integ:10
    defaults:
      run:
        working-directory: ${{ env.WORKDIR }}
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          path: ${{ env.WORKDIR }}
      # We want to run all integ tests periodically - "integ_tests_include || '*'", the '*' is used.
      # When running with workflow-dispatch, user is required to put some non-empty string into integ_tests_include.
      - run: echo 'INTEG_TESTS_INCLUDE=${{ github.event.inputs.integ_tests_include || env.INTEG_TESTS_INCLUDE_SCHEDULE }}' >> $GITHUB_ENV
      - run: echo 'INTEG_TESTS_EXCLUDE=${{ github.event.inputs.integ_tests_exclude || env.INTEG_TESTS_EXCLUDE_SCHEDULE }}' >> $GITHUB_ENV
      - id: set-matrix
        shell: bash
        run: |-
          echo "matrix=$(
            ls -r tests/integration/targets |
            grep -v -E "${{ env.INTEG_TESTS_EXCLUDE || '^dummy-no-such-test$' }}" |
            grep -E "${{ env.INTEG_TESTS_INCLUDE }}" |
            jq -R -s -c 'split("\n")[:-1]'
          )" >> $GITHUB_OUTPUT
    if: "${{ github.event.inputs.integ_tests_include || github.event.schedule }}"


  integ-seq-matrix:
    needs:
      - integ-matrix
    runs-on: [ubuntu-latest]
    container: quay.io/justinc1_github/scale_ci_integ:10
    defaults:
      run:
        working-directory: ${{ env.WORKDIR }}
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          path: ${{ env.WORKDIR }}
      - run: echo 'INTEG_SEQ_TESTS_INCLUDE=${{ github.event.inputs.integ_seq_tests_include || env.INTEG_SEQ_TESTS_INCLUDE_SCHEDULE }}' >> $GITHUB_ENV
      - run: echo 'INTEG_SEQ_TESTS_EXCLUDE=${{ github.event.inputs.integ_seq_tests_exclude || env.INTEG_SEQ_TESTS_EXCLUDE_SCHEDULE }}' >> $GITHUB_ENV
      - id: set-matrix
        shell: bash
        run: |-
          echo "matrix=$(
            ls -r tests/integration/targets |
            grep -v -E "${{ env.INTEG_SEQ_TESTS_EXCLUDE || '^dummy-no-such-test$' }}" |
            grep -E "${{ env.INTEG_SEQ_TESTS_INCLUDE }}" |
            jq -R -s -c 'split("\n")[:-1]'
          )" >> $GITHUB_OUTPUT
    if: "${{ github.event.inputs.integ_seq_tests_include || github.event.schedule }}"

  examples-matrix:
    needs:
      - integ-seq-matrix
    runs-on: [ ubuntu-latest ]
    container: quay.io/justinc1_github/scale_ci_integ:10
    defaults:
      run:
        working-directory: ${{ env.WORKDIR }}
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          path: ${{ env.WORKDIR }}
      - run: echo 'EXAMPLES_TESTS_INCLUDE=${{ github.event.inputs.examples_tests_include || env.EXAMPLES_TESTS_INCLUDE_SCHEDULE }}' >> $GITHUB_ENV
      - run: echo 'EXAMPLES_TESTS_EXCLUDE=${{ github.event.inputs.examples_tests_exclude || env.EXAMPLES_TESTS_EXCLUDE_SCHEDULE }}' >> $GITHUB_ENV
      # A few files are not an example playbook:
      #   hypercore_inventory.yml - inventory example
      #   cloud-init-user-data-example.yml - cloud-init user-data example
      - id: set-matrix
        shell: bash
        run: |-
          echo "matrix=$(
            ls -r examples |
            grep -v -E "^README.md$|^hypercore_inventory.yml$|^cloud-init-user-data-example.yml$" |
            grep -v -E "${{ env.EXAMPLES_TESTS_EXCLUDE || '^dummy-no-such-example$' }}" |
            grep -E "${{ env.EXAMPLES_TESTS_INCLUDE }}" |
            jq -R -s -c 'split("\n")[:-1]'
          )" >> $GITHUB_OUTPUT
    if: "${{ github.event.inputs || github.event.schedule }}"

  examples-run:
    needs:
      - examples-matrix
      # run examples after integ - they will both try to modify say syslog_server etc
      - integ-seq-run
    if: "(!cancelled())  && (needs.examples-matrix.result=='success')"
    runs-on: [ self-hosted2 ]
    container: quay.io/justinc1_github/scale_ci_integ:10
    env:
      ANSIBLE_COLLECTIONS_PATH: $GITHUB_WORKSPACE/work-dir
      DEBIAN_FRONTEND: noninteractive
    defaults:
      run:
        working-directory: ${{ env.WORKDIR }}
    strategy:
      fail-fast: false
      matrix:
        ansible: [ 2.16.0 ]
        example_name: ${{ fromJson(needs.examples-matrix.outputs.matrix) }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          path: ${{ env.WORKDIR }}
      - name: SKIP test if it succeeded before
        shell: bash
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          X_GITHUB_JOB_NAME: "examples-run (${{ matrix.ansible }}, ${{ matrix.example_name }})"
        run: |
          if ! .github/job_retry_needed.py
          then
            echo SKIP test, job retry not needed
            echo 'X_GITHUB_SKIP_TEST=1' >> $GITHUB_ENV
          else
            echo 'X_GITHUB_SKIP_TEST=0' >> $GITHUB_ENV
          fi
      - name: Install prerequisites
        shell: bash
        run: |
          [ "$X_GITHUB_SKIP_TEST" == "1" ] && exit 0
          pip install ansible-core~=${{ matrix.ansible }}
          ansible-galaxy collection install community.crypto community.general
          ansible-galaxy collection list
      # ${{ env.WORKDIR }} cannot be used in "uses:"
      - uses: ./work-dir/ansible_collections/scale_computing/hypercore/.github/actions/make-integ-config
        with:
          sc_host: https://10.5.11.205
          sc_password_50: ${{ secrets.CI_CONFIG_HC_IP50_SC_PASSWORD }}
          smb_password: ${{ secrets.CI_CONFIG_HC_IP50_SMB_PASSWORD }}
          oidc_client_secret: ${{ secrets.OIDC_CLIENT_SECRET }}
          oidc_users_0_password: ${{ secrets.OIDC_USERS_0_PASSWORD }}
          working_directory: ${{ env.WORKDIR }}
      - name: Debug test config
        shell: bash
        run: |
          pwd
          ls -al tests/integration/integration_config.yml
          head tests/integration/integration_config.yml
      - name: Run test
        shell: bash
        run: |
          [ "$X_GITHUB_SKIP_TEST" == "1" ] && exit 0
          eval export SC_HOST=$(cat tests/integration/integration_config.yml | yq '.sc_host')
          eval export SC_USERNAME=$(cat tests/integration/integration_config.yml | yq '.sc_config."https://10.5.11.200".sc_username')
          eval export SC_PASSWORD=$(cat tests/integration/integration_config.yml | yq '.sc_config."https://10.5.11.200".sc_password')
          eval export SC_TIMEOUT=$(cat tests/integration/integration_config.yml | yq '.sc_timeout')
          ansible-playbook -i localhost, -e hypercore_example_check_mode=true examples/${{ matrix.example_name }}

  integ-run:
    needs:
      - integ-matrix
    if: "(!cancelled()) && (needs.integ-matrix.result=='success')"
    strategy:
      fail-fast: false
      matrix:
        sc_host:
          - https://10.5.11.200
          - https://10.5.11.201
          - https://10.5.11.203
          - https://10.5.11.205
    secrets: inherit
    uses: ./.github/workflows/z_ansible-test.yml
    with:
      sc_host: ${{ matrix.sc_host }}
      test_names: ${{ needs.integ-matrix.outputs.matrix }}
      outer_job_name: integ-run

  integ-seq-run:
    needs:
      - integ-seq-matrix
      - integ-run
    if: "(!cancelled()) && (needs.integ-seq-matrix.result=='success')"
    strategy:
      fail-fast: false
      matrix:
        sc_host:
          - https://10.5.11.200
          - https://10.5.11.201
          - https://10.5.11.203
          - https://10.5.11.205
    secrets: inherit
    uses: ./.github/workflows/z_ansible-test.yml
    with:
      sc_host: ${{ matrix.sc_host }}
      test_names: ${{ needs.integ-seq-matrix.outputs.matrix }}
      outer_job_name: integ-seq-run

  # On physical HyperCore
  integ-physical:
    secrets: inherit
    uses: ./.github/workflows/z_ansible-test.yml
    with:
      sc_host: https://10.5.11.50
      test_names: >-
        [
          "vm_replication",
          'vm_replication_info",
          "vm_clone__replicated",
          "support_tunnel"
        ]
      outer_job_name: integ-physical

  replica_cleanup:
    needs: []
    runs-on: [self-hosted2]
    container: quay.io/justinc1_github/scale_ci_integ:10
    env:
      ANSIBLE_COLLECTIONS_PATH: $GITHUB_WORKSPACE/work-dir
    defaults:
      run:
        working-directory: ${{ env.WORKDIR }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          path: ${{ env.WORKDIR }}
      - run: pip install ansible-core~=2.16.0
      # ${{ env.WORKDIR }} cannot be used
      - uses: ./work-dir/ansible_collections/scale_computing/hypercore/.github/actions/make-integ-config
        with:
          sc_host: https://10.5.11.50
          sc_password_50: ${{ secrets.CI_CONFIG_HC_IP50_SC_PASSWORD }}
          smb_password: ${{ secrets.CI_CONFIG_HC_IP50_SMB_PASSWORD }}
          oidc_client_secret: ${{ secrets.OIDC_CLIENT_SECRET }}
          oidc_users_0_password: ${{ secrets.OIDC_USERS_0_PASSWORD }}
          working_directory: ${{ env.WORKDIR }}
      - run: ansible-galaxy collection install community.general
      - run: ansible-playbook tests/integration/cleanup/ci_replica_cleanup.yml

  smb_cleanup:
    needs: []
    runs-on: [self-hosted2]
    container: quay.io/justinc1_github/scale_ci_integ:10
    defaults:
      run:
        working-directory: ${{ env.WORKDIR }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          path: ${{ env.WORKDIR }}
      # ${{ env.WORKDIR }} cannot be used
      - uses: ./work-dir/ansible_collections/scale_computing/hypercore/.github/actions/make-integ-config
        with:
          sc_host: https://10.5.11.200
          sc_password_50: ${{ secrets.CI_CONFIG_HC_IP50_SC_PASSWORD }}
          smb_password: ${{ secrets.CI_CONFIG_HC_IP50_SMB_PASSWORD }}
          oidc_client_secret: ${{ secrets.OIDC_CLIENT_SECRET }}
          oidc_users_0_password: ${{ secrets.OIDC_USERS_0_PASSWORD }}
          working_directory: ${{ env.WORKDIR }}
      - run: |
          cd tests/integration/cleanup && ./smb_cleanup.sh \
            "$(cat ../integration_config.yml | yq '.smb_server' | sed -e 's/^"//' -e 's/"$//')" \
            "$(cat ../integration_config.yml | yq '.smb_share' | sed -e 's/^"//' -e 's/"$//')" \
            "$(cat ../integration_config.yml | yq '.smb_username' | sed -e 's/^"//' -e 's/"$//')" \
            "$(cat ../integration_config.yml | yq '.smb_password' | sed -e 's/^"//' -e 's/"$//')"