Skip to content

MINIFICPP-2717 Remove old docker testing framework #2110

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lordgamez wants to merge 1 commit into
base: MINIFICPP-2714
Choose a base branch
from
Open

MINIFICPP-2717 Remove old docker testing framework #2110

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 0 additions & 58 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -474,64 +474,6 @@ jobs:
with:
name: minifi_docker
path: build/minifi_docker.tar
legacy_docker_tests:
name: "Docker integration tests (x86_64)"
needs: docker_build
runs-on: ubuntu-24.04
timeout-minutes: 180
steps:
- id: checkout
uses: actions/checkout@v4
- id: run_cmake
name: Run CMake
run: |
mkdir build
cd build
cmake ${DOCKER_CMAKE_FLAGS} ..
- name: Download artifact
uses: actions/download-artifact@v4
with:
name: minifi_docker
path: build
- name: Load Docker image
run: |
docker load --input ./build/minifi_docker.tar
- id: install_deps
name: Install dependencies for Docker Verify
run: |
sudo apt update
sudo apt install -y python3-virtualenv
- id: free_disk_space
run: |
# We can gain additional disk space on the Ubuntu runners thanks to these suggestions:
# https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
# https://github.com/actions/runner-images/issues/2606#issuecomment-772683150
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
- id: test
name: Docker Verify
working-directory: ./build
run: make docker-verify
- name: Test Reporter
if: always()
uses: phoenix-actions/test-reporting@f957cd93fc2d848d556fa0d03c57bc79127b6b5e # v15
with:
name: Docker integration tests
path: build/behavex_output/behave/*.xml
reporter: java-junit
output-to: 'step-summary'
only-summary: 'true'
list-tests: 'failed'
list-suites: 'failed'
- name: Upload artifact
if: failure()
uses: actions/upload-artifact@v4
with:
name: behavex_output
path: build/behavex_output
modular_docker_tests:
name: "Modular Docker integration tests (x86_64)"
needs: docker_build
Expand Down
74 changes: 4 additions & 70 deletions .github/workflows/verify-package.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,77 +19,8 @@ env:
-DENABLE_ELASTICSEARCH=OFF -DENABLE_GRAFANA_LOKI=ON -DENABLE_COUCHBASE=ON -DDOCKER_BUILD_ONLY=ON

jobs:
docker-tests:
name: "${{ matrix.platform.name }} (${{ matrix.arch }})${{ inputs.enable_fips && ' (FIPS Mode)' || '' }}"
runs-on: ${{ matrix.arch == 'x86_64' && 'ubuntu-24.04' || 'ubuntu-24.04-arm' }}
timeout-minutes: 240
strategy:
fail-fast: false
matrix:
arch: [x86_64, aarch64]
platform: [
{ name: "Rocky Linux 9", id: "rocky9", build_cmd: "make rocky9_from_rocky_package", verify_cmd: "make docker-verify-rocky9" },
{ name: "Rocky Linux 10", id: "rocky10", build_cmd: "make rocky10_from_rocky_package", verify_cmd: "make docker-verify-rocky10" },
{ name: "Rocky Linux 9 from RPM", id: "rocky9-rpm", build_cmd: "make rocky9_from_rpm_package", verify_cmd: "make docker-verify-rocky9-rpm" },
{ name: "Rocky Linux 10 from RPM", id: "rocky10-rpm", build_cmd: "make rocky10_from_rpm_package", verify_cmd: "make docker-verify-rocky10-rpm" },
{ name: "Ubuntu 22.04", id: "ubuntu_jammy", build_cmd: "make jammy_from_rocky_package", verify_cmd: "make docker-verify-jammy" },
{ name: "Ubuntu 24.04", id: "ubuntu_noble", build_cmd: "make noble_from_rocky_package", verify_cmd: "make docker-verify-noble" },
{ name: "Debian 12", id: "debian_bookworm", build_cmd: "make bookworm_from_rocky_package", verify_cmd: "make docker-verify-bookworm" },
{ name: "Debian 13", id: "debian_trixie", build_cmd: "make trixie_from_rocky_package", verify_cmd: "make docker-verify-trixie" },
]
permissions:
contents: read
steps:
- id: checkout
uses: actions/checkout@v4

- uses: actions/download-artifact@v4
with:
run-id: ${{ inputs.artifacts_workflow_id }}
name: minifi-${{ matrix.arch }}-tar
path: build
github-token: ${{ github.token }}

- uses: actions/download-artifact@v4
with:
run-id: ${{ inputs.artifacts_workflow_id }}
name: minifi-${{ matrix.arch }}-rpm
path: build
github-token: ${{ github.token }}

- id: install_deps
run: |
sudo apt update
sudo apt install -y python3-virtualenv

- run: |
cd build && cmake ${DOCKER_CMAKE_FLAGS} ..
VERIFY_CMD="${{ matrix.platform.verify_cmd }}"
if [[ "${{ inputs.enable_fips }}" == "true" ]]; then
VERIFY_CMD="${VERIFY_CMD}-fips"
fi
${{ matrix.platform.build_cmd }} && $VERIFY_CMD

- name: Test Reporter
if: always()
uses: phoenix-actions/test-reporting@f957cd93fc2d848d556fa0d03c57bc79127b6b5e # v15
with:
name: "${{ matrix.platform.name }} (${{ matrix.arch }})${{ inputs.enable_fips && ' (FIPS Mode)' || '' }}"
path: build/behavex_output/behave/*.xml
reporter: java-junit
output-to: 'step-summary'
list-tests: 'failed'
list-suites: 'failed'

- name: Upload artifact
if: failure()
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.platform.id }}_${{ matrix.arch }}_behavex_output${{ inputs.enable_fips && '_fips' || '' }}
path: build/behavex_output

docker-test-modular:
name: "${{ matrix.platform.name }} (${{ matrix.arch }}) Modular"
name: "${{ matrix.platform.name }} (${{ matrix.arch }}) Modular${{ inputs.enable_fips && ' (FIPS Mode)' || '' }}"
runs-on: ${{ matrix.arch == 'x86_64' && 'ubuntu-24.04' || 'ubuntu-24.04-arm' }}
timeout-minutes: 240
strategy:
Expand Down Expand Up @@ -134,6 +65,9 @@ jobs:
- run: |
cd build && cmake ${DOCKER_CMAKE_FLAGS} ..
VERIFY_CMD="${{ matrix.platform.verify_cmd }}"
if [[ "${{ inputs.enable_fips }}" == "true" ]]; then
VERIFY_CMD="${VERIFY_CMD}-fips"
fi
${{ matrix.platform.build_cmd }} && $VERIFY_CMD

- name: Test Reporter
Expand Down
6 changes: 0 additions & 6 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -405,12 +405,6 @@ $ make docker-minimal
#### Executing integration tests with your docker image
You can execute system integration tests using a minifi docker image.</br>
Currently, there are two types of docker integration tests:
##### Monolith legacy tests (features locates in docker/test/integration/features)
(we are in the process of migrating these)
```
~/Development/code/apache/nifi-minifi-cpp/build
$ make docker-verify
```
##### Modular tests located near the tested extension (e.g. extensions/aws/tests/features)
```
~/Development/code/apache/nifi-minifi-cpp/build
Expand Down
33 changes: 14 additions & 19 deletions cmake/DockerConfig.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
set(PROJECT_VERSION_STR ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH})
include(ProcessorCount)
ProcessorCount(PROCESSOR_COUNT)
set(DOCKER_VERIFY_THREADS "${PROCESSOR_COUNT}" CACHE STRING "Number of threads that docker-verify can utilize")
set(DOCKER_VERIFY_THREADS "${PROCESSOR_COUNT}" CACHE STRING "Number of threads that docker-verify-modular can utilize")

# Create a custom build target called "docker" that will invoke DockerBuild.sh and create the NiFi-MiNiFi-CPP Docker image
add_custom_target(
Expand Down Expand Up @@ -141,26 +141,21 @@ add_custom_target(
-c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)

if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features)
set(ENABLED_TAGS "CORE")
foreach(MINIFI_OPTION ${MINIFI_OPTIONS})
string(FIND ${MINIFI_OPTION} "ENABLE" my_index)
if(my_index EQUAL -1)
continue()
elseif(${${MINIFI_OPTION}})
set(ENABLED_TAGS "${ENABLED_TAGS},${MINIFI_OPTION}")
endif()
endforeach()
set(ENABLED_TAGS "CORE")
foreach(MINIFI_OPTION ${MINIFI_OPTIONS})
string(FIND ${MINIFI_OPTION} "ENABLE" my_index)
if(my_index EQUAL -1)
continue()
elseif(${${MINIFI_OPTION}})
set(ENABLED_TAGS "${ENABLED_TAGS},${MINIFI_OPTION}")
endif()
endforeach()

set(DISABLED_TAGS "SKIP_CI")
set(DISABLED_TAGS "SKIP_CI")

add_custom_target(
docker-verify
COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh ${PROJECT_VERSION_STR} ${ENABLED_TAGS} --tags_to_exclude=${DISABLED_TAGS} --parallel_processes=${DOCKER_VERIFY_THREADS})
add_custom_target(
docker-verify-modular
COMMAND ${CMAKE_SOURCE_DIR}/docker/RunBehaveTests.sh ${PROJECT_VERSION_STR} ${ENABLED_TAGS} --tags_to_exclude=${DISABLED_TAGS} --parallel_processes=${DOCKER_VERIFY_THREADS})
endif()
add_custom_target(
docker-verify-modular
COMMAND ${CMAKE_SOURCE_DIR}/docker/RunBehaveTests.sh ${PROJECT_VERSION_STR} ${ENABLED_TAGS} --tags_to_exclude=${DISABLED_TAGS} --parallel_processes=${DOCKER_VERIFY_THREADS})

function(CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE BASE_IMAGE TAG_PREFIX INSTALL_PACKAGE_CMD)
add_custom_target(
Expand Down
31 changes: 13 additions & 18 deletions cmake/VerifyPackageWithDocker.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,12 @@ function(ADD_PACKAGE_VERIFY TAG_PREFIX)

set(DISABLED_TAGS "SKIP_CI")

add_custom_target(
docker-verify-${TAG_PREFIX}
COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --image-tag-prefix ${TAG_PREFIX} ${MINIFI_VERSION_STR} ${ENABLED_TAGS} --tags_to_exclude=${DISABLED_TAGS} --parallel_processes=${DOCKER_VERIFY_THREADS})
add_custom_target(
docker-verify-${TAG_PREFIX}-modular
COMMAND ${CMAKE_SOURCE_DIR}/docker/RunBehaveTests.sh --image-tag-prefix ${TAG_PREFIX} ${MINIFI_VERSION_STR} ${ENABLED_TAGS} --tags_to_exclude=${DISABLED_TAGS} --parallel_processes=${DOCKER_VERIFY_THREADS})
add_custom_target(
docker-verify-${TAG_PREFIX}-fips
COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --image-tag-prefix ${TAG_PREFIX} ${MINIFI_VERSION_STR} ${ENABLED_TAGS} --tags_to_exclude=${DISABLED_TAGS} --parallel_processes=${DOCKER_VERIFY_THREADS} --fips)
docker-verify-${TAG_PREFIX}-modular-fips
COMMAND ${CMAKE_SOURCE_DIR}/docker/RunBehaveTests.sh --image-tag-prefix ${TAG_PREFIX} ${MINIFI_VERSION_STR} ${ENABLED_TAGS} --tags_to_exclude=${DISABLED_TAGS} --parallel_processes=${DOCKER_VERIFY_THREADS} --fips)
endfunction()


Expand All @@ -54,16 +51,14 @@ CREATE_DOCKER_TARGET_FROM_RPM_PACKAGE(rockylinux:8 rocky8 "dnf install -y wget p
CREATE_DOCKER_TARGET_FROM_RPM_PACKAGE(rockylinux:9 rocky9 "dnf install -y wget python3-devel python3-pip gcc gcc-c++")
CREATE_DOCKER_TARGET_FROM_RPM_PACKAGE(rockylinux/rockylinux:10 rocky10 "dnf install -y wget python3-devel python3-pip gcc gcc-c++")

if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features)
ADD_PACKAGE_VERIFY(rocky8)
ADD_PACKAGE_VERIFY(rocky9)
ADD_PACKAGE_VERIFY(rocky10)
ADD_PACKAGE_VERIFY(jammy)
ADD_PACKAGE_VERIFY(noble)
ADD_PACKAGE_VERIFY(bookworm)
ADD_PACKAGE_VERIFY(bullseye)
ADD_PACKAGE_VERIFY(trixie)
ADD_PACKAGE_VERIFY(rocky8-rpm)
ADD_PACKAGE_VERIFY(rocky9-rpm)
ADD_PACKAGE_VERIFY(rocky10-rpm)
endif()
ADD_PACKAGE_VERIFY(rocky8)
ADD_PACKAGE_VERIFY(rocky9)
ADD_PACKAGE_VERIFY(rocky10)
ADD_PACKAGE_VERIFY(jammy)
ADD_PACKAGE_VERIFY(noble)
ADD_PACKAGE_VERIFY(bookworm)
ADD_PACKAGE_VERIFY(bullseye)
ADD_PACKAGE_VERIFY(trixie)
ADD_PACKAGE_VERIFY(rocky8-rpm)
ADD_PACKAGE_VERIFY(rocky9-rpm)
ADD_PACKAGE_VERIFY(rocky10-rpm)
Loading
Loading