From 0a99f3c515aedf913d78d2409bd46a4f1f30e8c2 Mon Sep 17 00:00:00 2001 From: Terrell Russell Date: Fri, 14 Nov 2025 19:44:16 -0500 Subject: [PATCH 1/6] [#697] Github workflows for running PRC tests Co-authored-by: d-w-moore Includes a docker compose configuration in which the PRC test suite can be run in a way representative of typical operational use, and the version of iRODS server and python interpreter that we install are also easily reconfigurable. Additionally, this affords us the opportunity to run the test suite on a client node all its own, the iRODS server being reachable only via the network. --- .github/workflows/main.yml | 2 + .../run-test-suite-multiple-node.yml | 39 +++++++++++ test_harness/multiple_node/README.md | 56 +++++++++++++++ .../harness-docker-compose-irods-4.yml | 37 ++++++++++ .../harness-docker-compose-irods-5.yml | 37 ++++++++++ .../multiple_node/harness-docker-compose.yml | 37 ++++++++++ .../multiple_node/irods_catalog_4/Dockerfile | 3 + .../irods_catalog_4/init-user-db.sh | 11 +++ .../multiple_node/irods_catalog_5/Dockerfile | 3 + .../irods_catalog_5/init-user-db.sh | 12 ++++ .../irods_catalog_provider_4/Dockerfile | 68 ++++++++++++++++++ .../irods_catalog_provider_4/entrypoint.sh | 49 +++++++++++++ .../setup-4.3.1.input | 28 ++++++++ .../setup-4.3.2.input | 28 ++++++++ .../setup-4.3.3.input | 28 ++++++++ .../setup-4.3.4.input | 28 ++++++++ .../irods_catalog_provider_5/Dockerfile | 69 +++++++++++++++++++ .../irods_catalog_provider_5/entrypoint.sh | 50 ++++++++++++++ .../setup-5.0.0.input | 26 +++++++ .../setup-5.0.1.input | 26 +++++++ .../setup-5.0.2.input | 26 +++++++ .../multiple_node/print_repo_root_location | 5 ++ .../multiple_node/python_client/Dockerfile | 2 + test_harness/multiple_node/run_tests.sh | 44 ++++++++++++ .../multiple_node/start_containers.sh | 53 ++++++++++++++ test_harness/multiple_node/stop_containers.sh | 24 +++++++ 26 files changed, 791 insertions(+) create mode 100644 .github/workflows/run-test-suite-multiple-node.yml create mode 100644 test_harness/multiple_node/README.md create mode 100644 test_harness/multiple_node/harness-docker-compose-irods-4.yml create mode 100644 test_harness/multiple_node/harness-docker-compose-irods-5.yml create mode 100644 test_harness/multiple_node/harness-docker-compose.yml create mode 100644 test_harness/multiple_node/irods_catalog_4/Dockerfile create mode 100644 test_harness/multiple_node/irods_catalog_4/init-user-db.sh create mode 100644 test_harness/multiple_node/irods_catalog_5/Dockerfile create mode 100644 test_harness/multiple_node/irods_catalog_5/init-user-db.sh create mode 100644 test_harness/multiple_node/irods_catalog_provider_4/Dockerfile create mode 100644 test_harness/multiple_node/irods_catalog_provider_4/entrypoint.sh create mode 100644 test_harness/multiple_node/irods_catalog_provider_4/setup-4.3.1.input create mode 100644 test_harness/multiple_node/irods_catalog_provider_4/setup-4.3.2.input create mode 100644 test_harness/multiple_node/irods_catalog_provider_4/setup-4.3.3.input create mode 100644 test_harness/multiple_node/irods_catalog_provider_4/setup-4.3.4.input create mode 100644 test_harness/multiple_node/irods_catalog_provider_5/Dockerfile create mode 100644 test_harness/multiple_node/irods_catalog_provider_5/entrypoint.sh create mode 100644 test_harness/multiple_node/irods_catalog_provider_5/setup-5.0.0.input create mode 100644 test_harness/multiple_node/irods_catalog_provider_5/setup-5.0.1.input create mode 100644 test_harness/multiple_node/irods_catalog_provider_5/setup-5.0.2.input create mode 100755 test_harness/multiple_node/print_repo_root_location create mode 100644 test_harness/multiple_node/python_client/Dockerfile create mode 100755 test_harness/multiple_node/run_tests.sh create mode 100755 test_harness/multiple_node/start_containers.sh create mode 100755 test_harness/multiple_node/stop_containers.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 40ad105d3..1c9fcf912 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,6 +2,8 @@ # https://github.com/actions/starter-workflows/blob/master/ci/python-package.yml # (C) Github, MIT License +# Static type checking tests using `mypy`. + name: "Python type checking" on: [push, pull_request] diff --git a/.github/workflows/run-test-suite-multiple-node.yml b/.github/workflows/run-test-suite-multiple-node.yml new file mode 100644 index 000000000..3a2d06fa9 --- /dev/null +++ b/.github/workflows/run-test-suite-multiple-node.yml @@ -0,0 +1,39 @@ +# Create a networked set of containers (via a Docker compose project) on which to run the client test suite. +# (For further information, see the README in `docker-testing`.) + +name: run-test-suite-multiple-node + +on: [push, pull_request] + +jobs: + tests: + name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./test_harness/multiple_node + strategy: + matrix: + python: ['3.9','3.13'] + irods_server: ['4.3.4','5.0.2'] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Start containers + run: ./start_containers.sh "${{ matrix.irods_server }}" "${{ matrix.python }}" + + - name: run test + run: | + while :; do + client_container=$(docker ps --format "{{.Names}}"|grep python.client) + [ -n "$client_container" ] && break + sleep 1 + done + echo "client_container = [$client_container]" + docker exec "${client_container}" /repo_root/test_harness/multiple_node/run_tests.sh + + - name: Stop containers + if: always() + run: ./stop_containers.sh "${{ matrix.irods_server }}" diff --git a/test_harness/multiple_node/README.md b/test_harness/multiple_node/README.md new file mode 100644 index 000000000..eee782eeb --- /dev/null +++ b/test_harness/multiple_node/README.md @@ -0,0 +1,56 @@ +# A Topological Setup for Testing the Python Client + +The `docker-testing` directory contains the necessary files for building and +running the client test suite from the perspective of a specific client node +(separate from the iRODS server node it targets for the tests). + +The client, provider, and database are currently run on distinct nodes within +a network topology set up by `docker compose` for the tests. + +We currently allow a choice of Python interpreter and iRODS server to be installed +on the client and provider nodes, respectively. + +The choice of versions are dictated when running the test: + +| Environment Variable | Min supported version | Max supported version | +| -------------------- | --------------------- | --------------------- | +| IRODS_PACKAGE_VERSION | 4.3.1 | 5.0.2 | +| PYTHON_VERSION | 3.9 | 3.13 | + +Currently the database server is fixed as Postgres. + +## Details of usage + +The file `.github/workflows/run-the-tests.yml` describes a Github action which +will be started to run the client test suite in response to creating a pull +request, or pushing new commits to the GitHub branch, containing it. + +The command-line recipe outlined in the file also supports running the +test suite on any workstation with docker compose installed. + +A summary of how to run the tests "at the bench" follows: + + 1. Change the working directory to the root directory of the repository, e.g.: + ``` + cd /path/to/python-irodsclient + ``` + + 2. Run: + ``` + ./docker-testing/start_containers.sh 4.3.4 3.11 + ``` + This builds and runs the docker images for the project, with "4.3.4" being the iRODS + version installed on the provider and "3.11" being the version of python installed on the client side. + + 3. Run: + ``` + docker exec /repo_root/docker-testing/run_tests.sh + ``` + (Note: `/repo_root` is an actual literal path, internal to the container.) + You'll see the test output displayed on the console. At completion, xmlrunner outputs are in + `/tmp/python-irodsclient/test-reports`. + + 4. Tail docker logs to see the iRODS server log. + ``` + docker logs -f + ``` diff --git a/test_harness/multiple_node/harness-docker-compose-irods-4.yml b/test_harness/multiple_node/harness-docker-compose-irods-4.yml new file mode 100644 index 000000000..4bba56ed5 --- /dev/null +++ b/test_harness/multiple_node/harness-docker-compose-irods-4.yml @@ -0,0 +1,37 @@ +services: + irods-catalog: + build: + context: irods_catalog_${irods_major} + environment: + - POSTGRES_PASSWORD=testpassword + + python-client: + build: + context: python_client + args: + python_version: ${python_version} + command: + tail -f /dev/null + volumes: + - ${repo_external}:/repo_root:ro + - /tmp/irods-client-share.py-${python_version}:/irods_shared + depends_on: + irods-catalog-provider: + condition: service_healthy + + irods-catalog-provider: + volumes: + - /tmp/irods-client-share.py-${python_version}:/irods_shared + build: + context: irods_catalog_provider_${irods_major} + args: + irods_version: ${irods_version} + shm_size: 500mb + healthcheck: + test: ["CMD", "su", "-", "irods", "-c", "ils || exit 1"] + interval: 10s + timeout: 10s + retries: 3 + depends_on: + - irods-catalog + diff --git a/test_harness/multiple_node/harness-docker-compose-irods-5.yml b/test_harness/multiple_node/harness-docker-compose-irods-5.yml new file mode 100644 index 000000000..4bba56ed5 --- /dev/null +++ b/test_harness/multiple_node/harness-docker-compose-irods-5.yml @@ -0,0 +1,37 @@ +services: + irods-catalog: + build: + context: irods_catalog_${irods_major} + environment: + - POSTGRES_PASSWORD=testpassword + + python-client: + build: + context: python_client + args: + python_version: ${python_version} + command: + tail -f /dev/null + volumes: + - ${repo_external}:/repo_root:ro + - /tmp/irods-client-share.py-${python_version}:/irods_shared + depends_on: + irods-catalog-provider: + condition: service_healthy + + irods-catalog-provider: + volumes: + - /tmp/irods-client-share.py-${python_version}:/irods_shared + build: + context: irods_catalog_provider_${irods_major} + args: + irods_version: ${irods_version} + shm_size: 500mb + healthcheck: + test: ["CMD", "su", "-", "irods", "-c", "ils || exit 1"] + interval: 10s + timeout: 10s + retries: 3 + depends_on: + - irods-catalog + diff --git a/test_harness/multiple_node/harness-docker-compose.yml b/test_harness/multiple_node/harness-docker-compose.yml new file mode 100644 index 000000000..4bba56ed5 --- /dev/null +++ b/test_harness/multiple_node/harness-docker-compose.yml @@ -0,0 +1,37 @@ +services: + irods-catalog: + build: + context: irods_catalog_${irods_major} + environment: + - POSTGRES_PASSWORD=testpassword + + python-client: + build: + context: python_client + args: + python_version: ${python_version} + command: + tail -f /dev/null + volumes: + - ${repo_external}:/repo_root:ro + - /tmp/irods-client-share.py-${python_version}:/irods_shared + depends_on: + irods-catalog-provider: + condition: service_healthy + + irods-catalog-provider: + volumes: + - /tmp/irods-client-share.py-${python_version}:/irods_shared + build: + context: irods_catalog_provider_${irods_major} + args: + irods_version: ${irods_version} + shm_size: 500mb + healthcheck: + test: ["CMD", "su", "-", "irods", "-c", "ils || exit 1"] + interval: 10s + timeout: 10s + retries: 3 + depends_on: + - irods-catalog + diff --git a/test_harness/multiple_node/irods_catalog_4/Dockerfile b/test_harness/multiple_node/irods_catalog_4/Dockerfile new file mode 100644 index 000000000..f02c4520c --- /dev/null +++ b/test_harness/multiple_node/irods_catalog_4/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:12 + +COPY init-user-db.sh /docker-entrypoint-initdb.d/init-user-db.sh diff --git a/test_harness/multiple_node/irods_catalog_4/init-user-db.sh b/test_harness/multiple_node/irods_catalog_4/init-user-db.sh new file mode 100644 index 000000000..5ff6b0375 --- /dev/null +++ b/test_harness/multiple_node/irods_catalog_4/init-user-db.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Adapted from "Initialization script" in documentation for official Postgres dockerhub: +# https://hub.docker.com/_/postgres/ +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE DATABASE "ICAT"; + CREATE USER irods WITH PASSWORD 'testpassword'; + GRANT ALL PRIVILEGES ON DATABASE "ICAT" to irods; +EOSQL diff --git a/test_harness/multiple_node/irods_catalog_5/Dockerfile b/test_harness/multiple_node/irods_catalog_5/Dockerfile new file mode 100644 index 000000000..112ffbaaa --- /dev/null +++ b/test_harness/multiple_node/irods_catalog_5/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:16 + +COPY init-user-db.sh /docker-entrypoint-initdb.d/init-user-db.sh diff --git a/test_harness/multiple_node/irods_catalog_5/init-user-db.sh b/test_harness/multiple_node/irods_catalog_5/init-user-db.sh new file mode 100644 index 000000000..f3c724e2f --- /dev/null +++ b/test_harness/multiple_node/irods_catalog_5/init-user-db.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Adapted from "Initialization script" in documentation for official Postgres dockerhub: +# https://hub.docker.com/_/postgres/ +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE DATABASE "ICAT"; + CREATE USER irods WITH PASSWORD 'testpassword'; + GRANT ALL PRIVILEGES ON DATABASE "ICAT" to irods; + ALTER DATABASE "ICAT" OWNER TO irods +EOSQL diff --git a/test_harness/multiple_node/irods_catalog_provider_4/Dockerfile b/test_harness/multiple_node/irods_catalog_provider_4/Dockerfile new file mode 100644 index 000000000..4dbce592e --- /dev/null +++ b/test_harness/multiple_node/irods_catalog_provider_4/Dockerfile @@ -0,0 +1,68 @@ +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y \ + apt-transport-https \ + gnupg2 \ + lsb-release \ + wget \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +RUN mkdir -p /etc/apt/keyrings && \ + wget -qO - https://packages.irods.org/irods-signing-key.asc | \ + gpg \ + --no-options \ + --no-default-keyring \ + --no-auto-check-trustdb \ + --homedir /dev/null \ + --no-keyring \ + --import-options import-export \ + --output /etc/apt/keyrings/renci-irods-archive-keyring.pgp \ + --import \ + && \ + echo "deb [signed-by=/etc/apt/keyrings/renci-irods-archive-keyring.pgp arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" | \ + tee /etc/apt/sources.list.d/renci-irods.list + +RUN apt-get update && \ + apt-get install -y \ + libcurl4-gnutls-dev \ + jq \ + python3 \ + python3-distro \ + python3-jsonschema \ + python3-pip \ + python3-psutil \ + python3-requests \ + rsyslog \ + unixodbc \ + gawk \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +ARG irods_version +ARG irods_package_version_suffix=-0~focal +ARG irods_package_version=${irods_version}${irods_package_version_suffix} +ARG irods_resource_plugin_version=${irods_version}.0${irods_package_version_suffix} + +RUN apt-get update && \ + apt-get install -y \ + irods-database-plugin-postgres=${irods_package_version} \ + irods-runtime=${irods_package_version} \ + irods-server=${irods_package_version} \ + irods-icommands=${irods_package_version} \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +COPY setup-${irods_version}.input / +RUN mv /setup-${irods_version}.input /irods_setup.input + +WORKDIR / +COPY entrypoint.sh . +RUN chmod u+x ./entrypoint.sh +ENTRYPOINT ["./entrypoint.sh"] diff --git a/test_harness/multiple_node/irods_catalog_provider_4/entrypoint.sh b/test_harness/multiple_node/irods_catalog_provider_4/entrypoint.sh new file mode 100644 index 000000000..eccdb39df --- /dev/null +++ b/test_harness/multiple_node/irods_catalog_provider_4/entrypoint.sh @@ -0,0 +1,49 @@ +#! /bin/bash -e + +catalog_db_hostname=irods-catalog + +echo "Waiting for iRODS catalog database to be ready" + +until pg_isready -h ${catalog_db_hostname} -d ICAT -U irods -q; do + sleep 1 +done + +echo "iRODS catalog database is ready" + +setup_input_file=/irods_setup.input + +if [ -e "${setup_input_file}" ]; then + echo "Running iRODS setup" + python3 /var/lib/irods/scripts/setup_irods.py <"${setup_input_file}" + rm /irods_setup.input +fi + +ORIG_SERVER_CONFIG=/etc/irods/server_config.json +MOD_SERVER_CONFIG=/tmp/server_config.json.$$ + +chown -R irods:irods /irods_shared + +{ + [ -f ~/provider-address.do_not_remove ] || { + jq <$ORIG_SERVER_CONFIG >$MOD_SERVER_CONFIG \ + '.host_resolution.host_entries += [ + { + "address_type": "local", + "addresses": [ + "irods-catalog-provider", + "'$(hostname)'" + ] + } + ]' && + cat <$MOD_SERVER_CONFIG >$ORIG_SERVER_CONFIG && + touch ~/provider-address.do_not_remove + } +} || { + echo >&2 "Error modifying $ORIG_SERVER_CONFIG" + exit 1 +} + +echo "Starting server" + +cd /usr/sbin +su irods -c './irodsServer -u' diff --git a/test_harness/multiple_node/irods_catalog_provider_4/setup-4.3.1.input b/test_harness/multiple_node/irods_catalog_provider_4/setup-4.3.1.input new file mode 100644 index 000000000..d8c10deca --- /dev/null +++ b/test_harness/multiple_node/irods_catalog_provider_4/setup-4.3.1.input @@ -0,0 +1,28 @@ + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +1248 + +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +32_byte_server_control_plane_key +rods + + diff --git a/test_harness/multiple_node/irods_catalog_provider_4/setup-4.3.2.input b/test_harness/multiple_node/irods_catalog_provider_4/setup-4.3.2.input new file mode 100644 index 000000000..d8c10deca --- /dev/null +++ b/test_harness/multiple_node/irods_catalog_provider_4/setup-4.3.2.input @@ -0,0 +1,28 @@ + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +1248 + +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +32_byte_server_control_plane_key +rods + + diff --git a/test_harness/multiple_node/irods_catalog_provider_4/setup-4.3.3.input b/test_harness/multiple_node/irods_catalog_provider_4/setup-4.3.3.input new file mode 100644 index 000000000..d8c10deca --- /dev/null +++ b/test_harness/multiple_node/irods_catalog_provider_4/setup-4.3.3.input @@ -0,0 +1,28 @@ + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +1248 + +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +32_byte_server_control_plane_key +rods + + diff --git a/test_harness/multiple_node/irods_catalog_provider_4/setup-4.3.4.input b/test_harness/multiple_node/irods_catalog_provider_4/setup-4.3.4.input new file mode 100644 index 000000000..d8c10deca --- /dev/null +++ b/test_harness/multiple_node/irods_catalog_provider_4/setup-4.3.4.input @@ -0,0 +1,28 @@ + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +1248 + +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +32_byte_server_control_plane_key +rods + + diff --git a/test_harness/multiple_node/irods_catalog_provider_5/Dockerfile b/test_harness/multiple_node/irods_catalog_provider_5/Dockerfile new file mode 100644 index 000000000..bc0923d3d --- /dev/null +++ b/test_harness/multiple_node/irods_catalog_provider_5/Dockerfile @@ -0,0 +1,69 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y \ + apt-transport-https \ + gnupg2 \ + lsb-release \ + wget \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +RUN wget -qO - https://packages.irods.org/irods-signing-key.asc | \ + gpg \ + --no-options \ + --no-default-keyring \ + --no-auto-check-trustdb \ + --homedir /dev/null \ + --no-keyring \ + --import-options import-export \ + --output /etc/apt/keyrings/renci-irods-archive-keyring.pgp \ + --import \ + && \ + echo "deb [signed-by=/etc/apt/keyrings/renci-irods-archive-keyring.pgp arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" | \ + tee /etc/apt/sources.list.d/renci-irods.list + +RUN apt-get update && \ + apt-get install -y \ + libcurl4-gnutls-dev \ + jq \ + python3 \ + python3-distro \ + python3-jsonschema \ + python3-pip \ + python3-psutil \ + python3-requests \ + rsyslog \ + unixodbc \ + gawk \ + postgresql-client-16 \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +# - postgres client 16 is for pg_isready + +ARG irods_version +ARG irods_package_version_suffix=-0~noble +ARG irods_package_version=${irods_version}${irods_package_version_suffix} + +RUN apt-get update && \ + apt-get install -y \ + irods-database-plugin-postgres=${irods_package_version} \ + irods-runtime=${irods_package_version} \ + irods-server=${irods_package_version} \ + irods-icommands=${irods_package_version} \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +COPY setup-${irods_version}.input / +RUN mv /setup-${irods_version}.input /irods_setup.input + +WORKDIR / +COPY entrypoint.sh . +RUN chmod u+x ./entrypoint.sh +ENTRYPOINT ["./entrypoint.sh"] diff --git a/test_harness/multiple_node/irods_catalog_provider_5/entrypoint.sh b/test_harness/multiple_node/irods_catalog_provider_5/entrypoint.sh new file mode 100644 index 000000000..198638a5e --- /dev/null +++ b/test_harness/multiple_node/irods_catalog_provider_5/entrypoint.sh @@ -0,0 +1,50 @@ +#! /bin/bash -e + +catalog_db_hostname=irods-catalog + +echo "Waiting for iRODS catalog database to be ready" + +until pg_isready -h ${catalog_db_hostname} -d ICAT -U irods -q; do + sleep 1 +done + +echo "iRODS catalog database is ready" + +setup_input_file=/irods_setup.input + +if [ -e "${setup_input_file}" ]; then + echo "Running iRODS setup" + python3 /var/lib/irods/scripts/setup_irods.py <"${setup_input_file}" + rm /irods_setup.input +fi + +ORIG_SERVER_CONFIG=/etc/irods/server_config.json +MOD_SERVER_CONFIG=/tmp/server_config.json.$$ + +chown -R irods:irods /irods_shared +chmod 0777 /irods_shared + +{ + [ -f ~/provider-address.do_not_remove ] || { + jq <$ORIG_SERVER_CONFIG >$MOD_SERVER_CONFIG \ + '.host_resolution.host_entries += [ + { + "address_type": "local", + "addresses": [ + "irods-catalog-provider", + "'$(hostname)'" + ] + } + ]' && + cat <$MOD_SERVER_CONFIG >$ORIG_SERVER_CONFIG && + touch ~/provider-address.do_not_remove + } +} || { + echo >&2 "Error modifying $ORIG_SERVER_CONFIG" + exit 1 +} + +echo "Starting server" + +cd /usr/sbin +su irods -c './irodsServer -p /tmp/irods.pid --stdout' diff --git a/test_harness/multiple_node/irods_catalog_provider_5/setup-5.0.0.input b/test_harness/multiple_node/irods_catalog_provider_5/setup-5.0.0.input new file mode 100644 index 000000000..9bcaf0852 --- /dev/null +++ b/test_harness/multiple_node/irods_catalog_provider_5/setup-5.0.0.input @@ -0,0 +1,26 @@ + + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +rods + + diff --git a/test_harness/multiple_node/irods_catalog_provider_5/setup-5.0.1.input b/test_harness/multiple_node/irods_catalog_provider_5/setup-5.0.1.input new file mode 100644 index 000000000..9bcaf0852 --- /dev/null +++ b/test_harness/multiple_node/irods_catalog_provider_5/setup-5.0.1.input @@ -0,0 +1,26 @@ + + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +rods + + diff --git a/test_harness/multiple_node/irods_catalog_provider_5/setup-5.0.2.input b/test_harness/multiple_node/irods_catalog_provider_5/setup-5.0.2.input new file mode 100644 index 000000000..9bcaf0852 --- /dev/null +++ b/test_harness/multiple_node/irods_catalog_provider_5/setup-5.0.2.input @@ -0,0 +1,26 @@ + + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +rods + + diff --git a/test_harness/multiple_node/print_repo_root_location b/test_harness/multiple_node/print_repo_root_location new file mode 100755 index 000000000..7dd2dd869 --- /dev/null +++ b/test_harness/multiple_node/print_repo_root_location @@ -0,0 +1,5 @@ +#!/bin/bash +# The following line needs be kept updated to reflect true position relative to repository root, +# in the event this script or any of its chain of containing directories (up to but not including the repo root) are moved. +REPO_ROOT_RELATIVE_TO_THIS_SCRIPT=../.. +realpath "$(dirname "$0")/$REPO_ROOT_RELATIVE_TO_THIS_SCRIPT" diff --git a/test_harness/multiple_node/python_client/Dockerfile b/test_harness/multiple_node/python_client/Dockerfile new file mode 100644 index 000000000..ca8559b61 --- /dev/null +++ b/test_harness/multiple_node/python_client/Dockerfile @@ -0,0 +1,2 @@ +ARG python_version +FROM python:${python_version} diff --git a/test_harness/multiple_node/run_tests.sh b/test_harness/multiple_node/run_tests.sh new file mode 100755 index 000000000..df5c32fe2 --- /dev/null +++ b/test_harness/multiple_node/run_tests.sh @@ -0,0 +1,44 @@ +#!/bin/bash +set -e -x +PYTHON=$(which python3) +if [ -z "$PYTHON" ]; then + PYTHON=$(which python) +fi +DIR=$(dirname "$0") +cd "$DIR" + +REPO="$(./print_repo_root_location)" + +# Set up /irods_shared (created as a shared mount by the docker compose configuration) and subfolders for tests. + +# Get the numeric user and group id's for irods service account on the provider. This helps to set up the test user +# with proper permissions for the shared volume on the client node. +groupadd -o -g $(stat -c%g /irods_shared) irods +useradd -g irods -u $(stat -c%u /irods_shared) irods + +# Set up useful subdirectories in the client/provider shared volume. +mkdir /irods_shared/{tmp,reg_resc} +chown irods:irods /irods_shared/{tmp,reg_resc} +chmod 777 /irods_shared/reg_resc +chmod g+ws /irods_shared/tmp + +# Make a test user in group irods, who will run the client tests. +useradd -G irods -m -s/bin/bash prc-test-user + +# Create writable copy of this repo. +cp -r /"$REPO"{,.copy} +REPO+=.copy +chown -R prc-test-user "$REPO" +chmod u+w "$REPO"/irods/test/test-data + +# Install PRC from the repo. +$PYTHON -m pip install "$REPO[tests]" + +su - prc-test-user -c "\ +$PYTHON '$DIR'/../utility/iinit.py \ + host irods-catalog-provider \ + port 1247 \ + user rods \ + password rods \ + zone tempZone +$PYTHON '$REPO'/irods/test/runner.py $*" diff --git a/test_harness/multiple_node/start_containers.sh b/test_harness/multiple_node/start_containers.sh new file mode 100755 index 000000000..ce9349acf --- /dev/null +++ b/test_harness/multiple_node/start_containers.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -e + +# This script is launched on the docker host. + +usage() { + echo >&2 "usage: $0 [-n] [-b ''] irods_version python_version" + exit 2 +} + +SHELL_DOCKER_COMPOSE_BUILD_ARGS="" +DO_NOT_RUN="" + +while [[ $1 = -* ]]; do + if [ "$1" = "-b" ]; then + SHELL_DOCKER_COMPOSE_BUILD_ARGS=$2 + shift 2 + fi + if [ "$1" = "-n" ]; then + DO_NOT_RUN=1 + shift + fi +done + +if [ $# -eq 2 ]; then + IRODS_VERSION=$1 + PYTHON_VERSION=$2 + shift 2 +else + usage +fi + +[ -n "$PYTHON_VERSION" -a -n "$IRODS_VERSION" ] || { + usage +} + +IRODS_MAJOR=${IRODS_VERSION//.*/} + +DIR=$(dirname "$0") +cd "${DIR}" +REPO_ROOT=$(realpath ../..) + +echo "\ +repo_external=\"${REPO_ROOT}\" +python_version=\"${PYTHON_VERSION}\" +irods_version=\"${IRODS_VERSION}\" +irods_major=\"${IRODS_MAJOR}\"" >.env + +docker compose -f harness-docker-compose-irods-${IRODS_MAJOR}.yml build $SHELL_DOCKER_COMPOSE_BUILD_ARGS + +if [ -z "$DO_NOT_RUN" ]; then + docker compose -f harness-docker-compose-irods-${IRODS_MAJOR}.yml up -d +fi diff --git a/test_harness/multiple_node/stop_containers.sh b/test_harness/multiple_node/stop_containers.sh new file mode 100755 index 000000000..5851ad5f6 --- /dev/null +++ b/test_harness/multiple_node/stop_containers.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -e + +# This script is launched on the docker host. + +usage() { + echo >&2 "usage: $0 irods_version" + exit 1 +} + +if [ $# -eq 1 ]; then + IRODS_VERSION=$1 + shift +else + usage +fi + +[ -n "$IRODS_VERSION" ] || { + usage +} + +IRODS_MAJOR=${IRODS_VERSION//.*/} + +docker compose -f harness-docker-compose-irods-${IRODS_MAJOR}.yml down From e2f49b0884bc1fbec87a16e196a7889daf5d926b Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Fri, 14 Nov 2025 19:44:38 -0500 Subject: [PATCH 2/6] [#502] test harness and container-based tests A new test harness is introduced in which we construct a new container (using either Docker or podman) for each test program we run. This allows full customization of the container environment for the particular needs of each test. Accordingly, also included with the Github workflows is a full run of the PRC test suite with the iRODS server and catalog DB server running in the same container as the client. In the process of putting old tests through new rigors, faults were found and corrected in some of those tests. --- .../run-test-programs-single-node.yml | 40 +++ .../workflows/run-test-suite-single-node.yml | 29 +++ Dockerfile.prc_test.centos | 1 - Dockerfile.prc_test.ubuntu | 1 - irods/message/__init__.py | 2 +- irods/test/access_test.py | 6 +- irods/test/exception_test.py | 4 +- irods/test/login_auth_test.sh | 90 +++++++ irods/test/login_auth_test_1.py | 1 + irods/test/login_auth_test_2.py | 1 + .../test/login_auth_test_must_run_manually.py | 23 +- irods/test/meta_test.py | 11 +- irods/test/pam.bats/funcs | 108 -------- .../test001_pam_password_expiration.bats | 68 ----- ...pam_interactive_test_must_run_manually.py} | 0 irods/test/rule_test.py | 12 +- irods/test/runner.py | 67 ++++- irods/test/scripts/run_suite_locally.sh | 57 ++++ ...write_pam_credentials_to_secrets_file.bats | 9 +- ...c_write_irodsA_utility_in_native_mode.bats | 3 + ...ssue_362_rogue_chars_in_pam_password.bats} | 46 ++-- irods/test/scripts/test_support_functions | 244 ++++++++++++++++++ irods/test/scripts/update_json_for_test | 69 +++++ irods/test/setupssl.py | 2 +- irods/test/test_ssl_context.bats | 4 +- .../single_node/000_install-irods.Dockerfile | 14 + .../001_bats-and-system-python.Dockerfile | 6 + .../002_compile-specific-python.Dockerfile | 16 ++ test_harness/single_node/README.md | 60 +++++ test_harness/single_node/build-docker.sh | 39 +++ .../single_node/create_docker_images.sh | 14 + .../single_node/docker_container_driver.sh | 169 ++++++++++++ test_harness/single_node/install.sh | 161 ++++++++++++ .../single_node/install_python_rule_engine | 34 +++ .../irods_version_greater_or_equal_to | 43 +++ test_harness/single_node/manage_irods5_procs | 59 +++++ .../single_node/most_recent_python.sh | 25 ++ .../single_node/print_repo_root_location | 5 + .../single_node/setup_python_rule_engine | 93 +++++++ .../single_node/start_postgresql_and_irods.sh | 40 +++ .../single_node/test_script_parameters | 44 ++++ test_harness/utility/iinit.py | 47 ++++ 42 files changed, 1532 insertions(+), 235 deletions(-) create mode 100644 .github/workflows/run-test-programs-single-node.yml create mode 100644 .github/workflows/run-test-suite-single-node.yml create mode 100755 irods/test/login_auth_test.sh create mode 120000 irods/test/login_auth_test_1.py create mode 120000 irods/test/login_auth_test_2.py delete mode 100644 irods/test/pam.bats/funcs delete mode 100644 irods/test/pam.bats/test001_pam_password_expiration.bats rename irods/test/{pam_interactive_test.py => pam_interactive_test_must_run_manually.py} (100%) create mode 100755 irods/test/scripts/run_suite_locally.sh rename irods/test/{PRC_issue_362.bats => scripts/test010_issue_362_rogue_chars_in_pam_password.bats} (59%) mode change 100644 => 100755 create mode 100644 irods/test/scripts/test_support_functions create mode 100644 irods/test/scripts/update_json_for_test create mode 100644 test_harness/single_node/000_install-irods.Dockerfile create mode 100644 test_harness/single_node/001_bats-and-system-python.Dockerfile create mode 100644 test_harness/single_node/002_compile-specific-python.Dockerfile create mode 100644 test_harness/single_node/README.md create mode 100755 test_harness/single_node/build-docker.sh create mode 100755 test_harness/single_node/create_docker_images.sh create mode 100755 test_harness/single_node/docker_container_driver.sh create mode 100755 test_harness/single_node/install.sh create mode 100755 test_harness/single_node/install_python_rule_engine create mode 100755 test_harness/single_node/irods_version_greater_or_equal_to create mode 100755 test_harness/single_node/manage_irods5_procs create mode 100755 test_harness/single_node/most_recent_python.sh create mode 100755 test_harness/single_node/print_repo_root_location create mode 100755 test_harness/single_node/setup_python_rule_engine create mode 100755 test_harness/single_node/start_postgresql_and_irods.sh create mode 100644 test_harness/single_node/test_script_parameters create mode 100755 test_harness/utility/iinit.py diff --git a/.github/workflows/run-test-programs-single-node.yml b/.github/workflows/run-test-programs-single-node.yml new file mode 100644 index 000000000..69473f1c7 --- /dev/null +++ b/.github/workflows/run-test-programs-single-node.yml @@ -0,0 +1,40 @@ +# Run a set of tests, each in its own container and with a potentially customized setup. + +# To this end, each test is launched via docker_container_driver.sh, part of the single-node +# test harness machinery whose implementation and documentation is to be found in the +# `irods/test/harness` directory. + +# A test may be written as a Bash or BATS script, but the only real requirement is that it be +# executable. + +name: run-test-programs-single-node + +on: [push, pull_request] + +jobs: + tests: + name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./test_harness/single_node + strategy: + matrix: + python: ['3.9','3.13'] + irods_server: ['4.3.4','5.0.2'] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build images + run: ./create_docker_images.sh "${{ matrix.irods_server }}" "${{ matrix.python }}" + + - name: run tests + run: | + for script in \ + ../../irods/test/scripts/test[0-9]* \ + ../../irods/test/login_auth_test_*.py + do + ./docker_container_driver.sh -V $script + done diff --git a/.github/workflows/run-test-suite-single-node.yml b/.github/workflows/run-test-suite-single-node.yml new file mode 100644 index 000000000..7f3b88acc --- /dev/null +++ b/.github/workflows/run-test-suite-single-node.yml @@ -0,0 +1,29 @@ +# Run the client test suite in a Docker container, targeting a locally running instance of the iRODS server. +# (Documentation and implementation for the test harness may be found in `irods/test/harness`.) + +name: run-test-suite-single-node + +on: [push, pull_request] + +jobs: + tests: + name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./test_harness/single_node + strategy: + matrix: + python: ['3.9','3.13'] + irods_server: ['4.3.4','5.0.2'] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build images + run: ./create_docker_images.sh "${{ matrix.irods_server }}" "${{ matrix.python }}" + + - name: run tests + run: | + ./docker_container_driver.sh -V ../../irods/test/scripts/run_suite_locally.sh diff --git a/Dockerfile.prc_test.centos b/Dockerfile.prc_test.centos index debed6d6d..c171fa62e 100644 --- a/Dockerfile.prc_test.centos +++ b/Dockerfile.prc_test.centos @@ -24,6 +24,5 @@ RUN python${py_N} repo/docker_build/iinit.py \ password rods SHELL ["/bin/bash","-c"] CMD echo "Waiting on iRODS server... " ; \ - python${PY_N} repo/docker_build/recv_oneshot -h irods-provider -p 8888 -t 360 && \ sudo groupadd -o -g $(stat -c%g /irods_shared) irods && sudo usermod -aG irods user && \ newgrp irods < repo/run_python_tests.sh diff --git a/Dockerfile.prc_test.ubuntu b/Dockerfile.prc_test.ubuntu index e8c958a85..79ed07e12 100644 --- a/Dockerfile.prc_test.ubuntu +++ b/Dockerfile.prc_test.ubuntu @@ -31,6 +31,5 @@ SHELL ["/bin/bash","-c"] # 3. run python tests as the new group CMD echo "Waiting on iRODS server... " ; \ - python${PY_N} repo/docker_build/recv_oneshot -h irods-provider -p 8888 -t 360 && \ sudo groupadd -o -g $(stat -c%g /irods_shared) irods && sudo usermod -aG irods user && \ newgrp irods < repo/run_python_tests.sh diff --git a/irods/message/__init__.py b/irods/message/__init__.py index 818ae3677..9b8e0ec80 100644 --- a/irods/message/__init__.py +++ b/irods/message/__init__.py @@ -181,7 +181,7 @@ def ET(xml_type=(), server_version=None): logger = logging.getLogger(__name__) -IRODS_VERSION = (5, 0, 1, "d") +IRODS_VERSION = (5, 0, 2, "d") UNICODE = str diff --git a/irods/test/access_test.py b/irods/test/access_test.py index 7c496c685..73c122b76 100644 --- a/irods/test/access_test.py +++ b/irods/test/access_test.py @@ -533,11 +533,7 @@ def test_iRODSAccess_cannot_be_constructed_using_unsupported_type__issue_558(sel # Before the fix in #558, this would have been allowed and only later would the type discrepancy be revealed, # leading to opaque error messages. Now, the types are checked on the way in to ensure clarity and correctness. # TODO(#480): We cannot use the unittest.assertRaises context manager as this was introduced in python 3.1. - assertCall = getattr(self, "assertRaisesRegex", None) - if assertCall is None: - assertCall = self.assertRaisesRegexp - - assertCall( + self.assertRaisesRegex( TypeError, "'path' parameter must be of type 'str', 'irods.collection.iRODSCollection', " "'irods.data_object.iRODSDataObject', or 'irods.path.iRODSPath'.", diff --git a/irods/test/exception_test.py b/irods/test/exception_test.py index 16a95c989..b400cf2aa 100644 --- a/irods/test/exception_test.py +++ b/irods/test/exception_test.py @@ -41,8 +41,8 @@ def test_400(self): excep_repr = repr(exc) errno_object = irods.exception.Errno(errno.EACCES) errno_repr = repr(errno_object) - self.assertRegexpMatches(errno_repr, r"\bErrno\b") - self.assertRegexpMatches( + self.assertRegex(errno_repr, r"\bErrno\b") + self.assertRegex( errno_repr, """['"]{msg}['"]""".format(msg=os.strerror(errno.EACCES)) ) self.assertIn(errno_repr, excep_repr) diff --git a/irods/test/login_auth_test.sh b/irods/test/login_auth_test.sh new file mode 100755 index 000000000..39eb711da --- /dev/null +++ b/irods/test/login_auth_test.sh @@ -0,0 +1,90 @@ +#!/bin/bash +. "$(dirname "$0")/scripts/test_support_functions" +. "$(dirname "$0")/scripts/update_json_for_test" + +IRODS_SERVER_CONFIG=/etc/irods/server_config.json +IRODS_SERVICE_ACCOUNT_ENV_FILE=~irods/.irods/irods_environment.json +LOCAL_ACCOUNT_ENV_FILE=~/.irods/irods_environment.json + +cannot_iinit='' +tries=8 +while true; do + iinit_as_rods >/dev/null 2>&1 && break + [ $((--tries)) -le 0 ] && { + cannot_iinit=1 + break + } + sleep 5 +done +[ -n "$cannot_iinit" ] && { + echo >&2 "Could not iinit as rods." + exit 2 +} + +setup_preconnect_preference DONT_CARE + +add_irods_to_system_pam_configuration + +# set up /etc/irods/ssl directory and files +set_up_ssl sudo + +sudo useradd -ms/bin/bash alissa +sudo chpasswd <<<"alissa:test123" + +update_json_file $IRODS_SERVICE_ACCOUNT_ENV_FILE \ + "$(newcontent $IRODS_SERVICE_ACCOUNT_ENV_FILE ssl_keys)" + +# This is mostly so we can call python3 as just "python" +activate_virtual_env_with_prc_installed >/dev/null 2>&1 || { + echo >&2 "couldn't set up virtual environment" + exit 1 +} + +server_hup= +if irods_server_version ge 5.0.0; then + server_hup="y" + update_json_file $IRODS_SERVER_CONFIG \ + "$(newcontent $IRODS_SERVER_CONFIG tls_server_items tls_client_items)" + + sudo su - irods -c "$IRODS_CONTROL_PATH/manage_irods5_procs rescan-config" +fi + +# Configure clients with admin user + TLS + +update_json_file $LOCAL_ACCOUNT_ENV_FILE \ + "$(newcontent $LOCAL_ACCOUNT_ENV_FILE ssl_keys encrypt_keys)" + +update_time=0 +# We won't time out, however we will warn for each minute the server +# has not returned to readiness. +if [ "$server_hup" = "y" ]; then + # wait for server to be ready after configuration reload + while true; do + sleep 2 + if ils >/dev/null 2>&1; then + break + else + now=$(date +%s) + if [ $now -ge $((update_time + 60)) ]; then + echo >&2 "At [$(date)] ... still waiting on server reload" + update_time=$now + fi + fi + done +fi + +if [ -n "$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" ]; then + original_script="/prc/$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" + + # Run tests. + if [ -x "$original_script" ]; then + command "$original_script" $* + elif [[ $original_script =~ \.py$ ]]; then + python "$original_script" $* + elif [[ $original_script =~ \.bats$ ]]; then + bats "$original_script" + else + echo >&2 "I don't know how to run this: original_script=[$original_script]" + fi + +fi diff --git a/irods/test/login_auth_test_1.py b/irods/test/login_auth_test_1.py new file mode 120000 index 000000000..23402ed84 --- /dev/null +++ b/irods/test/login_auth_test_1.py @@ -0,0 +1 @@ +login_auth_test_must_run_manually.py \ No newline at end of file diff --git a/irods/test/login_auth_test_2.py b/irods/test/login_auth_test_2.py new file mode 120000 index 000000000..23402ed84 --- /dev/null +++ b/irods/test/login_auth_test_2.py @@ -0,0 +1 @@ +login_auth_test_must_run_manually.py \ No newline at end of file diff --git a/irods/test/login_auth_test_must_run_manually.py b/irods/test/login_auth_test_must_run_manually.py index 4303e47c1..027206894 100644 --- a/irods/test/login_auth_test_must_run_manually.py +++ b/irods/test/login_auth_test_must_run_manually.py @@ -23,7 +23,7 @@ from re import compile as regex from typing import Dict, Optional import gc -from irods.test.setupssl import create_ssl_dir +from irods.test.setup_ssl import create_ssl_dir # # Allow override to specify the PAM password in effect for the test rodsuser. @@ -160,7 +160,7 @@ class TestLogins(unittest.TestCase): single-node iRODS system, by the service account user. This ensures the /etc/irods directory is local and writable. - 2. ./setupssl.py (sets up SSL keys etc. in /etc/irods/ssl) should be run + 2. ./setup_ssl.py (sets up SSL keys etc. in /etc/irods/ssl) should be run first to create (or overwrite, if appropriate) the /etc/irods/ssl directory and its contents. @@ -207,6 +207,9 @@ def create_env_dirs(self): authentication_scheme=lookup["AUTH"], password=lookup["PASSWORD"], port=1247, + **( + {**SERVER_ENV_SSL_SETTINGS, **CLIENT_OPTIONS_FOR_SSL} if self.admin.server_version >= (5,) else {} + ) ) try: pam_hashes = ses.pam_pw_negotiated @@ -295,6 +298,7 @@ def _setup_rodsuser_and_optional_pw(self, name, make_irods_pw=False): def tst0( self, ssl_opt, auth_opt, env_opt, name=TEST_RODS_USER, make_irods_pw=False ): + session = None _auth_opt = auth_opt if auth_opt in ("pam", "pam_password"): auth_opt = self.PAM_SCHEME_STRING @@ -377,7 +381,10 @@ def tst0( ) print("---") - return session + if session: + session.cleanup() + return session + # == test defaulting to 'native' @@ -413,14 +420,16 @@ def test_5(self): self.tst0(ssl_opt=True, auth_opt="pam", env_opt=False) def test_6(self): + if self.admin.server_version >= (5,): + self.skipTest("iRODS 5 does not permit sending the raw PAM password on an unencrypted connection.") try: - ses = self.tst0(ssl_opt=False, auth_opt="pam", env_opt=False) + session = self.tst0(ssl_opt=False, auth_opt="pam", env_opt=False) except PlainTextPAMPasswordError: pass else: # -- no exception raised (this is expected behavior in 4.3+ with the new authentication framework, # but for 4.2 and previous, we expect the PlainTextPAMPasswordError to be raised. - if ses.server_version_without_auth() < (4, 3): + if session.server_version_without_auth() < (4, 3): self.fail("PlainTextPAMPasswordError should have been raised") def test_7(self): @@ -534,7 +543,7 @@ def test_nonanonymous_login_without_auth_file_fails__290(self): s.users.get("bob") os.unlink(bob_auth) # -- Check that we raise an appropriate exception pointing to the missing auth file path -- - with self.assertRaisesRegexp(NonAnonymousLoginWithoutPassword, bob_auth): + with self.assertRaisesRegex(NonAnonymousLoginWithoutPassword, bob_auth): with helpers.make_session(**login_options) as s: s.users.get("bob") finally: @@ -610,7 +619,7 @@ def setUp(self): self.skipTest("TestWithSSL may not be run by user irods") if not os.path.exists("/etc/irods/ssl"): self.skipTest( - "Running setupssl.py as irods user is prerequisite for this test." + "Running setup_ssl.py as irods user is prerequisite for this test." ) with helpers.make_session() as session: if not session.host in ("localhost", socket.gethostname()): diff --git a/irods/test/meta_test.py b/irods/test/meta_test.py index aa33dae26..1a0d01bf4 100644 --- a/irods/test/meta_test.py +++ b/irods/test/meta_test.py @@ -683,16 +683,16 @@ def test_AVUs_populated_improperly_with_empties_or_nonstrings_fail_identically__ def test_nonstring_as_AVU_value_raises_an_error__issue_434(self): args = ("an_attribute", 0) - with self.assertRaisesRegexp(Bad_AVU_Field, "incorrect type"): + with self.assertRaisesRegex(Bad_AVU_Field, "incorrect type"): self.coll.metadata.set(*args) - with self.assertRaisesRegexp(Bad_AVU_Field, "incorrect type"): + with self.assertRaisesRegex(Bad_AVU_Field, "incorrect type"): self.coll.metadata.add(*args) def test_empty_string_as_AVU_value_raises_an_error__issue_434(self): args = ("an_attribute", "") - with self.assertRaisesRegexp(Bad_AVU_Field, "zero-length"): + with self.assertRaisesRegex(Bad_AVU_Field, "zero-length"): self.coll.metadata.set(*args) - with self.assertRaisesRegexp(Bad_AVU_Field, "zero-length"): + with self.assertRaisesRegex(Bad_AVU_Field, "zero-length"): self.coll.metadata.add(*args) @unittest.skipUnless( @@ -724,10 +724,11 @@ def test_that_all_column_mappings_are_uniquely_and_properly_defined__issue_643( prepend_col_prefix_if_needed = lambda s: ( "COL_" + s if not s.startswith("COL_") else s ) + current_server_version = self.sess.server_version prc_column_defs = sorted( [ (prepend_col_prefix_if_needed(i[1].icat_key), i[1].icat_id) - for i in ModelBase.column_items + for i in ModelBase.column_items if current_server_version >= i[1].min_version ] ) diff --git a/irods/test/pam.bats/funcs b/irods/test/pam.bats/funcs deleted file mode 100644 index 30539a03a..000000000 --- a/irods/test/pam.bats/funcs +++ /dev/null @@ -1,108 +0,0 @@ -dot_to_space() { - sed 's/\./ /g'<<<"$1" -} - -CLEANUP=$':\n' - -GT() { (return 1); echo $?; } -LT() { (return -1); echo $?; } -EQ() { (return 0); echo $?; } - -compare_int_tuple() { - local x=($1) y=($2) - local lx=${#x[@]} ly=${#y[@]} - local i maxlen=$((lx > ly ? lx : ly)) - for ((i=0;i ~/.irods/irods_environment.json - iinit <<<"$1" 2>/dev/tty -} - -_end_pam_environment_and_password() { - rm -fr ~/.irods - mv ~/.irods.$$ ~/.irods -} - -setup_pam_login_for_alice() { - sudo useradd alice --create-home - local PASSWD=${1:-test123} - sudo chpasswd <<<"alice:$PASSWD" - iadmin mkuser alice rodsuser - _begin_pam_environment_and_password "$PASSWD" -} - -finalize_pam_login_for_alice() { - _end_pam_environment_and_password - iadmin rmuser alice - sudo userdel alice --remove -} - -test_specific_cleanup() { - eval "$CLEANUP" -} diff --git a/irods/test/pam.bats/test001_pam_password_expiration.bats b/irods/test/pam.bats/test001_pam_password_expiration.bats deleted file mode 100644 index 3e29100ef..000000000 --- a/irods/test/pam.bats/test001_pam_password_expiration.bats +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bats - -. "$BATS_TEST_DIRNAME"/test_support_functions -PYTHON=python3 - -# Setup/prerequisites are same as for login_auth_test. -# Run as ubuntu user with sudo; python_irodsclient must be installed (in either ~/.local or a virtualenv) -# - -PASSWD=test123 - -setup() -{ - setup_pam_login_for_alice $PASSWD -} - -teardown() -{ - finalize_pam_login_for_alice - test_specific_cleanup -} - -@test f001 { - - # Define the core Python to be run, basically a minimal code block ensuring that we can authenticate to iRODS - # without an exception being raised. - - local SCRIPT=" -import irods.test.helpers as h -ses = h.make_session() -ses.collections.get(h.home_collection(ses)) -print ('env_auth_scheme=%s' % ses.pool.account._original_authentication_scheme) -" - - # Test that the first run of the code in $SCRIPT is successful, i.e. normal authenticated operations are possible. - - local OUTPUT=$($PYTHON -c "$SCRIPT") - - [[ $OUTPUT =~ ^env_auth_scheme=pam_password$ ]] - - SET_CLEANUP=yes \ - with_change_auth_params_for_test password_min_time 4 \ - password_max_time 5 - - # Test that running the $SCRIPT raises an exception if the PAM password has expired. - - iinit <<<"$PASSWD" - HOME_COLLECTION=$(ipwd) - sleep 9 - OUTPUT=$($PYTHON -c "$SCRIPT" 2>&1 >/dev/null || true) - grep 'RuntimeError: Time To Live' <<<"$OUTPUT" - - # Test that the $SCRIPT, when run with proper settings, can successfully reset the password. - - with_change_auth_params_for_test password_max_time 3600 - - OUTPUT=$($PYTHON -c "import irods.client_configuration as cfg -cfg.legacy_auth.pam.password_for_auto_renew = '$PASSWD' -cfg.legacy_auth.pam.time_to_live_in_hours = 1 -cfg.legacy_auth.pam.store_password_to_environment = True -$SCRIPT") - - [[ $OUTPUT =~ ^env_auth_scheme=pam_password$ ]] - - # Test that iCommands can authenticate with the newly written .irodsA file - - iquest "%s" "select COLL_NAME where COLL_NAME like '%/home/alice%'"| grep "^$HOME_COLLECTION\$" -} diff --git a/irods/test/pam_interactive_test.py b/irods/test/pam_interactive_test_must_run_manually.py similarity index 100% rename from irods/test/pam_interactive_test.py rename to irods/test/pam_interactive_test_must_run_manually.py diff --git a/irods/test/rule_test.py b/irods/test/rule_test.py index be95302b7..39b718a26 100644 --- a/irods/test/rule_test.py +++ b/irods/test/rule_test.py @@ -20,7 +20,7 @@ RE_Plugins_installed_run_condition_args = ( os.environ.get("PYTHON_RULE_ENGINE_INSTALLED", "*").lower()[:1] == "y", - "Test depends on server having Python-REP installed beyond the default options", + "Test depends on server having Python-REP installed (set PYTHON_RULE_ENGINE_INSTALLED=yes in environment)." ) @@ -420,7 +420,7 @@ def test_rulefile_in_file_like_object_1__336(self): ) output = r.execute() lines = self.lines_from_stdout_buf(output) - self.assertRegexpMatches(lines[0], r".*\[Hello world!\]") + self.assertRegex(lines[0], r".*\[Hello world!\]") def test_rulefile_in_file_like_object_2__336(self): @@ -442,8 +442,8 @@ def test_rulefile_in_file_like_object_2__336(self): r = Rule(self.sess, rule_file=io.BytesIO(rule_file_contents.encode("utf-8"))) output = r.execute() lines = self.lines_from_stdout_buf(output) - self.assertRegexpMatches(lines[0], r"\[STRING\]\[\]") - self.assertRegexpMatches(lines[1], r"\[STRING\]\[\]") + self.assertRegex(lines[0], r"\[STRING\]\[\]") + self.assertRegex(lines[1], r"\[STRING\]\[\]") r = Rule( self.sess, @@ -452,8 +452,8 @@ def test_rulefile_in_file_like_object_2__336(self): ) output = r.execute() lines = self.lines_from_stdout_buf(output) - self.assertRegexpMatches(lines[0], r"\[INTEGER\]\[5\]") - self.assertRegexpMatches(lines[1], r"\[STRING\]\[A String\]") + self.assertRegex(lines[0], r"\[INTEGER\]\[5\]") + self.assertRegex(lines[1], r"\[STRING\]\[A String\]") if __name__ == "__main__": diff --git a/irods/test/runner.py b/irods/test/runner.py index f9f9fa610..8782d6391 100644 --- a/irods/test/runner.py +++ b/irods/test/runner.py @@ -7,6 +7,7 @@ """ +import argparse import os import sys from unittest import TestLoader, TestSuite @@ -22,20 +23,74 @@ h.setFormatter(f) logger.addHandler(h) +parser = argparse.ArgumentParser() -# Load all tests in the current directory and run them +def abs_path(initial_dir, levels_up = 0): + directory = initial_dir + while levels_up > 0: + levels_up -= 1 + directory = os.path.join(directory,'..') + return os.path.abspath(directory) + +# Load all tests in the current directory and run them. if __name__ == "__main__": - # must set the path for the imported tests - sys.path.insert(0, os.path.abspath("../..")) + + # Get path to script directory for test import and/or discovery. + script_dir = os.path.abspath(os.path.dirname(sys.argv[0])) + + # Must set the path for the imported tests. + sys.path.insert(0, abs_path(script_dir, levels_up = 2)) + + parser.add_argument('--tests', '-t', + metavar='TESTS', + dest='tests', + nargs='+', + help='List of tests to run.') + + parser.add_argument('--environment_variable', '-e', + metavar='ENVIRONMENT_VARIABLE', + dest='env_var', + type=str, + help='Name of environment variable name to scan for in reason strings when filtering skipped test names to be output.') + + parser.add_argument('--output_tests_skipped', '-s', + metavar='SKIPPED_TESTS_OUTPUT_FILENAME', + dest='skipped_tests_output_filename', + type=str, + help='Name of a file into which to write names of skipped tests.') + + parser.add_argument('--tests_file', '-f', + metavar='TESTS_FILE', + dest='tests_file', + help='Name of a file containing a list of tests to run.') + + args = parser.parse_args() + + if args.tests_file: + if args.tests: + print ('Cannot specify both --tests and --tests_file', file = sys.stderr) + exit(2) + args.tests = filter(None,open(args.tests_file).read().split("\n")) loader = TestLoader() - suite = TestSuite( - loader.discover(start_dir=".", pattern="*_test.py", top_level_dir=".") - ) + + if args.tests: + suite = TestSuite(loader.loadTestsFromNames(args.tests)) + else: + suite = TestSuite(loader.discover(start_dir = script_dir, pattern = '*_test.py', top_level_dir = script_dir)) result = xmlrunner.XMLTestRunner( verbosity=2, output="/tmp/python-irodsclient/test-reports" ).run(suite) + + if args.skipped_tests_output_filename: + with open(args.skipped_tests_output_filename,'w') as skip_file: + do_output = (lambda reason: (args.env_var in reason) if args.env_var + else True) + for testinfo, reason in result.skipped: + if do_output(reason): + print(testinfo.test_id, file=skip_file) + if result.wasSuccessful(): sys.exit(0) diff --git a/irods/test/scripts/run_suite_locally.sh b/irods/test/scripts/run_suite_locally.sh new file mode 100755 index 000000000..5420ba679 --- /dev/null +++ b/irods/test/scripts/run_suite_locally.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -e + +SCRIPT_DIR=$(dirname "$0") +. "$SCRIPT_DIR"/test_support_functions + +report_environment_variables() { + echo "PRC under test with these environment variables active:" + python -c " +import os, sys +for name in ['IRODS_PACKAGE_VERSION','PYTHON_VERSION']: + value = os.environ.get(name) + print(f' {name}=[{value}]') +print(f'{sys.executable = }') +print(f'{sys.version = }') + " +} + +run_tests() { + setup_pyN + su - testuser -c " + set -e + source /pyN/bin/activate + pip install -e /prc.rw[tests] + cd /prc.rw/irods/test + python /prc.rw/test_harness/utility/iinit.py \ + host localhost \ + port 1247 \ + user rods \ + zone tempZone \ + password rods + $(declare -f report_environment_variables) + report_environment_variables + python runner.py --output_tests_skipped /tmp/skipped.txt -e PYTHON_RULE_ENGINE_INSTALLED + " + + # Install PREP (Python Rule Engine Plugin). + ( + set -e + cd "/prc/test_harness/single_node" + apt update + ./install_python_rule_engine + su irods -c './setup_python_rule_engine --wait' + ) + + # Run PREP-dependent tests that were previously skipped. + su - testuser -c " + set -e + source /pyN/bin/activate + cd /prc.rw/irods/test + $(declare -f report_environment_variables) + report_environment_variables + env PYTHON_RULE_ENGINE_INSTALLED=yes python runner.py --tests_file /tmp/skipped.txt + " +} + +run_tests diff --git a/irods/test/scripts/test003_write_pam_credentials_to_secrets_file.bats b/irods/test/scripts/test003_write_pam_credentials_to_secrets_file.bats index acf9c9594..33f23b6f0 100755 --- a/irods/test/scripts/test003_write_pam_credentials_to_secrets_file.bats +++ b/irods/test/scripts/test003_write_pam_credentials_to_secrets_file.bats @@ -43,9 +43,14 @@ except irods.client_init.irodsA_already_exists: [ -n "$CONTENTS1" -a "$CONTENTS1" = "$CONTENTS2" ] # Now delete the already existing irodsA and repeat without negating overwrite. + TIMESTAMP_0=$(stat -c%Y $auth_file) + # The sleep ensures the mtime of $auth_file will change if/when it is rewritten. + sleep 2 $PYTHON -c "import irods.client_init; irods.client_init.write_pam_irodsA_file('$ALICES_NEW_PAM_PASSWD')" - CONTENTS3=$(cat $auth_file) - [ "$CONTENTS2" != "$CONTENTS3" ] + TIMESTAMP=$(stat -c%Y $auth_file) + + # Test only the timestamp of the new auth_file, not the content, since that is implicitly asserted by the next step. + [ $(($TIMESTAMP-TIMESTAMP_0)) -ge 1 ] # Define the core Python to be run, basically a minimal code block ensuring that we can authenticate to iRODS # without an exception being raised. diff --git a/irods/test/scripts/test008_prc_write_irodsA_utility_in_native_mode.bats b/irods/test/scripts/test008_prc_write_irodsA_utility_in_native_mode.bats index 23aecd8ce..de9304461 100755 --- a/irods/test/scripts/test008_prc_write_irodsA_utility_in_native_mode.bats +++ b/irods/test/scripts/test008_prc_write_irodsA_utility_in_native_mode.bats @@ -48,6 +48,9 @@ print ('env_auth_scheme=%s' % ses.pool.account._original_authentication_scheme) # Write another .irodsA prc_write_irodsA.py native <<<"rods" + CLIENT_JSON=~/.irods/irods_environment.json + jq '.["irods_client_server_policy"]="CS_NEG_REFUSE"' <$CLIENT_JSON >/tmp/client_json_test008.$$ + mv /tmp/client_json_test008.$$ $CLIENT_JSON # Verify new .irodsA for both iCommands and PRC use. ils >/tmp/stdout OUTPUT=$($PYTHON -c "$SCRIPT") diff --git a/irods/test/PRC_issue_362.bats b/irods/test/scripts/test010_issue_362_rogue_chars_in_pam_password.bats old mode 100644 new mode 100755 similarity index 59% rename from irods/test/PRC_issue_362.bats rename to irods/test/scripts/test010_issue_362_rogue_chars_in_pam_password.bats index c3a1ef80f..e0e004242 --- a/irods/test/PRC_issue_362.bats +++ b/irods/test/scripts/test010_issue_362_rogue_chars_in_pam_password.bats @@ -1,27 +1,26 @@ +#!/usr/bin/env bats + # The tests in this BATS module must be run as a (passwordless) sudo-enabled user. # It is also required that the python irodsclient be installed under irods' ~/.local environment. -. $BATS_TEST_DIRNAME/scripts/test_support_functions +. $BATS_TEST_DIRNAME/test_support_functions setup() { - - iinit_as_rods - - setup_pam_login_for_user "test123" alice - - cat >~/test_get_home_coll.py <<-EOF - import irods.test.helpers as h - ses = h.make_session() - home_coll = h.home_collection(ses) - exit(0 if ses.collections.get(home_coll).path == home_coll - and ses.pool.account._original_authentication_scheme.lower() in ('pam','pam_password') - else 1) - EOF -} - -teardown() { - iinit_as_rods - finalize_pam_login_for_user alice + [ -f /tmp/test010_one_time_initialize_flag ] || { + rm -fr ~/.irods + /prc/test_harness/utility/iinit.py host localhost \ + port 1247 \ + zone tempZone \ + user rods \ + password rods \ + ## Because iRODS 5+ negotiates for SSL automatically: + CLIENT_JSON=~/.irods/irods_environment.json + jq '.["irods_client_server_policy"]="CS_NEG_REFUSE"' >$CLIENT_JSON.$$ <$CLIENT_JSON + mv $CLIENT_JSON.$$ $CLIENT_JSON + + setup_pam_login_for_user "test123" alice + } + touch /tmp/test010_one_time_initialize_flag } prc_test() @@ -36,7 +35,14 @@ prc_test() local USER="alice" local PASSWORD="my${CHR}pass" sudo chpasswd <<<"$USER:$PASSWORD" - env PYTHON_IRODSCLIENT_CONFIGURATION_PATH='' python ~/test_get_home_coll.py + env PYTHON_IRODSCLIENT_CONFIGURATION_PATH='' python <<-EOF + import irods.test.helpers as h + ses = h.make_session() + home_coll = h.home_collection(ses) + exit(0 if ses.collections.get(home_coll).path == home_coll + and ses.pool.account._original_authentication_scheme.lower() in ('pam','pam_password') + else 1) + EOF } @test "test_with_atsymbol" { prc_test "@"; } diff --git a/irods/test/scripts/test_support_functions b/irods/test/scripts/test_support_functions new file mode 100644 index 000000000..a7e40bfe4 --- /dev/null +++ b/irods/test/scripts/test_support_functions @@ -0,0 +1,244 @@ +# This file may be sourced from a test script written in Bash or BATS. It +# contains a number of utility functions, most of which have to do with: +# +# 1. implementing common tasks, often setup or configuration wrt iRODS; or +# 2. more primitive functions e.g. string manipulations or comparisons, etc. + +SCRIPTDIR=${BASH_SOURCE[0]} +up_from_script_dir() { + local x incr="" + for ((x=0;x<${1:-0};x++)); do incr+="/.."; done + realpath "$(dirname "$SCRIPTDIR")""$incr" +} + +# Sample usages: +# By user irods: set_up_ssl "" "-q" +# By sudo enabled user: set_up_ssl "sudo" "-q" +set_up_ssl() { + local SUDO=${1:-""} + local OPTS=${2:-""} + $SUDO su - irods -c "python3 $(up_from_script_dir 1)/setup_ssl.py $OPTS" +} + +# Clears out environment and resets to rodsadmin 'rods'. +# Meant mostly to allow initial steps by a rodsadmin for setting up tests. + +iinit_as_rods() { + rm -fr ~/.irods + iinit <<<$(hostname)$'\n1247\nrods\ntempZone\nrods' +} + +dot_to_space() { + sed 's/\./ /g'<<<"$1" +} + +CLEANUP=$':\n' + +GT() { (return 1); echo $?; } +LT() { (return -1); echo $?; } +EQ() { (return 0); echo $?; } + +compare_int_tuple() { + local x=($1) y=($2) + local lx=${#x[@]} ly=${#y[@]} + local i maxlen=$((lx > ly ? lx : ly)) + for ((i=0;i ~/.irods/irods_environment.json + + if [ -n "$1" -a -z "$SKIP_IINIT_FOR_PASSWORD" ]; then + iinit <<<"$1" 2>/tmp/iinit_as_alice.log + fi +} + +_end_pam_environment_and_password() { + rm -fr ~/.irods + mv ~/.irods.$$ ~/.irods +} + +setup_pam_login_for_user() { + local user=${2:-alice} + sudo useradd $user --create-home + local PASSWD=${1:-test123} + sudo chpasswd <<<"$user:$PASSWD" + iadmin mkuser $user rodsuser + _begin_pam_environment_and_password "$PASSWD" $user +} + +setup_pam_login_for_alice() { + setup_pam_login_for_user "$1" alice +} + +finalize_pam_login_for_user() { + local USER=${1} + _end_pam_environment_and_password + iadmin rmuser "$USER" + sudo userdel "$USER" --remove +} + +finalize_pam_login_for_alice() { + finalize_pam_login_for_user alice +} + +test_specific_cleanup() { + eval "$CLEANUP" +} + +# PostgreSQL only +age_out_pam_password() { + # sets create_ts and modify_ts (timestamps) to older values, decreasing them by an amount of (offset + 1) where offset + # is the number of seconds for expiry_ts stored in the ICAT for the given user and password. In this way, we can + # artificially age out an existing pam password. + # Parameters: + # $1 - The username + # $2 - (optional) override the amount used for offsetting the create & modify timestamps. + local id=$(iquest %s "select USER_ID where USER_NAME = '$1'") + local offset=$(sudo su - postgres -c "psql -t ICAT -c 'select pass_expiry_ts from r_user_password where user_id = $id'") + local mtime=$(sudo su - postgres -c "psql -t ICAT -c 'select modify_ts from r_user_password where user_id = $id'") + mtime=$(sed 's/^\s*0//' <<<"$mtime") + [ -n "$2" ] && offset="$2" + ((offset+=1)) + local new_time=$((mtime - offset)) + sudo su - postgres -c "psql ICAT -c 'update r_user_password set create_ts=$new_time, modify_ts=$new_time where user_id=$id'" +} + +call_irodsctl() { + local arg=${1:-restart} + sudo su - irods -c "./irodsctl $arg" +} + +add_irods_to_system_pam_configuration() { + local tempfile=/tmp/irods-pam-config.$$ + cat <<-EOF >$tempfile + auth required pam_env.so + auth sufficient pam_unix.so + auth requisite pam_succeed_if.so uid >= 500 quiet + auth required pam_deny.so + EOF + sudo chown root.root $tempfile + sudo mv $tempfile /etc/pam.d/irods +} + +setup_preconnect_preference() { + sudo su irods -c "sed -i.orig 's/\(^\s*acPreConnect.*CS_NEG\)\([A-Z_]*\)/\1_$1/' /etc/irods/core.re" +} + +setup_pyN() { + if [ ! -d /pyN ]; then + mkdir /pyN ; chown testuser /pyN + su - testuser -c "/root/python/bin/python3 -m virtualenv /pyN" + cp -r /prc{,.rw} + chown -R testuser /prc.rw + fi +} + +# Call to both initialize and activate a virtualenv with the requested Python3 interpreter +# installed. Requires image to descend from 'bats-and-system-python'. +activate_virtual_env_with_prc_installed() +{ + local py_venv=${1:-pyN} + # install python client. We use a recursive copy of /prc so bdist doesn't try to build from a readonly mount + [ "$py_venv" = pyN ] && sudo bash -c "$(declare -f setup_pyN); setup_pyN" && \ + sudo su - -c "source /${py_venv}/bin/activate && cp -rp /prc /prc-copy && \ + pip install '/prc-copy[tests]' && sudo rm -fr /prc-copy" && \ + source /${py_venv}/bin/activate && \ + echo "---> Python virtual environment activated. Interpreter Version is: $(python -V)" && \ + python -c 'import irods' || { echo >&2 "ERROR: python-irodsclient install failed."; false; } +} + +# Produces an output of a precise timestamp with the file content appended, +# as an indicator of whether a file has been modified. +mtime_and_content() +{ + stat -c%y "$1" + cat "$1" +} + +irods_server_version() { + python -c "import irods.helpers as h +import operator,sys +if len(sys.argv) == 1: + (comparison,relto)=('','') +elif len(sys.argv) == 3: + (comparison,relto)=sys.argv[1:3] +fm_tuple = lambda tup: '.'.join(str(_) for _ in tup) +to_tuple = lambda vstr: tuple(int(_) for _ in vstr.split('.')) +svt = h.make_session().server_version_without_auth() +if relto: + exit(0 if vars(operator)[comparison](svt,to_tuple(relto)) else 1) +print(fm_tuple(svt)) +" $1 $2 +} diff --git a/irods/test/scripts/update_json_for_test b/irods/test/scripts/update_json_for_test new file mode 100644 index 000000000..76e19082c --- /dev/null +++ b/irods/test/scripts/update_json_for_test @@ -0,0 +1,69 @@ +#!/bin/bash + +declare -A tls_server_items=( + [tls_server]='{"certificate_chain_file":"/etc/irods/ssl/irods.crt", + "certificate_key_file":"/etc/irods/ssl/irods.key", + "dh_params_file":"/etc/irods/ssl/dhparams.pem"}' +) + +declare -A tls_client_items=( + [tls_client]='{"ca_certificate_file":"/etc/irods/ssl/irods.crt", + "ca_certificate_path":"/etc/ssl/certs", + "verify_server":"cert"}' +) + +declare -A ssl_keys=( + [irods_client_server_negotiation]='"request_server_negotiation"' + [irods_client_server_policy]='"CS_NEG_REQUIRE"' + [irods_ssl_ca_certificate_file]='"/etc/irods/ssl/irods.crt"' + [irods_ssl_certificate_chain_file]='"/etc/irods/ssl/irods.crt"' + [irods_ssl_certificate_key_file]='"/etc/irods/ssl/irods.key"' + [irods_ssl_dh_params_file]='"/etc/irods/ssl/dhparams.pem"' + [irods_ssl_verify_server]='"cert"' +) + +declare -A pam_keys=( + [irods_authentication_scheme]="\"$(pam_auth_string)\"" +) + +declare -A encrypt_keys=( + [irods_encryption_key_size]=16 + [irods_encryption_salt_size]=8 + [irods_encryption_num_hash_rounds]=16 + [irods_encryption_algorithm]='"AES-256-CBC"' +) + +declare -A RESTORE_FILES=() + +update_json_file() { + local file=$1 content=$2 + local bn=$(basename "$file") + local orig=/tmp/$bn.orig.$$ + local newfile=/tmp/$bn.new.$$ + echo "$content" >"$newfile" + sudo chmod --reference "$file" "$newfile" + sudo chown --reference "$file" "$newfile" + { sudo mv "$file" "$orig" && sudo mv "$newfile" "$file"; } || return 1 + RESTORE_FILES["$file"]="$orig" +} + +restore_json_files() { + local kk + for kk in ${!RESTORE_FILES[@]}; do + sudo mv -f "${RESTORE_FILES["$kk"]}" "$kk" + done +} + +newcontent() { + local file=$1 + shift + local j=$(sudo cat "$file") + while [ $# -gt 0 ]; do + eval ' + for kk in ${!'$1'[@]}; do + j=$(jq ".$kk=${'$1'[$kk]}" <<<"$j") + done' + shift + done + echo "$j" +} diff --git a/irods/test/setupssl.py b/irods/test/setupssl.py index 3d3c20205..d14e682c0 100755 --- a/irods/test/setupssl.py +++ b/irods/test/setupssl.py @@ -60,7 +60,7 @@ def create_ssl_dir( # https://www.openssl.org/docs/man1.0.2/man1/dhparam.html#:~:text=DH%20parameter%20generation%20with%20the,that%20may%20be%20possible%20otherwise. if use_strong_primes_for_dh_generation: dhparam_generation_command = ( - "openssl dhparam -2 -out dhparams.pem" + "openssl dhparam -2 -out dhparams.pem 2048" ) else: dhparam_generation_command = ( diff --git a/irods/test/test_ssl_context.bats b/irods/test/test_ssl_context.bats index 8c92791a5..3aa954528 100755 --- a/irods/test/test_ssl_context.bats +++ b/irods/test/test_ssl_context.bats @@ -82,10 +82,10 @@ if [ "`cat /tmp/run`" != "$RUN" ]; then sudo $REPO_SCRIPTS/json_config --clear-store ${ABBREVIATIONS[*]} # Set up the basic server cert, key, and DH params file. - [ -e /etc/irods/ssl ] || sudo su irods -c "$REPO_SCRIPTS/setupssl.py -f" + [ -e /etc/irods/ssl ] || sudo su irods -c "$REPO_SCRIPTS/setup_ssl.py -f" # Set up another cert with non-matching hostname. - sudo su irods -c "$REPO_SCRIPTS/setupssl.py -kf -x.localhost -hlocalhost" + sudo su irods -c "$REPO_SCRIPTS/setup_ssl.py -kf -x.localhost -hlocalhost" sudo su irods -c "c_rehash /etc/irods/ssl" # Change the iRODS svc account user's (and current user's) iRODS environment file for SSL. diff --git a/test_harness/single_node/000_install-irods.Dockerfile b/test_harness/single_node/000_install-irods.Dockerfile new file mode 100644 index 000000000..feac0dbc6 --- /dev/null +++ b/test_harness/single_node/000_install-irods.Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:22.04 +COPY install.sh / +ARG irods_package_version +ENV IRODS_PACKAGE_VERSION="$irods_package_version" +RUN for phase in initialize install-essential-packages add-package-repo; do \ + bash /install.sh --w=$phase 0; \ + done && \ + /install.sh 4 +COPY start_postgresql_and_irods.sh manage_irods5_procs / +RUN apt install -y sudo && \ + useradd -ms/bin/bash testuser && \ + echo 'testuser ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers +ENV IRODS_CONTROL_PATH="" +CMD bash $IRODS_CONTROL_PATH/start_postgresql_and_irods.sh diff --git a/test_harness/single_node/001_bats-and-system-python.Dockerfile b/test_harness/single_node/001_bats-and-system-python.Dockerfile new file mode 100644 index 000000000..9f697a787 --- /dev/null +++ b/test_harness/single_node/001_bats-and-system-python.Dockerfile @@ -0,0 +1,6 @@ +FROM install-irods +RUN apt update +RUN apt install -y python3-pip bats && \ + python3 -m pip install --upgrade pip && \ + python3 -m pip install virtualenv && \ + python3 -m virtualenv /py3 diff --git a/test_harness/single_node/002_compile-specific-python.Dockerfile b/test_harness/single_node/002_compile-specific-python.Dockerfile new file mode 100644 index 000000000..f67fbf554 --- /dev/null +++ b/test_harness/single_node/002_compile-specific-python.Dockerfile @@ -0,0 +1,16 @@ +FROM bats-and-system-python +RUN apt update && \ + apt install -y wget build-essential && \ + apt install -y libssl-dev zlib1g-dev libffi-dev libncurses-dev wget build-essential +ARG python_version +RUN wget https://www.python.org/ftp/python/${python_version}/Python-${python_version}.tar.xz && \ + tar xf Python-${python_version}.tar.xz +WORKDIR /Python-${python_version} +RUN ./configure --prefix /root/python --with-ensurepip=install && \ + make -j && \ + mkdir /root/python && \ + make install +WORKDIR / +RUN /root/python/bin/python3 -m pip install virtualenv && \ + chmod a+rx /root +ENV PYTHON_VERSION=${python_version} diff --git a/test_harness/single_node/README.md b/test_harness/single_node/README.md new file mode 100644 index 000000000..6647a92ab --- /dev/null +++ b/test_harness/single_node/README.md @@ -0,0 +1,60 @@ +# Docker Powered Test Harness + +## Description + +A series of Docker images which support running isolated test scripts (using BATS, bash, or Python). +Once built, the images allow loading and customizing the Docker container environment for a given +test script. + +The general form for test invocation is: `docker_container_driver.sh ` + +Within the container, a computed internal path to the same script is executed, whether directly or +indirectly by a wrapper script. The wrapper for many of the PRC authentication-via-PAM tests is +irods/test/login_auth_test.sh. + +The test_script_parameters file, located in the irods/test/harness directory, contains customized +settings for each test script run, including: + + - Docker image name to be used. + + - Wrapper to be invoked, if any. Wrappers shall perform common setup tasks up to and including + invoking the test script itself. + + - Which user is running the test. (Unless otherwise specified, this is the passwordless-sudo- + enabled user `testuser`). + +When done with a test, the `docker_container_driver.sh` exit code mirrors the return code from the +run of the test script. The container itself is removed unless the `-L` ("leak") option is given. + +## Sample Runs + +### To build required images + +For our convenience in this doc, set a shell variable `REPOROOT` to `~/python-irodsclient` (or +similar) to specify the path to the top level of the local repository. + +Sample command lines to build Docker images: + +1. ``` + cd $REPO_ROOT/irods/test/harness + ./build_docker.sh + ``` + + Builds docker images in proper sequence. + +2. ``` + cd $REPO_ROOT/irods/test/harness; + IRODS_PACKAGE_VERSION=4.3.4 PYTHON_VERSION=3.11 NO_CACHE=1 ./build-docker.sh [ Dockerfiles... ] + ``` + + Builds (ignoring docker cache) images based on specific iRODS package version and desired + Python Interpreter version, optionally with a restricted list of Docker files in need of rebulding. + +### To run a test script + +``` +$REPO_ROOT/irods/test/harness/docker_container_driver.sh $REPO_ROOT/irods/test/scripts/run_local_suite +``` + +For both builder and driver script, the environment variable `DOCKER` may be set to `podman` to run +the alternative container engine. Otherwise it defaults to a value of `docker`. diff --git a/test_harness/single_node/build-docker.sh b/test_harness/single_node/build-docker.sh new file mode 100755 index 000000000..979e233e7 --- /dev/null +++ b/test_harness/single_node/build-docker.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# environment variables for build +# IRODS_PACKAGE_VERSION if defined is like "4.3.4" or "5.0.1". +# (but contains no '~' suffix for irods versions <= 4.2.10) +# PYTHON_VERSION is usually two dot-separated numbers: example "3.13", but could also have zero, one or three version numbers. +# (Do not specify the triple form, X.Y.Z, if that release is not known to exist - not counting alphas and release candidates) + +DIR=$(realpath "$(dirname "$0")") +: ${DOCKER:=docker} + +if [ $# -gt 0 ]; then + IFS=$'\n' read -ra ARGS -d '' < <(realpath --relative-to "$DIR" "$@") + cd "$DIR" +else + cd "$DIR" + ARGS=([0-9]*.Dockerfile) +fi + +: ${PYTHON_VERSION:=3.13} export PYTHON_VERSION + +for dockerfile in "${ARGS[@]}"; do + image_name=${dockerfile#[0-9]*_} + image_name=${image_name%.Dockerfile} + irods_package_version_option="" + python_version_option="" + if [ "$image_name" = "install-irods" ]; then + irods_package_version_option=${IRODS_PACKAGE_VERSION:+"--build-arg=irods_package_version=$IRODS_PACKAGE_VERSION"} + elif [ "$image_name" = "compile-specific-python" ]; then + temp=$(./most_recent_python.sh "$PYTHON_VERSION") + if [ -n "$temp" ]; then + PYTHON_VERSION="$temp" + fi + python_version_option=${PYTHON_VERSION:+"--build-arg=python_version=$PYTHON_VERSION"} + fi + $DOCKER build -f $dockerfile -t $image_name . $irods_package_version_option $python_version_option \ + ${NO_CACHE+"--no-cache"} || \ + { STATUS=$?; echo "*** Failure while building [$image_name]"; exit $STATUS; } +done diff --git a/test_harness/single_node/create_docker_images.sh b/test_harness/single_node/create_docker_images.sh new file mode 100755 index 000000000..bf4cd5db9 --- /dev/null +++ b/test_harness/single_node/create_docker_images.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +export IRODS_PACKAGE_VERSION=$1 +export PYTHON_VERSION=$2 + +[ -z "$1" -o -z "$2" ] && { + echo >&2 "usage: $0 " + exit 2 +} +shift 2 + +DIR=$(dirname "$0") + +"$DIR"/build-docker.sh $* diff --git a/test_harness/single_node/docker_container_driver.sh b/test_harness/single_node/docker_container_driver.sh new file mode 100755 index 000000000..996f54778 --- /dev/null +++ b/test_harness/single_node/docker_container_driver.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash + +# Runs a test program within a new container. The container is dispatched and/or disposed of, and the exit +# status code of the target test program collected and returned, by this script. + +# The repository containing this harness directory is mapped to a direct subdirectory of / within the container. +# (By present convention that subdirectory is: /prc) The test program to be run is given by its host path, and +# the internal (to the container) path will be computed. + +# A sourced header for this script, 'test_script_parameters', contains configuration for each script that will +# be run under its control. + +# Options: + +IRODS_CONTROL_PATH="" +KILL_TEST_CONTAINER=1 +RUN_AS_USER="" +ECHO_CONTAINER="" +REMOVE_OPTION="--rm" +EXPLICIT_WORKDIR="" +INTERACTIVE_OPTION="" +VERBOSITY=0 + +usage() { + echo "Usage: + $0 [options] /external/path/to/script" + + echo 'Options : + -V (extra verbosity). Prints out useful stuff including VERSION information + -u USERNAME run test in container as this USERNAME + -i invoke container with -i and -t + -c verbosely prints the container name + -L leaks the container (i.e. does not kill it after the test run) + -p "/path/value" sets the IRODS_CONTROL_PATH for changing where iRODS server control scripts are looked for + -r "remove_option_value" (default: "--rm") may be used to suppress ("") automatic removal of container when stopped + -w "explicit_workdir" specify a work directory when running the test +' + exit 2 +} + +while [[ $1 = -* ]]; do + if [ "$1" = -i ]; then + INTERACTIVE_OPTION="-it" + shift + elif [ "$1" = -V ]; then + VERBOSITY=1 + shift + elif [ "$1" = -c ]; then + ECHO_CONTAINER=1 + shift + elif [ "$1" = -L ]; then + KILL_TEST_CONTAINER=0 + shift + elif [ "$1" = -p ]; then + IRODS_CONTROL_PATH="$2" + shift 2 + elif [ "$1" = -u ]; then + RUN_AS_USER="$2" + shift 2 + elif [ "$1" = -r ]; then + REMOVE_OPTION="$2" + shift 2 + elif [ "$1" = -w ]; then + EXPLICIT_WORKDIR="$2" + shift 2 + else + usage + fi +done + +if [ "$1" = "" ]; then + usage +fi + +DIR=$(dirname "$0") +. "$DIR"/test_script_parameters + +testscript=${1} +shift + +testscript_basename=$(basename "$testscript") +arglist=${wrapper_arglist[$testscript_basename]:-$*} # arglist dominated by symbolic link name if any + +if [ -L "$testscript" ]; then + testscript=$(realpath "$testscript") + testscript_basename=$(basename "$testscript") +fi + +original_testscript_abspath=$(realpath "$testscript") + +wrapped=${wrappers["$testscript_basename"]} + +if [ -n "$wrapped" ]; then + # wrapped is assumed to contain a leading path element relative to the referencing script's containing directory + testscript="$(dirname "$testscript")/$wrapped" + testscript_basename=$(basename "$testscript") +fi + +testscript_abspath=$(realpath "$testscript") + +cd "$DIR" + +image=${images[$testscript_basename]} + +if [ -z "$RUN_AS_USER" ]; then + RUN_AS_USER=${user[$testscript_basename]} +fi + +# Tests are run as testuser by default +: ${RUN_AS_USER:='testuser'} + +WORKDIR="" +if [ -n "$EXPLICIT_WORKDIR" ]; then + WORKDIR="$EXPLICIT_WORKDIR" +else + WORKDIR=${workdirs[$RUN_AS_USER]} +fi + +reporoot=$(./print_repo_root_location) +ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$(realpath --relative-to "$reporoot" "$original_testscript_abspath") + +echo "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=[$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT]" +INNER_MOUNT=/prc + +: ${DOCKER:=docker} + +# Start the container. +echo image="[$image]" +CONTAINER=$($DOCKER run -d -v "$reporoot:$INNER_MOUNT:ro" $INTERACTIVE_OPTION $REMOVE_OPTION \ + -e "IRODS_CONTROL_PATH=$IRODS_CONTROL_PATH" $image) + +# Wait for iRODS and database to start up. +TIME0=$(date +%s) +while :; do + [ $(date +%s) -gt $((TIME0 + 30)) ] && { + echo >&2 "Waited too long for DB and iRODS to start" + exit 124 + } + sleep 1 + $DOCKER exec $CONTAINER grep '(0)' /tmp/irods_status 2>/dev/null >/dev/null + [ $? -ne 0 ] && { + echo -n . >&2 + continue + } + break +done + +if [ $VERBOSITY -gt 0 ]; then + echo $'\n'"==> Running script [$testscript_abspath]" + echo "in container [$CONTAINER]" + echo "with these *_VERSION variables in environment: " + $DOCKER exec $CONTAINER bash -c 'env|grep _VERSION' | sed $'s/^/\t/' +fi + +$DOCKER exec ${RUN_AS_USER:+"-u$RUN_AS_USER"} \ + ${WORKDIR:+"-w$WORKDIR"} \ + -e "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" \ + -e "IRODS_CONTROL_PATH=$IRODS_CONTROL_PATH" \ + $INTERACTIVE_OPTION $CONTAINER \ + "$INNER_MOUNT/$(realpath --relative-to "$reporoot" "$testscript_abspath")" \ + $arglist +STATUS=$? + +if [ $((0 + KILL_TEST_CONTAINER)) -ne 0 ]; then + echo >&2 'Killed:' $($DOCKER stop --timeout=0 $CONTAINER) +fi + +[ -n "$ECHO_CONTAINER" ] && echo $CONTAINER +exit $STATUS diff --git a/test_harness/single_node/install.sh b/test_harness/single_node/install.sh new file mode 100755 index 000000000..98d5ed20a --- /dev/null +++ b/test_harness/single_node/install.sh @@ -0,0 +1,161 @@ +#!/bin/bash + +# A script to manage the main steps in installing an iRODS server as well as all necessary support. (Dependencies, +# catalog database, etc.) + +DEV_HOME="$HOME" +: ${DEV_REPOS:="$DEV_HOME/github"} + +add_package_repo() { + echo >&2 "... installing package repo" + sudo apt update + sudo apt install -y lsb-release apt-transport-https gnupg2 + wget -qO - https://packages.irods.org/irods-signing-key.asc | + gpg \ + --no-options \ + --no-default-keyring \ + --no-auto-check-trustdb \ + --homedir /dev/null \ + --no-keyring \ + --import-options import-export \ + --output /etc/apt/keyrings/renci-irods-archive-keyring.pgp \ + --import && + echo "deb [signed-by=/etc/apt/keyrings/renci-irods-archive-keyring.pgp arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" | + tee /etc/apt/sources.list.d/renci-irods.list + + sudo apt update +} + +# Expand a spec of the leading version tuple eg. 4.3.4 out to the full name of +# the most recent matching version of the package + +# Report the latest version spec (including OS) that matches the env var IRODS_PACKAGE_VERSION (eg. "5.0.2" -> "5.0.2-0~jammy) + +irods_package_vsn() { + apt list -a irods-server 2>/dev/null | awk '{print $2}' | grep '\w' | sort | + grep "$(perl -e 'print quotemeta($ARGV[0])' "$IRODS_PACKAGE_VERSION")" | tail -1 +} + +# Report the version number of the installed iRODS server if any. + +irods_vsn() { + local V=$(dpkg -l irods-server 2>/dev/null | grep '^ii\s' | awk '{print $3}') + echo "${V}" +} + +while [[ "$1" = -* ]]; do + ARG="$1" + shift + case $ARG in + --i=* | --irods=* | --irods-version=*) IRODS_PACKAGE_VERSION=${ARG#*=} ;; + --w=* | --with=* | --with-options=*) withopts=${ARG#*=} ;; + esac +done + +run_phase() { + + local PHASE=$1 + local with_opts=" $2 " + + case "$PHASE" in + + 0) + + if [[ $with_opts = *\ initialize\ * ]]; then + apt-get -y update + apt-get install -y apt-transport-https wget lsb-release sudo jq + fi + + if [[ $with_opts = *\ sudo-without-pw\ * ]]; then + if [ $(id -u) = 0 -a "${USER:-root}" = root ]; then + echo >&2 "root authorization for 'sudo' is automatic - no /etc/sudoers modification needed" + else + if [ -f "/etc/sudoers" ]; then + # add a line with our USER name to /etc/sudoers if not already there + sudo su -c "sed -n '/^\s*[^#]/p' /etc/sudoers | grep '^$USER\s*ALL=(ALL)\s*NOPASSWD:\s*ALL\s*$' >/dev/null" || + sudo su -c "echo '$USER ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers" + else + echo >&2 "WARNING - Could not modify sudoers files" + fi + fi # not root + fi # with-opts + + #------ (needed for both package install and build from source) + + if [[ $with_opts = *\ install-essential-packages\ * ]]; then + + if ! dpkg -l tzdata >/dev/null 2>&1; then + sudo su - root -c \ + "env DEBIAN_FRONTEND=noninteractive bash -c 'apt-get install -y tzdata'" + fi + sudo apt-get update + sudo apt-get install -y software-properties-common postgresql + sudo apt-get update && + sudo apt-get install -y libfuse2 unixodbc rsyslog + fi + + if [[ $with_opts = *\ add-package-repo\ * ]]; then + add_package_repo -f + fi + + if [[ $with_opts = *\ create-db\ * ]]; then + sudo su - postgres -c " + { dropdb --if-exists ICAT + dropuser --if-exists irods ; } >/dev/null 2>&1" + sudo su - postgres -c "psql <<\\ +________ + CREATE DATABASE \"ICAT\"; + CREATE USER irods WITH PASSWORD 'testpassword'; + GRANT ALL PRIVILEGES ON DATABASE \"ICAT\" to irods; +________" + echo >&2 "-- status of create-db = $? -- " + fi + ;; + + 4) + IRODS_TO_INSTALL=$(irods_package_vsn) + sudo apt install -y irods-{dev,runtime}${IRODS_TO_INSTALL:+"=$IRODS_TO_INSTALL"} + if [[ $with_opts != *\ basic\ * ]]; then + sudo apt install -y irods-{icommands,server,database-plugin-postgres}${IRODS_TO_INSTALL:+"=$IRODS_TO_INSTALL"} + fi + ;; + + 5) + if [ ! $(irods_vsn) '<' "4.3" ]; then + PYTHON=python3 + else + PYTHON=python2 + fi + sudo $PYTHON /var/lib/irods/scripts/setup_irods.py &2 "unrecognized phase: '$PHASE'." + QUIT=1 + ;; + esac + return $? +} + +#-------------------------- main + +QUIT=0 +while [ $# -gt 0 ]; do + ARG=$1 + shift + NOP="" + run_phase $ARG " $withopts " + sts=$? + [ $QUIT != 0 ] && break + [ -n "$NOP" ] && continue + echo -n "== $ARG == " + if [ $sts -eq 0 ]; then + echo Y >&2 + else + [ $quit_on_phase_err ] && { + echo >&2 "N - quitting" + exit 1 + } + echo N >&2 + fi +done diff --git a/test_harness/single_node/install_python_rule_engine b/test_harness/single_node/install_python_rule_engine new file mode 100755 index 000000000..c3b35598b --- /dev/null +++ b/test_harness/single_node/install_python_rule_engine @@ -0,0 +1,34 @@ +#!/bin/bash + +python_rule_plugin_package_spec() { + local PKG=irods-rule-engine-plugin-python + + # Grep pattern to filter plugin package names for those compatible with the target version of iRODS. + local search_str="+$IRODS_PACKAGE_VERSION" + if [ "$1" = "all" ]; then + search_str="" + fi + + # Collect available version strings for the plugin for our iRODS server version. + local VERSIONS=$( + apt list -a $PKG 2>/dev/null | + awk '{print $2}' | + grep '\w' | + grep "$( + perl -e 'print quotemeta($ARGV[0])' "$search_str" + )" + ) + + # Get the highest version number among the list in $VERSIONS + local LATEST_VERSION=$(sort -V <<<"$VERSIONS" | tail -1) + + if [ "$1" = "latest" ]; then + # Output the appropriate format for apt install targets + echo "$PKG=$LATEST_VERSION" + else + echo "$VERSIONS" + fi +} + +# Download/install the latest Python Rule Engine plugin appropriate for the iRODS server we have installed. +apt install -y "$(python_rule_plugin_package_spec latest)" diff --git a/test_harness/single_node/irods_version_greater_or_equal_to b/test_harness/single_node/irods_version_greater_or_equal_to new file mode 100755 index 000000000..c485c0764 --- /dev/null +++ b/test_harness/single_node/irods_version_greater_or_equal_to @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import getopt +import glob +import json +import os +import sys + +def version_to_tuple(dot_str): + return tuple(int(_) for _ in dot_str.split('.')) + +# Fail unless iRODS version is greater than or equal to a version string given on the command line. +# With -e, we determine the installed iRODS version from the specified environment variable. +# Else use ~irods/version.json + +if __name__ == '__main__': + opts,arg = getopt.getopt (sys.argv[1:],'e:',['use_env_var=']) + + env_var = '' + + for opt,val in opts: + if opt in {'-e','--use_env_var'}: + env_var = val + + pattern = os.path.join(os.path.expanduser('~irods'), "*.json") + + # get full path of version.json + + if env_var: + version_to_test = os.environ[env_var] + else: + version_files = list( + filter( + (lambda name: name.lower().endswith('version.json')), + glob.glob(pattern) + ) + ) + # Load JSON struct containing iRODS version. + j = json.load(open(version_files[0])) + version_to_test = j["irods_version"] + + if version_to_tuple(version_to_test) < version_to_tuple(arg[0]): + exit(1) diff --git a/test_harness/single_node/manage_irods5_procs b/test_harness/single_node/manage_irods5_procs new file mode 100755 index 000000000..e49943551 --- /dev/null +++ b/test_harness/single_node/manage_irods5_procs @@ -0,0 +1,59 @@ +#!/bin/bash + +if [ $(id -un) = irods ]; then + LAUNCH='bash -c' +else + LAUNCH='sudo su - irods -c' +fi + +STDOUT="" +PID="" + +start() { + if [ -z "$STDOUT" ]; then + $LAUNCH 'irodsServer -d -p /tmp/irods.pid' + else + $LAUNCH 'irodsServer --stdout -p /tmp/irods.pid >/tmp/irods.log &' + fi +} + +rm_pid_file() { + if [ -z "$PID" ]; then + PID=$($LAUNCH 'cat /tmp/irods.pid') + fi + $LAUNCH 'rm -f /tmp/irods.pid >/dev/null 2>&1' +} + +stop() { + $LAUNCH 'kill -QUIT $(cat /tmp/irods.pid)' + rm_pid_file +} + +wait() { + $LAUNCH " + [ -z '$PID' ] && { echo >&2 'nothing to wait for.' ; exit 2; } + while ps -eo pid |grep $PID >/dev/null 2>&1; do sleep 1; done;" +} + +# ----------------------------------- +while [ -n "$1" ]; do + if [ "$1" = "stdout" ]; then + STDOUT=1 + elif [ "$1" = "start" ]; then + start + elif [ "$1" = "rescan-config" ]; then + $LAUNCH 'pkill -HUP irodsServer' + elif [ "$1" = "status" ]; then + pgrep -afl "irods(Delay|Agent|Server)" + elif [ "$1" = "stop" ]; then + stop + elif [ "$1" = "restart" ]; then + stop && start + elif [ "$1" = "wait" ]; then + wait + else + echo >&2 "usage: $0 [start|status|stop]" + exit 2 + fi + shift +done diff --git a/test_harness/single_node/most_recent_python.sh b/test_harness/single_node/most_recent_python.sh new file mode 100755 index 000000000..91448d293 --- /dev/null +++ b/test_harness/single_node/most_recent_python.sh @@ -0,0 +1,25 @@ +#!/bin/bash +usage() { + echo >&2 "Usage: + $0 major.minor" + echo >&2 "Output: + prints full latest python version inclusive of the patch level." + exit 2 +} +MAJOR_MINOR=$1 +if [ -z "${MAJOR_MINOR}" ]; then # allow blank specification: most recent overall + MAJOR_MINOR='[0-9]\+\.[0-9]\+' +elif [[ $MAJOR_MINOR =~ ^[0-9]+$ ]]; then # allow single integer, eg. 3 for most recent 3.y.z + MAJOR_MINOR+='\.[0-9]\+' +elif [[ $MAJOR_MINOR =~ [0-9]+\.[0-9]+ ]]; then # allow x.y form, will yield output of most recent x.y.z + MAJOR_MINOR=$(sed 's/\./\\./' <<<"${MAJOR_MINOR}") # insert backslash in front of "." +elif ! [[ $MAJOR_MINOR =~ [0-9]+\\?.[0-9]+ ]]; then + usage +fi + +url='https://www.python.org/ftp/python/' + +# Fetch the directory listing, extract version numbers, sort them to find the largest numerically. +curl --silent "$url" | + sed -n 's!.*href="\('"${MAJOR_MINOR}"'\.[0-9]\+\)/".*!\1!p' | sort -rV | + head -n 1 diff --git a/test_harness/single_node/print_repo_root_location b/test_harness/single_node/print_repo_root_location new file mode 100755 index 000000000..7dd2dd869 --- /dev/null +++ b/test_harness/single_node/print_repo_root_location @@ -0,0 +1,5 @@ +#!/bin/bash +# The following line needs be kept updated to reflect true position relative to repository root, +# in the event this script or any of its chain of containing directories (up to but not including the repo root) are moved. +REPO_ROOT_RELATIVE_TO_THIS_SCRIPT=../.. +realpath "$(dirname "$0")/$REPO_ROOT_RELATIVE_TO_THIS_SCRIPT" diff --git a/test_harness/single_node/setup_python_rule_engine b/test_harness/single_node/setup_python_rule_engine new file mode 100755 index 000000000..ad2fa5ad0 --- /dev/null +++ b/test_harness/single_node/setup_python_rule_engine @@ -0,0 +1,93 @@ +#!/bin/bash + +# This script should be run as the service account user. + +wait="" +if [ $1 = --wait ]; then + wait=1 + shift +fi + +DIR=$(dirname "$0") + +server_ctl() { + # This script takes one argument. + # Valid arguments: start, stop, or restart. The appropriate action is then taken for the resident iRODS server. + if "$DIR"/irods_version_greater_or_equal_to --use_env_var=IRODS_PACKAGE_VERSION 5.0; then + # Make our ps-based script wait for process shutdown like 'irodsctl stop' does. + W="" + if [ "$1" = stop ]; then + W=wait + fi + "$IRODS_CONTROL_PATH"/manage_irods5_procs $1 $W + else + ~/irodsctl $1 + fi +} + +jq_process_in_place() { + local filename=$1 + shift + local basenm=$(basename "$filename") + local tempname=/tmp/.$$.$basenm + + jq "$@" <"$filename" >"$tempname" && + cp "$tempname" "$filename" + STATUS=$? + rm -f "$tempname" + [ $STATUS = 0 ] || echo "**** jq process error" >&2 +} + +# -- Main part of script -- + +server_ctl stop + +jq_process_in_place /etc/irods/server_config.json \ + '.plugin_configuration.rule_engines[1:1]=[ { "instance_name": "irods_rule_engine_plugin-python-instance", + "plugin_name": "irods_rule_engine_plugin-python", + "plugin_specific_configuration": {} + } + ]' +echo ' +defined_in_both { + writeLine("stdout", "native rule") +} + +generic_failing_rule { + fail +} + +failing_with_message { + failmsg(-2, "error with code of minus 2") +} + +' >>/etc/irods/core.re + +echo ' +def defined_in_both(rule_args,callback,rei): + callback.writeLine("stdout", "python rule") + +def generic_failing_rule(*_): + raise RuntimeError + +def failing_with_message_py(rule_args,callback,rei): + callback.failing_with_message() + +' >/etc/irods/core.py + +server_ctl start + +# Wait until 'irule -a' shows Python Rule Engine Plugin among the choices +if [ -n "$wait" ]; then + times=0 + OUTFILE=/tmp/irule_output.stderr + while :; do + irule -a 2>/dev/null | grep irods_rule_engine_plugin-python-instance >/dev/null + [ ${PIPESTATUS[1]} -eq 0 ] && break + sleep 1 + if [ $((++times)) -ge 10 ]; then + echo >&2 "Failed to configure Python rule engine." + exit 2 + fi + done +fi diff --git a/test_harness/single_node/start_postgresql_and_irods.sh b/test_harness/single_node/start_postgresql_and_irods.sh new file mode 100755 index 000000000..fda260cc9 --- /dev/null +++ b/test_harness/single_node/start_postgresql_and_irods.sh @@ -0,0 +1,40 @@ +#!/bin/bash +service postgresql start + +# This section is equivalent to, and might be replaced by, pg_isready. +x=${DB_WAIT_SEC:-20} +while [ $x -ge 0 ] && { ! $SUDO su - postgres -c "psql -c '\l' >/dev/null 2>&1" || x=""; }; do + [ -z "$x" ] && break + echo >&2 "$((x--)) secs til database timeout" + sleep 1 +done +[ -z "$x" ] || { + echo >&2 "Error -- database didn't start" + exit 1 +} + +VERSION_file=$(ls /var/lib/irods/{VERSION,version}.json.dist 2>/dev/null) +if ! id -u irods >/dev/null 2>&1; then + /install.sh --w=create-db 0 + /install.sh 5 +fi +IRODS_VSN=$(jq -r '.irods_version' $VERSION_file) +IRODS_VSN_MAJOR=${IRODS_VSN//.*/} +if [ "$IRODS_VSN_MAJOR" -lt 5 ]; then + su - irods -c '~/irodsctl restart' +else + "$IRODS_CONTROL_PATH"/manage_irods5_procs stdout start +fi +IRODS_WAIT_SEC=20 +x=$IRODS_WAIT_SEC +SLEEPTIME="" +while [ $((x--)) -gt 0 ]; do + sleep $((SLEEPTIME + 0)) + pgrep irodsServer + STATUS=$? + [ $STATUS -eq 0 ] && break + SLEEPTIME=1 +done +echo "($STATUS)" >/tmp/irods_status +[ $STATUS -eq 0 ] || exit 125 +tail -f /dev/null diff --git a/test_harness/single_node/test_script_parameters b/test_harness/single_node/test_script_parameters new file mode 100644 index 000000000..4b94d58e6 --- /dev/null +++ b/test_harness/single_node/test_script_parameters @@ -0,0 +1,44 @@ +# keys for Arglist refer to argument given, which could be a symlink. + +declare -A wrapper_arglist=( + [login_auth_test_must_run_manually.py]="-v TestLogins" + [login_auth_test_1.py]="-v TestAnonymousUser TestMiscellaneous" + [login_auth_test_2.py]="-v TestWithSSL" +) + +# keys for Wrapper refer to argument after resolution of any symlinks + +declare -A wrappers=( + [login_auth_test_must_run_manually.py]=./login_auth_test.sh + [PRC_issue_362.bats]=./login_auth_test.sh + [test001_pam_password_expiration.bats]=../login_auth_test.sh + [test002_write_native_credentials_to_secrets_file.bats]=../login_auth_test.sh + [test003_write_pam_credentials_to_secrets_file.bats]=../login_auth_test.sh + [test004_prc_pam_password_internal_secrets_file_generation.bats]=../login_auth_test.sh + [test005_test_special_characters_in_pam_passwords.bats]=../login_auth_test.sh + [test006_connection_timeout_on_ssl_socket.bats]=../login_auth_test.sh + [test007_pam_features_in_new_auth_framework.bats]=../login_auth_test.sh + [test008_prc_write_irodsA_utility_in_native_mode.bats]=../login_auth_test.sh + [test009_test_special_characters_in_pam_passwords_auth_framework.bats]=../login_auth_test.sh + [test010_issue_362_rogue_chars_in_pam_password.bats]=../login_auth_test.sh +) + +# keys for Image and User refer to the basename after resolution to a wrapper if one is used + +declare -A images=( + [login_auth_test.sh]=compile-specific-python + [login_auth_test_must_run_manually.py]=compile-specific-python + [run_suite_locally.sh]=compile-specific-python +) + +declare -A user=( + [run_suite_locally.sh]=root +) + +# keys for WorkDir refer to user + +declare -A workdirs=( + [testuser]=/home/testuser + [irods]=/var/lib/irods + [root]=/ +) diff --git a/test_harness/utility/iinit.py b/test_harness/utility/iinit.py new file mode 100755 index 000000000..881556bc9 --- /dev/null +++ b/test_harness/utility/iinit.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +# This script creates the client environment to authenticate natively +# as the 'rods' admin on the Docker node running the Python client tests. +# Thus, we don't need irods-icommands to be installed on that node. + +from irods.password_obfuscation import encode +import json +import os +import sys +from os import chmod +from os.path import expanduser,exists,join + + +home_env_path = expanduser('~/.irods') +env_file_path = join(home_env_path,'irods_environment.json') +auth_file_path = join(home_env_path,'.irodsA') + + +def do_iinit(host, port, user, zone, password): + if not exists(home_env_path): + os.makedirs(home_env_path) + else: + raise RuntimeError('~/.irods already exists') + + with open(env_file_path,'w') as env_file: + json.dump ( { "irods_host": host, + "irods_port": int(port), + "irods_user_name": user, + "irods_zone_name": zone }, env_file, indent=4) + with open(auth_file_path,'w') as auth_file: + auth_file.write(encode(password)) + chmod (auth_file_path,0o600) + + +def get_kv_pairs_from_cmdline(*args): + arglist = list(args) + while arglist: + k = arglist.pop(0) + v = arglist.pop(0) + yield k,v + + +if __name__ == '__main__': + args = sys.argv[1:] + dct = {k:v for k,v in get_kv_pairs_from_cmdline(*args)} + do_iinit(**dct) From 9fdbb29ad4d76c9613fbb4863165f67f58cc2b95 Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Tue, 11 Nov 2025 12:07:14 -0500 Subject: [PATCH 3/6] [#781] modify access_time test with guaranteed passing assertions The test now fetches the modify and access timestamps during the interval in which the replica is open, then asserts those timestamps as equal to each other, as well as that they are greater than a system-generated datetime of about two seconds prior. (This is due to some timestamps including a microseconds count whereas others do not.) --- irods/test/data_obj_test.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/irods/test/data_obj_test.py b/irods/test/data_obj_test.py index b6ed95886..071771717 100644 --- a/irods/test/data_obj_test.py +++ b/irods/test/data_obj_test.py @@ -65,7 +65,7 @@ def is_localhost_synonym(name): from irods.manager import data_object_manager from irods.message import RErrorStack from irods.message import ET, XML_Parser_Type, default_XML_parser, current_XML_parser -from datetime import datetime +from datetime import datetime, timezone, timedelta from tempfile import NamedTemporaryFile, gettempdir, mktemp from irods.test.helpers import unique_name, my_function_name from irods.ticket import Ticket @@ -3305,16 +3305,20 @@ def test_access_time__issue_700(self): if self.sess.server_version < (5,): self.skipTest("iRODS servers < 5.0.0 do not provide an access_time attribute for data objects.") - data_path= iRODSPath(self.coll.path, - unique_name(my_function_name(), datetime.now()) - ) - with self.sess.data_objects.open(data_path,"w") as f: - f.write(b'_') - with self.sess.data_objects.open(data_path,"r") as f: - f.read() + prior_ts = datetime.now(timezone.utc) - timedelta(seconds=2) + + # Create a new, uniquely named test data object. + data = self.sess.data_objects.create( + logical_path:=f'{helpers.home_collection(self.sess)}/{unique_name(my_function_name(), datetime.now())}' + ) + + with data.open('w') as f: + data = self.sess.data_objects.get(logical_path) + self.assertEqual(data.access_time, data.modify_time) + self.assertGreaterEqual(data.access_time, prior_ts) - data = self.sess.data_objects.get(data_path) - self.assertGreaterEqual(data.access_time, data.modify_time) + # Test that access_time is there, and of the right type. + self.assertIs(type(data.access_time), datetime) if __name__ == "__main__": # let the tests find the parent irods lib From 45dfe924279fa94eea1667f96a16eb084359708c Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Thu, 13 Nov 2025 23:02:26 -0500 Subject: [PATCH 4/6] [#778] remove Jenkins test framework support --- Dockerfile.prc_test.centos | 28 ---------- Dockerfile.prc_test.ubuntu | 35 ------------- docker-compose.yml | 42 --------------- docker_build/Dockerfile.provider | 46 ----------------- docker_build/ICAT.sql | 3 -- docker_build/build_deps_list | 22 -------- docker_build/iinit.py | 51 ------------------- docker_build/install_python_rule_engine | 9 ---- docker_build/pgpass | 1 - docker_build/recv_oneshot | 35 ------------- docker_build/send_oneshot | 6 --- docker_build/setup_python_rule_engine | 48 ----------------- docker_build/wait_on_condition | 34 ------------- ...tium_continuous_integration_test_module.py | 26 ---------- run_python_tests.sh | 13 ----- 15 files changed, 399 deletions(-) delete mode 100644 Dockerfile.prc_test.centos delete mode 100644 Dockerfile.prc_test.ubuntu delete mode 100644 docker-compose.yml delete mode 100644 docker_build/Dockerfile.provider delete mode 100644 docker_build/ICAT.sql delete mode 100755 docker_build/build_deps_list delete mode 100644 docker_build/iinit.py delete mode 100755 docker_build/install_python_rule_engine delete mode 100644 docker_build/pgpass delete mode 100755 docker_build/recv_oneshot delete mode 100755 docker_build/send_oneshot delete mode 100755 docker_build/setup_python_rule_engine delete mode 100755 docker_build/wait_on_condition delete mode 100644 irods_consortium_continuous_integration_test_module.py delete mode 100644 run_python_tests.sh diff --git a/Dockerfile.prc_test.centos b/Dockerfile.prc_test.centos deleted file mode 100644 index c171fa62e..000000000 --- a/Dockerfile.prc_test.centos +++ /dev/null @@ -1,28 +0,0 @@ -ARG os_image -FROM ${os_image} -ARG log_output_dir=/tmp -ENV LOG_OUTPUT_DIR="$log_output_dir" -ARG py_N -ENV PY_N "$py_N" - -RUN yum install -y epel-release -RUN yum install -y git nmap-ncat sudo -RUN yum install -y python${py_N} python${py_N}-pip -RUN useradd -md /home/user -s /bin/bash user -RUN echo "user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers -WORKDIR /home/user -COPY ./ ./repo/ -RUN chown -R user repo/ -USER user -RUN pip${py_N} install --user --upgrade pip==20.3.4 # - limit pip version for C7 system python2.7 -RUN cd repo && python${py_N} -m pip install --user '.[tests]' -RUN python${py_N} repo/docker_build/iinit.py \ - host irods-provider \ - port 1247 \ - user rods \ - zone tempZone \ - password rods -SHELL ["/bin/bash","-c"] -CMD echo "Waiting on iRODS server... " ; \ - sudo groupadd -o -g $(stat -c%g /irods_shared) irods && sudo usermod -aG irods user && \ - newgrp irods < repo/run_python_tests.sh diff --git a/Dockerfile.prc_test.ubuntu b/Dockerfile.prc_test.ubuntu deleted file mode 100644 index 79ed07e12..000000000 --- a/Dockerfile.prc_test.ubuntu +++ /dev/null @@ -1,35 +0,0 @@ -ARG os_image -FROM ${os_image} -ARG log_output_dir=/tmp -ENV LOG_OUTPUT_DIR="$log_output_dir" -ARG py_N -ENV PY_N "$py_N" - -RUN apt update -RUN apt install -y git netcat-openbsd sudo -RUN apt install -y python${py_N} python${py_N}-pip -RUN useradd -md /home/user -s /bin/bash user -RUN echo "user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers -WORKDIR /home/user -COPY ./ ./repo/ -RUN chown -R user repo/ -USER user -RUN pip${py_N} install --user --upgrade pip==20.3.4 # -- version specified for Ub16 -RUN cd repo && python${py_N} -m pip install --user '.[tests]' -RUN python${py_N} repo/docker_build/iinit.py \ - host irods-provider \ - port 1247 \ - user rods \ - zone tempZone \ - password rods - -SHELL ["/bin/bash","-c"] - -# -- At runtime: -- -# 1. wait for provider to run. -# 2. give user group permissions to access shared irods directories -# 3. run python tests as the new group - -CMD echo "Waiting on iRODS server... " ; \ - sudo groupadd -o -g $(stat -c%g /irods_shared) irods && sudo usermod -aG irods user && \ - newgrp irods < repo/run_python_tests.sh diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 5bbe1e1b3..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,42 +0,0 @@ -version: '3' -services: - - icat: - image: postgres:10 - environment: - - POSTGRES_HOST_AUTH_METHOD=md5 - - POSTGRES_PASSWORD=pg_password - - irods-provider: - environment: - - PYTHON_RULE_ENGINE_INSTALLED=${python_rule_engine_installed} - hostname: irods-provider - build: - context: docker_build - dockerfile: Dockerfile.provider - args: - server_py: "${server_python_version}" - volumes: - - "${irods_pkg_dir}:/irods_packages:ro" - - ./irods_shared:/irods_shared:rw - depends_on: - - icat - networks: - default: - aliases: - - irods-provider - - client-runner: - env_file: client-runner.env - environment: - - PYTHON_RULE_ENGINE_INSTALLED=${python_rule_engine_installed} - volumes: - - ./irods_shared:/irods_shared:rw - build: - context: . - dockerfile: Dockerfile.prc_test.${client_os_generic} - args: - os_image: "$client_os_image" - py_N: "$client_python_version" - depends_on: - - irods-provider diff --git a/docker_build/Dockerfile.provider b/docker_build/Dockerfile.provider deleted file mode 100644 index 812863f1e..000000000 --- a/docker_build/Dockerfile.provider +++ /dev/null @@ -1,46 +0,0 @@ -FROM ubuntu:18.04 - -ARG irods_pkg_dir -ARG server_py=${server_python_version} -ENV SERVER_PY "${server_py}" - -RUN apt update -RUN apt install -y wget sudo lsb-release apt-transport-https gnupg2 postgresql-client python3 -RUN wget -qO - https://packages.irods.org/irods-signing-key.asc | sudo apt-key add - -RUN echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/renci-irods.list -RUN apt update - -SHELL [ "/bin/bash","-c" ] - -COPY ICAT.sql /tmp -COPY pgpass root/.pgpass -RUN chmod 600 root/.pgpass - -RUN apt install -y rsyslog gawk -RUN apt install -y jq -ADD build_deps_list wait_on_condition send_oneshot install_python_rule_engine setup_python_rule_engine /tmp/ - -# At Runtime: 1. Install apt dependencies for the iRODS package files given. -# 2. Install the package files. -# 3. Wait on database container. -# 4. Configure iRODS provider and make sure it is running. -# 5. Open a server port, informing the client to start tests now that iRODS is up. -# 6. Configure shared folder for tests that need to register data objects. -# (We opt out if /irods_shared does not exist, ie is omitted in the docker-compose.yml). -# 7. Wait forever. - -CMD apt install -y $(/tmp/build_deps_list /irods_packages/irods*{serv,dev,icommand,runtime,database-*postgres}*.deb) && \ - dpkg -i /irods_packages/irods*{serv,dev,icommand,runtime,database-*postgres}*.deb && \ - /tmp/wait_on_condition -i 5 -n 12 "psql -h icat -U postgres -c '\\l' >/dev/null" && \ - psql -h icat -U postgres -f /tmp/ICAT.sql && \ - sed 's/localhost/icat/' < /var/lib/irods/packaging/localhost_setup_postgres.input \ - | python${SERVER_PY} /var/lib/irods/scripts/setup_irods.py && \ - { [ "${PYTHON_RULE_ENGINE_INSTALLED}" = '' ] || { /tmp/install_python_rule_engine "$PYTHON_RULE_ENGINE_INSTALLED" /irods_packages \ - && /tmp/setup_python_rule_engine; } } && \ - { pgrep -u irods irodsServer >/dev/null || su irods -c '~/irodsctl start'; \ - env PORT=8888 /tmp/send_oneshot "iRODS is running..." & } && \ - { [ ! -d /irods_shared ] || { mkdir -p /irods_shared/reg_resc && mkdir -p /irods_shared/tmp && \ - chown -R irods.irods /irods_shared && chmod g+ws /irods_shared/tmp && \ - chmod 777 /irods_shared/reg_resc ; } } && \ - echo $'*********\n' $'*********\n' $'*********\n' $'*********\n' $'*********\n' IRODS IS UP && \ - tail -f /dev/null diff --git a/docker_build/ICAT.sql b/docker_build/ICAT.sql deleted file mode 100644 index abb706ade..000000000 --- a/docker_build/ICAT.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE USER irods WITH PASSWORD 'testpassword'; -CREATE DATABASE "ICAT"; -GRANT ALL PRIVILEGES ON DATABASE "ICAT" TO irods; diff --git a/docker_build/build_deps_list b/docker_build/build_deps_list deleted file mode 100755 index 7bf3798a7..000000000 --- a/docker_build/build_deps_list +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -build_deps_list() -{ - local -A pkglist - local pkg - while [ $# -gt 0 ] - do - while read f - do - if [[ ! $f =~ \(.*\)\s*$ ]]; then # todo: include version-specific ? - pkglist["$f"]="" - fi - done < <(dpkg -I "$1"|grep -i '^ *depends:'|tr ',:' \\n | tail -n +2) - shift - done - for pkg in "${!pkglist[@]}" # package list de-duped by associative array - do - echo "$pkg" - done -} -build_deps_list "$@" diff --git a/docker_build/iinit.py b/docker_build/iinit.py deleted file mode 100644 index c63ff619a..000000000 --- a/docker_build/iinit.py +++ /dev/null @@ -1,51 +0,0 @@ -from getpass import getpass -from irods.password_obfuscation import encode -import json -import os -import sys -from os import chmod -from os.path import expanduser, exists, join -from getopt import getopt - - -home_env_path = expanduser("~/.irods") -env_file_path = join(home_env_path, "irods_environment.json") -auth_file_path = join(home_env_path, ".irodsA") - - -def do_iinit(host, port, user, zone, password): - if not exists(home_env_path): - os.makedirs(home_env_path) - else: - raise RuntimeError("~/.irods already exists") - - with open(env_file_path, "w") as env_file: - json.dump( - { - "irods_host": host, - "irods_port": int(port), - "irods_user_name": user, - "irods_zone_name": zone, - }, - env_file, - indent=4, - ) - with open(auth_file_path, "w") as auth_file: - auth_file.write(encode(password)) - chmod(auth_file_path, 0o600) - - -def get_kv_pairs_from_cmdline(*args): - arglist = list(args) - while arglist: - k = arglist.pop(0) - v = arglist.pop(0) - yield k, v - - -if __name__ == "__main__": - import sys - - args = sys.argv[1:] - dct = {k: v for k, v in get_kv_pairs_from_cmdline(*args)} - do_iinit(**dct) diff --git a/docker_build/install_python_rule_engine b/docker_build/install_python_rule_engine deleted file mode 100755 index a308c6405..000000000 --- a/docker_build/install_python_rule_engine +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# usage $0 [""|"y"|"/"*] [container_irods_packages_path] -if [[ $1 = /* ]]; then - apt install -y "$2"/irods*rule*python*.deb -elif [ "$1" != "" ]; then - apt install -y irods-rule.\*python -else - : # nop -fi diff --git a/docker_build/pgpass b/docker_build/pgpass deleted file mode 100644 index 55a6bdfd4..000000000 --- a/docker_build/pgpass +++ /dev/null @@ -1 +0,0 @@ -icat:5432:postgres:postgres:pg_password diff --git a/docker_build/recv_oneshot b/docker_build/recv_oneshot deleted file mode 100755 index 47e2bdd09..000000000 --- a/docker_build/recv_oneshot +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function -import sys, os, time -from socket import * -import getopt - -def try_connect(host,port): - try: - s=socket(AF_INET,SOCK_STREAM) - s.connect((host,port)) - return s - except: - s.close() - return None - -# Options: -# -# -t timeout -# -h host -# -p port - -t = now = time.time() -opts = dict(getopt.getopt(sys.argv[1:],'t:h:p:')[0]) - -host = opts['-h'] -port = int(opts['-p']) -timeout = float(opts['-t']) - -while time.time() < now + timeout: - time.sleep(1) - s = try_connect(host, port) - if s: - print(s.recv(32767).decode('utf-8'),end='') - exit(0) -exit(1) diff --git a/docker_build/send_oneshot b/docker_build/send_oneshot deleted file mode 100755 index b265af15d..000000000 --- a/docker_build/send_oneshot +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/gawk -f -BEGIN { - SERVER = "/inet/tcp/"ENVIRON["PORT"]"/0/0" - print ARGV[1] " - " strftime() |& SERVER - close(SERVER) -} diff --git a/docker_build/setup_python_rule_engine b/docker_build/setup_python_rule_engine deleted file mode 100755 index eaa56113b..000000000 --- a/docker_build/setup_python_rule_engine +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -jq_process_in_place() { - local filename=$1 - shift - local basenm=$(basename "$filename") - local tempname=/tmp/.$$.$basenm - - jq "$@" <"$filename" >"$tempname" && \ - cp "$tempname" "$filename" - STATUS=$? - rm -f "$tempname" - [ $STATUS = 0 ] || echo "**** jq process error" >&2 -} - -jq_process_in_place /etc/irods/server_config.json \ - '.plugin_configuration.rule_engines[1:1]=[ { "instance_name": "irods_rule_engine_plugin-python-instance", - "plugin_name": "irods_rule_engine_plugin-python", - "plugin_specific_configuration": {} - } - ]' - -echo ' -defined_in_both { - writeLine("stdout", "native rule") -} - -generic_failing_rule { - fail -} - -failing_with_message { - failmsg(-2, "error with code of minus 2") -} - -' >> /etc/irods/core.re - -echo ' -def defined_in_both(rule_args,callback,rei): - callback.writeLine("stdout", "python rule") - -def generic_failing_rule(*_): - raise RuntimeError - -def failing_with_message_py(rule_args,callback,rei): - callback.failing_with_message() - -' > /etc/irods/core.py diff --git a/docker_build/wait_on_condition b/docker_build/wait_on_condition deleted file mode 100755 index ce2c29b3e..000000000 --- a/docker_build/wait_on_condition +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# wait for a program to run with 0 return status - -interval=3; ntimes=20; verbose="" - -usage() { - echo "$0 [options] " - printf "\t options are: -i (default %d)\n" $interval - printf "\t -n (default %d)\n" $ntimes - printf "\t -v : for verbose reporting\n" - exit 1 -} >&2 - -while [[ "$1" = -* ]] ; do - case $1 in - -i) shift; interval=$1; shift ;; - -n) shift; ntimes=$1; shift ;; - -v) verbose=1 ; shift;; - *) usage;; - esac -done -[ $# -eq 0 ] && usage - -n=1 -while : ; do - eval "$@" - STATUS=$? - [ -n "$verbose" ] && echo "$n:" 'STATUS =' $STATUS `date` - [ $((++n)) -gt $ntimes -o $STATUS -eq 0 ] && break - sleep $interval -done - -exit $STATUS diff --git a/irods_consortium_continuous_integration_test_module.py b/irods_consortium_continuous_integration_test_module.py deleted file mode 100644 index 8355b4be0..000000000 --- a/irods_consortium_continuous_integration_test_module.py +++ /dev/null @@ -1,26 +0,0 @@ -import json -import sys - - -def run(CI): - - final_config = CI.store_config( - { - "yaml_substitutions": { # -> written to ".env" - "client_python_version": "3", - "client_os_generic": "ubuntu", - "client_os_image": "ubuntu:18.04", - "python_rule_engine_installed": "y", - }, - "container_environments": { - "client-runner": { # -> written to "client-runner.env" - "TESTS_TO_RUN": "" # run test subset, e.g. "irods.test.data_obj_test" - } - }, - } - ) - - print("----------\nconfig after CI modify pass\n----------", file=sys.stderr) - print(json.dumps(final_config, indent=4), file=sys.stderr) - - return CI.run_and_wait_on_client_exit() diff --git a/run_python_tests.sh b/run_python_tests.sh deleted file mode 100644 index 5ec220737..000000000 --- a/run_python_tests.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -o pipefail -cd repo/irods/test - -export PYTHONUNBUFFERED="Y" - -if [ -z "${TESTS_TO_RUN}" ] ; then - python"${PY_N}" runner.py 2>&1 | tee "${LOG_OUTPUT_DIR}"/prc_test_logs.txt -else - python"${PY_N}" -m unittest -v ${TESTS_TO_RUN} 2>&1 | tee "${LOG_OUTPUT_DIR}"/prc_test_logs.txt -fi - From 89375e93e8a0a6edba3ee2a8b4e9befa150f9d87 Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Mon, 17 Nov 2025 11:24:34 -0500 Subject: [PATCH 5/6] [#3] rename setupssl.py to setup_ssl.py for consistency --- irods/test/{setupssl.py => setup_ssl.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename irods/test/{setupssl.py => setup_ssl.py} (100%) diff --git a/irods/test/setupssl.py b/irods/test/setup_ssl.py similarity index 100% rename from irods/test/setupssl.py rename to irods/test/setup_ssl.py From 14a3895d1d26c6953fba251dece7074025df350f Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Fri, 21 Nov 2025 10:30:39 -0500 Subject: [PATCH 6/6] [#783] add pyproject.toml --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..fed528d4a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta"