From a63532169409d61ecf4aa324700956225d1e7437 Mon Sep 17 00:00:00 2001 From: Keith James Date: Thu, 22 Aug 2024 11:37:33 +0100 Subject: [PATCH 1/2] Add simplified Docker Compose support for tests Remove the requirement to set up iRODS client wrappers before tests will run. Support both plain CLI and IDEs. --- Dockerfile.dev | 61 ++++++++++++++++++++++ README.md | 80 ++++++----------------------- docker-compose.yml | 38 ++++++++------ docker/entrypoint.sh | 12 +++++ docker/install_pyenv.sh | 15 ++++++ pyproject.toml | 2 +- pytest.ini | 4 +- src/partisan/icommands.py | 50 +++++++++++++++++- tests/.irods/irods_environment.json | 1 + tests/__init__.py | 24 +++++++++ tests/conftest.py | 3 +- tests/test_irods.py | 6 +-- 12 files changed, 208 insertions(+), 88 deletions(-) create mode 100644 Dockerfile.dev create mode 100755 docker/entrypoint.sh create mode 100755 docker/install_pyenv.sh diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 0000000..31edb01 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,61 @@ +FROM ghcr.io/wtsi-npg/ub-18.04-baton-irods-4.2.11:latest + +ARG PYTHON_VERSION=3.12 + +ENV DEBIAN_FRONTEND=noninteractive + +USER root + +RUN apt-get update && \ + apt-get install -q -y --no-install-recommends \ + apt-transport-https \ + apt-utils \ + build-essential \ + ca-certificates \ + curl \ + gcc \ + git \ + make \ + libbz2-dev \ + libncurses-dev \ + libreadline-dev \ + libssl-dev \ + zlib1g-dev + +# Install the iRODS icommands package because it's useful for interactions with \ +# the server during development +RUN echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" |\ + tee /etc/apt/sources.list.d/renci-irods.list && \ + apt-get update && \ + apt-get install -q -y --no-install-recommends \ + irods-icommands="4.2.11-1~$(lsb_release -sc)" + +WORKDIR /app + +COPY . /app + +# It's more practical to build from an iRODS client image and install recent Python +# than to build from a recent Python image and install iRODS clients. +ENV PYENV_ROOT="/app/.pyenv" + +# Put PYENV first to ensure we use the pyenv-installed Python +ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin" + +RUN ./docker/install_pyenv.sh + +RUN pyenv install "$PYTHON_VERSION" +RUN pyenv global "$PYTHON_VERSION" + +RUN pip install --no-cache-dir -r requirements.txt && \ + pip install --no-cache-dir -r test-requirements.txt && \ + pip install --no-cache-dir . && \ + git status && \ + ls -al + +RUN chown -R appuser:appuser /app + +USER appuser + +ENTRYPOINT ["/app/docker/entrypoint.sh"] + +CMD ["/bin/bash"] diff --git a/README.md b/README.md index 155cc06..523bd80 100644 --- a/README.md +++ b/README.md @@ -272,77 +272,29 @@ distinguish between these circumstances. These tools should be present on the `PATH`, when required. -## Testing with Docker +## Running tests -An iRODS server and clients are available as Docker images which may be used -with Docker Compose to set up a standard test environment. The test -environment consists of an `irods-server` container and an `irods-clients` -container. +### Running directly on your local machine -Before running the tests, start the containers and supporting network: +To run the tests locally, you will need to have the `irods` clients installed (`icommands` +and `baton`, which means your local machine must be either be running Linux, or have +containerised versions of these tools installed and runnable via proxy wrappers of the +same name, to emulate the Linux environment. -```commandline - docker-compose up -d -``` +You will also need to have a working iRODS server to connect to. -The environment variables `IRODS_VERSION` (defaults to `4.2.11`) and -`DOCKER_TAG` (defaults to `latest`) may be used to choose particular -Docker images. +With this in place, you can run the tests with the following command: + pytest --it -```commandline - IRODS_VERSION="4.2.11" DOCKER_TAG="latest" docker-compose up -d -``` +### Running in a container -The `./tests/bin` directory contains a universal iRODS proxy script to be used -instead of native iRODS clients. It forwards any client operations to the -real iRODS clients inside the `irods-clients` container. This directory -should be on your `PATH` while running the tests. The iRODS authentication -file can then be created using `iinit`: +The tests can be run in a container, which is requires less setup and will be less likely +to be affected by your local environment. A Docker Composer file is provided to run the +tests in a Linux container, against a containerised iRODS server. -```commandline - export PATH="${PWD}/tests/bin:$PATH" - iinit -``` +To run the tests in a container, you will need to have Docker installed. -The tests should be run in the root of the repository, with `tmp` redirected -to a destination in a shared volume: +With this in place, you can run the tests with the following command: -```commandline - pytest --basetemp=./tests/tmp -``` - -Finally, to destroy the test containers and network: - -````commandline - docker-compose down -```` - - -### Test troubleshooting - -When starting the containers, you may see an error similar to: - -``` - invalid interpolation format for services.irods-clients.environment.CLIENT_USER_ID: - "required variable UID is missing a value: \nERROR: The UID environment - variable is unset". You may need to escape any $ with another $ -``` - -which is caused by the `UID` shell variable being unset or not exported. See -this [Docker Compose issue](https://github.com/docker/compose/issues/2380) for -more details. - - -You can work around this by exporting the relevant variable(s): - -```commandline - export UID - docker-compose up -d -``` - -or: - -```commandline - UID=$(id -u) docker-compose up -d -``` + docker-compose run app pytest --it diff --git a/docker-compose.yml b/docker-compose.yml index 0acb53d..391445c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,23 +1,31 @@ + services: irods-server: + platform: linux/amd64 container_name: irods-server - image: "ghcr.io/wtsi-npg/ub-18.04-irods-${IRODS_VERSION:-4.2.11}:${DOCKER_TAG:-latest}" - restart: always + image: "ghcr.io/wtsi-npg/ub-16.04-irods-4.2.7:latest" ports: - - "1247:1247" - - "20000-20199:20000-20199" + - "127.0.0.1:1247:1247" + - "127.0.0.1:20000-20199:20000-20199" + restart: always + healthcheck: + test: ["CMD", "nc", "-z", "-v", "localhost", "1247"] + start_period: 30s + interval: 5s + timeout: 10s + retries: 12 - irods-clients: - container_name: irods-clients - image: "ghcr.io/wtsi-npg/ub-18.04-irods-clients-${IRODS_VERSION:-4.2.11}:${DOCKER_TAG:-latest}" + app: + platform: linux/amd64 + build: + context: . + dockerfile: Dockerfile.dev + restart: always volumes: - - "${PWD}:${PWD}" - - "${PWD}/tests/.irods:${HOME}/.irods/" + - "./tests/.irods:/home/appuser/.irods/" environment: - CLIENT_USER: "${USER:? ERROR: The USER environment variable is unset}" - CLIENT_USER_ID: "${UID:? ERROR: The UID environment variable is unset}" - CLIENT_USER_HOME: "${HOME}" - IRODS_ENVIRONMENT_FILE: "${HOME}/.irods/irods_environment.json" - command: sleep infinity + IRODS_ENVIRONMENT_FILE: "/home/appuser/.irods/irods_environment.json" + IRODS_PASSWORD: "irods" depends_on: - - irods-server + irods-server: + condition: service_healthy diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..00036c8 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +export PYENV_ROOT="/app/.pyenv" + +# Put PYENV first to ensure we use the pyenv-installed Python +export PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:/bin:/usr/bin:/usr/local/bin" +export PYTHONUNBUFFERED=1 +export PYTHONPATH="" + +exec "$@" diff --git a/docker/install_pyenv.sh b/docker/install_pyenv.sh new file mode 100755 index 0000000..42d160f --- /dev/null +++ b/docker/install_pyenv.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -ex + +PYENV_RELEASE_VERSION=${PYENV_RELEASE_VERSION:="2.4.1"} +export PYENV_GIT_TAG="v${PYENV_RELEASE_VERSION}" + +PYENV_ROOT=${PYENV_ROOT:-"$HOME/.pyenv"} +export PATH="$PYENV_ROOT/bin:$PATH" + +PYENV_SHA256="a1ad63c22842dce498b441551e2f83ede3e3b6ebb33f62013607bba424683191" +curl -sSL -O https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer +sha256sum ./pyenv-installer | grep "$PYENV_SHA256" +/bin/bash ./pyenv-installer +rm ./pyenv-installer diff --git a/pyproject.toml b/pyproject.toml index 3a5c1d7..1495a53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ homepage = "https://github.com/wtsi-npg/partisan" repository = "https://github.com/wtsi-npg/partisan.git" [build-system] -requires = ["setuptools>=41", "wheel", "setuptools-git-versioning<2"] +requires = ["setuptools>=41", "wheel", "setuptools-git-versioning>=2.0,<3"] [tool.setuptools] # Note: we are relying on setuptools' automatic package discovery, so no further diff --git a/pytest.ini b/pytest.ini index fe58a2d..ac6d3ff 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,5 +2,5 @@ pythonpath = src testpaths = tests python_functions = test_* -log_cli = True -log_cli_level = INFO +log_cli = False +log_cli_level = ERROR diff --git a/src/partisan/icommands.py b/src/partisan/icommands.py index 816338b..e2ff96b 100644 --- a/src/partisan/icommands.py +++ b/src/partisan/icommands.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright © 2020, 2021, 2022, 2023 Genome Research Ltd. All rights +# Copyright © 2020, 2021, 2022, 2023, 2024 Genome Research Ltd. All rights # reserved. # # This program is free software: you can redistribute it and/or modify @@ -18,9 +18,12 @@ # # @author Keith James +import json +import os +import shlex import subprocess from io import StringIO -from pathlib import PurePath +from pathlib import Path, PurePath from typing import List, Union from structlog import get_logger @@ -62,6 +65,49 @@ def imkdir(remote_path: Union[PurePath, str], make_parents=True): _run(cmd) +def iinit(): + password = os.environ.get("IRODS_PASSWORD") + if password is None or password == "": + log.info( + "Not authenticating with iRODS; no password specified by the " + "IRODS_PASSWORD environment variable. Assuming the user is already " + "authenticated." + ) + return + + env_val = os.environ.get("IRODS_ENVIRONMENT_FILE") + if env_val is None or env_val == "": + log.info( + "No iRODS environment file specified by the IRODS_ENVIRONMENT_FILE " + "environment variable; using the default" + ) + env_path = Path("~/.irods/irods_environment.json").expanduser().as_posix() + else: + env_path = Path(env_val).resolve().as_posix() + + log.info("Using iRODS environment file", env_path=env_path) + + with open(env_path) as f: + env = json.load(f) + if "irods_authentication_file" not in env: + log.info( + "No iRODS authentication file specified in the environment file; " + "using the default" + ) + auth_path = Path("~/.irods/.irodsA").expanduser().as_posix() + else: + auth_path = Path(env["irods_authentication_file"]).as_posix() + + if Path(auth_path).exists(): + log.info("Updating the existing iRODS auth file", auth_path=auth_path) + else: + log.info("Creating a new iRODS auth file", auth_path=auth_path) + + password = shlex.quote(password) + cmd = ["/bin/sh", "-c", f"echo {password} | iinit"] + _run(cmd) + + def iget( remote_path: Union[PurePath, str], local_path: Union[PurePath, str], diff --git a/tests/.irods/irods_environment.json b/tests/.irods/irods_environment.json index ce1799d..6727312 100644 --- a/tests/.irods/irods_environment.json +++ b/tests/.irods/irods_environment.json @@ -2,6 +2,7 @@ "irods_host": "irods-server", "irods_port": 1247, "irods_user_name": "irods", + "irods_authentication_file": "./tests/.irods/auth_file", "irods_zone_name": "testZone", "irods_home": "/testZone/home/irods", "irods_default_resource": "replResc", diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..d4357d5 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# +# Copyright © 2024 Genome Research Ltd. All rights reserved. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +from partisan.icommands import iinit + +# Ensure that the iRODS environment is initialised before running any tests. Calling +# this function will create or update a local iRODS auth file ready for use by the +# iRODS clients used in the tests. +iinit() diff --git a/tests/conftest.py b/tests/conftest.py index 0d91b11..40382c4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright © 2020, 2021, 2023 Genome Research Ltd. All rights reserved. +# Copyright © 2020, 2021, 2023, 2024 Genome Research Ltd. All rights reserved. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -31,6 +31,7 @@ from partisan.icommands import ( add_specific_sql, have_admin, + iinit, imkdir, iput, iquest, diff --git a/tests/test_irods.py b/tests/test_irods.py index 9c6cbca..c9d4370 100644 --- a/tests/test_irods.py +++ b/tests/test_irods.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright © 2020, 2021, 2023 Genome Research Ltd. All rights reserved. +# Copyright © 2020, 2021, 2023, 2024 Genome Research Ltd. All rights reserved. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ import hashlib import os.path -from datetime import datetime +from datetime import datetime, timezone from pathlib import PurePath import pytest @@ -996,7 +996,7 @@ def test_supersede_meta_data_object(self, simple_data_object): # Replace avu1, avu3 with avu4, avu5 (leaving avu2 in place) avu4 = AVU("abcde", "99999") avu5 = AVU("abcde", "00000") - date = datetime.utcnow() + date = datetime.now(timezone.utc) assert obj.supersede_metadata(avu4, avu5, history=True, history_date=date) == ( 2, 3, From 8483d360493a11b757d913206fa8988c4b853e8d Mon Sep 17 00:00:00 2001 From: Keith James Date: Thu, 22 Aug 2024 15:31:15 +0100 Subject: [PATCH 2/2] Add a note about time taken to build a new Doocker image --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 523bd80..0395a25 100644 --- a/README.md +++ b/README.md @@ -289,7 +289,7 @@ With this in place, you can run the tests with the following command: ### Running in a container -The tests can be run in a container, which is requires less setup and will be less likely +The tests can be run in a container, which requires less setup and will be less likely to be affected by your local environment. A Docker Composer file is provided to run the tests in a Linux container, against a containerised iRODS server. @@ -298,3 +298,8 @@ To run the tests in a container, you will need to have Docker installed. With this in place, you can run the tests with the following command: docker-compose run app pytest --it + +There will be a delay the first time this is run because the Docker image will be built. +To pre-build the image, you can run: + + docker-compose build