Merge pull request #210 from kjsanger/feature/tests-in-docker

Add simplified Docker Compose support for tests

Dockerfile.dev

@@ -0,0 +1,61 @@
+FROM ghcr.io/wtsi-npg/ub-18.04-baton-irods-4.2.11:latest
+
+ARG PYTHON_VERSION=3.12
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+USER root
+
+RUN apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    apt-transport-https \
+    apt-utils \
+    build-essential \
+    ca-certificates \
+    curl \
+    gcc \
+    git \
+    make \
+    libbz2-dev \
+    libncurses-dev \
+    libreadline-dev \
+    libssl-dev \
+    zlib1g-dev
+
+# Install the iRODS icommands package because it's useful for interactions with \
+# the server during development
+RUN echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" |\
+    tee /etc/apt/sources.list.d/renci-irods.list && \
+    apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    irods-icommands="4.2.11-1~$(lsb_release -sc)"
+
+WORKDIR /app
+
+COPY . /app
+
+# It's more practical to build from an iRODS client image and install recent Python
+# than to build from a recent Python image and install iRODS clients.
+ENV PYENV_ROOT="/app/.pyenv"
+
+# Put PYENV first to ensure we use the pyenv-installed Python
+ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
+
+RUN ./docker/install_pyenv.sh
+
+RUN pyenv install "$PYTHON_VERSION"
+RUN pyenv global "$PYTHON_VERSION"
+
+RUN pip install --no-cache-dir -r requirements.txt && \
+    pip install --no-cache-dir -r test-requirements.txt && \
+    pip install --no-cache-dir . && \
+    git status && \
+    ls -al
+
+RUN chown -R appuser:appuser /app
+
+USER appuser
+
+ENTRYPOINT ["/app/docker/entrypoint.sh"]
+
+CMD ["/bin/bash"]

README.md

@@ -272,77 +272,34 @@ distinguish between these circumstances.
 
 These tools should be present on the `PATH`, when required.
 
-## Testing with Docker
+## Running tests
 
-An iRODS server and clients are available as Docker images which may be used
-with Docker Compose to set up a standard test environment. The test 
-environment consists of an `irods-server` container and an `irods-clients` 
-container.
+### Running directly on your local machine
 
-Before running the tests, start the containers and supporting network:
+To run the tests locally, you will need to have the `irods` clients installed (`icommands`
+and `baton`, which means your local machine must be either be running Linux, or have
+containerised versions of these tools installed and runnable via proxy wrappers of the
+same name, to emulate the Linux environment.
 
-```commandline
-    docker-compose up -d
-```
+You will also need to have a working iRODS server to connect to.
 
-The environment variables `IRODS_VERSION` (defaults to `4.2.11`) and 
-`DOCKER_TAG` (defaults to `latest`) may be used to choose particular 
-Docker images.
+With this in place, you can run the tests with the following command:
 
+    pytest --it 
 
-```commandline
-    IRODS_VERSION="4.2.11" DOCKER_TAG="latest" docker-compose up -d
-```
+### Running in a container
 
-The `./tests/bin` directory contains a universal iRODS proxy script to be used
-instead of native iRODS clients. It forwards any client operations to the 
-real iRODS clients inside the `irods-clients` container. This directory 
-should be on your `PATH` while running the tests. The iRODS authentication 
-file can then be created using `iinit`: 
+The tests can be run in a container, which requires less setup and will be less likely
+to be affected by your local environment. A Docker Composer file is provided to run the
+tests in a Linux container, against a containerised iRODS server.
 
-```commandline
-    export PATH="${PWD}/tests/bin:$PATH"
-    iinit
-```
+To run the tests in a container, you will need to have Docker installed.
 
-The tests should be run in the root of the repository, with `tmp` redirected 
-to a destination in a shared volume:
+With this in place, you can run the tests with the following command:
 
-```commandline
-    pytest --basetemp=./tests/tmp
-```
+    docker-compose run app pytest --it
 
-Finally, to destroy the test containers and network:
+There will be a delay the first time this is run because the Docker image will be built.
+To pre-build the image, you can run:
 
-````commandline
-     docker-compose down
-````
-
-
-### Test troubleshooting
-
-When starting the containers, you may see an error similar to:
-
-```
-    invalid interpolation format for services.irods-clients.environment.CLIENT_USER_ID:
-    "required variable UID is missing a value:  \nERROR: The UID environment 
-    variable is unset". You may need to escape any $ with another $
-```
-
-which is caused by the `UID` shell variable being unset or not exported. See
-this [Docker Compose issue](https://github.com/docker/compose/issues/2380) for
-more details.
-
-
-You can work around this by exporting the relevant variable(s):
-
-```commandline
-    export UID
-    docker-compose up -d
-```
-
-or:
-
-```commandline
-    UID=$(id -u) docker-compose up -d
-```
+    docker-compose build

docker-compose.yml

@@ -1,23 +1,31 @@
+
 services:
   irods-server:
+    platform: linux/amd64
     container_name: irods-server
-    image: "ghcr.io/wtsi-npg/ub-18.04-irods-${IRODS_VERSION:-4.2.11}:${DOCKER_TAG:-latest}"
-    restart: always
+    image: "ghcr.io/wtsi-npg/ub-16.04-irods-4.2.7:latest"
     ports:
-      - "1247:1247"
-      - "20000-20199:20000-20199"
+      - "127.0.0.1:1247:1247"
+      - "127.0.0.1:20000-20199:20000-20199"
+    restart: always
+    healthcheck:
+      test: ["CMD", "nc", "-z", "-v", "localhost", "1247"]
+      start_period: 30s
+      interval: 5s
+      timeout: 10s
+      retries: 12
 
-  irods-clients:
-    container_name: irods-clients
-    image: "ghcr.io/wtsi-npg/ub-18.04-irods-clients-${IRODS_VERSION:-4.2.11}:${DOCKER_TAG:-latest}"
+  app:
+    platform: linux/amd64
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+    restart: always
     volumes:
-      - "${PWD}:${PWD}"
-      - "${PWD}/tests/.irods:${HOME}/.irods/"
+      - "./tests/.irods:/home/appuser/.irods/"
     environment:
-      CLIENT_USER: "${USER:? ERROR: The USER environment variable is unset}"
-      CLIENT_USER_ID: "${UID:? ERROR: The UID environment variable is unset}"
-      CLIENT_USER_HOME: "${HOME}"
-      IRODS_ENVIRONMENT_FILE: "${HOME}/.irods/irods_environment.json"
-    command: sleep infinity
+      IRODS_ENVIRONMENT_FILE: "/home/appuser/.irods/irods_environment.json"
+      IRODS_PASSWORD: "irods"
     depends_on:
-      - irods-server
+      irods-server:
+        condition: service_healthy

docker/entrypoint.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+export PYENV_ROOT="/app/.pyenv"
+
+# Put PYENV first to ensure we use the pyenv-installed Python
+export PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:/bin:/usr/bin:/usr/local/bin"
+export PYTHONUNBUFFERED=1
+export PYTHONPATH=""
+
+exec "$@"

docker/install_pyenv.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -ex
+
+PYENV_RELEASE_VERSION=${PYENV_RELEASE_VERSION:="2.4.1"}
+export PYENV_GIT_TAG="v${PYENV_RELEASE_VERSION}"
+
+PYENV_ROOT=${PYENV_ROOT:-"$HOME/.pyenv"}
+export PATH="$PYENV_ROOT/bin:$PATH"
+
+PYENV_SHA256="a1ad63c22842dce498b441551e2f83ede3e3b6ebb33f62013607bba424683191"
+curl -sSL -O https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer
+sha256sum ./pyenv-installer | grep "$PYENV_SHA256"
+/bin/bash ./pyenv-installer
+rm ./pyenv-installer

pyproject.toml

@@ -35,7 +35,7 @@ homepage = "https://github.com/wtsi-npg/partisan"
 repository = "https://github.com/wtsi-npg/partisan.git"
 
 [build-system]
-requires = ["setuptools>=41", "wheel", "setuptools-git-versioning<2"]
+requires = ["setuptools>=41", "wheel", "setuptools-git-versioning>=2.0,<3"]
 
 [tool.setuptools]
 # Note: we are relying on setuptools' automatic package discovery, so no further

pytest.ini

@@ -2,5 +2,5 @@
 pythonpath = src
 testpaths = tests
 python_functions = test_*
-log_cli = True
-log_cli_level = INFO
+log_cli = False
+log_cli_level = ERROR

src/partisan/icommands.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright © 2020, 2021, 2022, 2023 Genome Research Ltd. All rights
+# Copyright © 2020, 2021, 2022, 2023, 2024 Genome Research Ltd. All rights
 # reserved.
 #
 # This program is free software: you can redistribute it and/or modify
@@ -18,9 +18,12 @@
 #
 # @author Keith James <kdj@sanger.ac.uk>
 
+import json
+import os
+import shlex
 import subprocess
 from io import StringIO
-from pathlib import PurePath
+from pathlib import Path, PurePath
 from typing import List, Union
 
 from structlog import get_logger
@@ -62,6 +65,49 @@ def imkdir(remote_path: Union[PurePath, str], make_parents=True):
     _run(cmd)
 
 
+def iinit():
+    password = os.environ.get("IRODS_PASSWORD")
+    if password is None or password == "":
+        log.info(
+            "Not authenticating with iRODS; no password specified by the "
+            "IRODS_PASSWORD environment variable. Assuming the user is already "
+            "authenticated."
+        )
+        return
+
+    env_val = os.environ.get("IRODS_ENVIRONMENT_FILE")
+    if env_val is None or env_val == "":
+        log.info(
+            "No iRODS environment file specified by the IRODS_ENVIRONMENT_FILE "
+            "environment variable; using the default"
+        )
+        env_path = Path("~/.irods/irods_environment.json").expanduser().as_posix()
+    else:
+        env_path = Path(env_val).resolve().as_posix()
+
+    log.info("Using iRODS environment file", env_path=env_path)
+
+    with open(env_path) as f:
+        env = json.load(f)
+        if "irods_authentication_file" not in env:
+            log.info(
+                "No iRODS authentication file specified in the environment file; "
+                "using the default"
+            )
+            auth_path = Path("~/.irods/.irodsA").expanduser().as_posix()
+        else:
+            auth_path = Path(env["irods_authentication_file"]).as_posix()
+
+    if Path(auth_path).exists():
+        log.info("Updating the existing iRODS auth file", auth_path=auth_path)
+    else:
+        log.info("Creating a new iRODS auth file", auth_path=auth_path)
+
+    password = shlex.quote(password)
+    cmd = ["/bin/sh", "-c", f"echo {password} | iinit"]
+    _run(cmd)
+
+
 def iget(
     remote_path: Union[PurePath, str],
     local_path: Union[PurePath, str],

tests/.irods/irods_environment.json

@@ -2,6 +2,7 @@
     "irods_host": "irods-server",
     "irods_port": 1247,
     "irods_user_name": "irods",
+    "irods_authentication_file": "./tests/.irods/auth_file",
     "irods_zone_name": "testZone",
     "irods_home": "/testZone/home/irods",
     "irods_default_resource": "replResc",

tests/__init__.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2024 Genome Research Ltd. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from partisan.icommands import iinit
+
+# Ensure that the iRODS environment is initialised before running any tests. Calling
+# this function will create or update a local iRODS auth file ready for use by the
+# iRODS clients used in the tests.
+iinit()

tests/conftest.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright © 2020, 2021, 2023 Genome Research Ltd. All rights reserved.
+# Copyright © 2020, 2021, 2023, 2024 Genome Research Ltd. All rights reserved.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -31,6 +31,7 @@
 from partisan.icommands import (
     add_specific_sql,
     have_admin,
+    iinit,
     imkdir,
     iput,
     iquest,

tests/test_irods.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright © 2020, 2021, 2023 Genome Research Ltd. All rights reserved.
+# Copyright © 2020, 2021, 2023, 2024 Genome Research Ltd. All rights reserved.
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,7 +19,7 @@
 
 import hashlib
 import os.path
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import PurePath
 
 import pytest
@@ -996,7 +996,7 @@ def test_supersede_meta_data_object(self, simple_data_object):
         # Replace avu1, avu3 with avu4, avu5 (leaving avu2 in place)
         avu4 = AVU("abcde", "99999")
         avu5 = AVU("abcde", "00000")
-        date = datetime.utcnow()
+        date = datetime.now(timezone.utc)
         assert obj.supersede_metadata(avu4, avu5, history=True, history_date=date) == (
             2,
             3,