Set PIP_USER in base image (#437)

Ensure that pip installs to ~/.local by default, instead of /opt/conda.

Other minor changes:

* Do not pin pip version and always upgrade to latest, 
as recommended by pip maintainers. 
f there are any breaking changes in the future, 
they should be caught by the integration tests.
* Install appmode from PyPI
* Cleanup custom logo setup
* pytest: Make --variant a required parameter
* Simplify aiidalab_exec fixture usage
* Move test_pip_check to common tests

Author: danielhollas

docker-bake.hcl

@@ -83,6 +83,9 @@ target "base-with-services" {
     "PGSQL_VERSION" = "${PGSQL_VERSION}"
   }
 }
+# PYTHON_MINOR_VERSION is a Python version string
+# without the patch version (e.g. "3.9")
+# Used to construct paths to Python site-packages folder.
 target "lab" {
   inherits = ["lab-meta"]
   context = "stack/lab"
@@ -93,6 +96,7 @@ target "lab" {
   args = {
     "AIIDALAB_VERSION"      = "${AIIDALAB_VERSION}"
     "AIIDALAB_HOME_VERSION" = "${AIIDALAB_HOME_VERSION}"
+    "PYTHON_MINOR_VERSION" = join(".", slice(split(".", "${PYTHON_VERSION}"), 0, 2))
   }
 }
 target "full-stack" {

stack/base/Dockerfile

@@ -20,8 +20,7 @@ ARG AIIDA_VERSION
 # We pin aiida-core to the exact installed version,
 # to prevent accidental upgrade or downgrade, that might
 # induce DB migration or break shared dependencies of AiiDAlab Apps.
-RUN echo "pip==23.3.1" > /opt/requirements.txt && \
-  echo "aiida-core==${AIIDA_VERSION}" >> /opt/requirements.txt
+RUN echo "aiida-core==${AIIDA_VERSION}" > /opt/requirements.txt
 
 # Install the shared requirements.
 RUN mamba install --yes \
@@ -32,11 +31,15 @@ RUN mamba install --yes \
      fix-permissions "${CONDA_DIR}" && \
      fix-permissions "/home/${NB_USER}"
 
-# Pin shared requirements in the base environemnt.
+# Pin shared requirements in the conda base environment.
 RUN cat /opt/requirements.txt | xargs -I{} conda config --system --add pinned_packages {}
 
+# Upgrade pip to latest
+RUN pip install --upgrade --no-cache-dir pip
 # Configure pip to use requirements file as constraints file.
-ENV PIP_CONSTRAINT=/opt/requirements.txt
+ENV PIP_CONSTRAINT /opt/requirements.txt
+# Ensure that pip installs packages to ~/.local by default
+ENV PIP_USER 1
 
 # Enable verdi autocompletion.
 RUN mkdir -p "${CONDA_DIR}/etc/conda/activate.d" && \

stack/lab/Dockerfile

@@ -41,20 +41,20 @@ ARG AIIDALAB_HOME_VERSION
 RUN git clone https://github.com/aiidalab/aiidalab-home && \
      cd aiidalab-home && \
      git checkout v"${AIIDALAB_HOME_VERSION}" && \
-     pip install --quiet --no-cache-dir "./" && \
+     pip install --no-user --quiet --no-cache-dir "./" && \
      fix-permissions "./" && \
      fix-permissions "${CONDA_DIR}" && \
      fix-permissions "/home/${NB_USER}"
 
-# Install and enable appmode.
-RUN git clone https://github.com/oschuett/appmode.git && \
-     cd appmode && \
-     git checkout v0.8.0
-COPY gears.svg ./appmode/appmode/static/gears.svg
-RUN pip install ./appmode --no-cache-dir && \
-     jupyter nbextension enable --py --sys-prefix appmode && \
+# Install and enable appmode, turning Jupyter notebooks to Apps
+RUN pip install appmode==0.8.0 --no-cache-dir --no-user
+# Enable appmode extension
+RUN jupyter nbextension enable --py --sys-prefix appmode && \
      jupyter serverextension enable --py --sys-prefix appmode
 
+# Swap appmode icon for AiiDAlab gears icon, shown during app load
+COPY --chown=${NB_UID}:${NB_GID} gears.svg ${CONDA_DIR}/share/jupyter/nbextensions/appmode/gears.svg
+
 # Copy start-up scripts for AiiDAlab.
 COPY before-notebook.d/* /usr/local/bin/before-notebook.d/
 
@@ -107,8 +107,5 @@ ENV NOTEBOOK_ARGS \
      "--TerminalManager.cull_interval=60"
 
 # Set up the logo of notebook interface
-COPY --chown=${NB_UID}:${NB_GID} aiidalab-wide-logo.png /tmp/notebook-logo.png
-
-# The directory location of logo.png is in the `${CONDA_DIR}/lib/python3.9/site-packages/notebook/static/base/images/logo.png`,
-# but the python version may change in the future, thus we use the wildcard to match the python version.
-RUN mv /tmp/notebook-logo.png ${CONDA_DIR}/lib/python3*/site-packages/notebook/static/base/images/logo.png
+ARG PYTHON_MINOR_VERSION
+COPY --chown=${NB_UID}:${NB_GID} aiidalab-wide-logo.png ${CONDA_DIR}/lib/python${PYTHON_MINOR_VERSION}/site-packages/notebook/static/base/images/logo.png

tests/conftest.py

@@ -6,6 +6,8 @@
 
 from requests.exceptions import ConnectionError
 
+VARIANTS = ("base", "lab", "base-with-services", "full-stack")
+
 
 def is_responsive(url):
     try:
@@ -16,12 +18,20 @@ def is_responsive(url):
         return False
 
 
+def variant_checker(value):
+    msg = f"Invalid image variant '{value}', must be one of: {VARIANTS}"
+    if value not in VARIANTS:
+        raise pytest.UsageError(msg)
+    return value
+
+
 def pytest_addoption(parser):
     parser.addoption(
         "--variant",
         action="store",
-        default="base",
+        required=True,
         help="Variant (image name) of the docker-compose file to use.",
+        type=variant_checker,
     )
 
 
@@ -54,14 +64,30 @@ def execute(command, user=None, **kwargs):
             command = f"exec -T --user={user} aiidalab {command}"
         else:
             command = f"exec -T aiidalab {command}"
-        return docker_compose.execute(command, **kwargs)
+        out = docker_compose.execute(command, **kwargs)
+        return out.decode()
 
     return execute
 
 
 @pytest.fixture
 def nb_user(aiidalab_exec):
-    return aiidalab_exec("bash -c 'echo \"${NB_USER}\"'").decode().strip()
+    return aiidalab_exec("bash -c 'echo \"${NB_USER}\"'").strip()
+
+
+@pytest.fixture
+def pip_install(aiidalab_exec, nb_user):
+    """Temporarily install package via pip"""
+    package = None
+
+    def _pip_install(pkg, **args):
+        nonlocal package
+        package = pkg
+        return aiidalab_exec(f"pip install {pkg}", **args)
+
+    yield _pip_install
+    if package:
+        aiidalab_exec(f"pip uninstall --yes {package}")
 
 
 @pytest.fixture(scope="session")

tests/test-base-with-services.py

@@ -5,11 +5,8 @@
 
 
 def test_correct_pgsql_version_installed(aiidalab_exec, pgsql_version):
-    info = json.loads(
-        aiidalab_exec(
-            "mamba list -n aiida-core-services --json --full-name postgresql"
-        ).decode()
-    )[0]
+    cmd = "mamba list -n aiida-core-services --json --full-name postgresql"
+    info = json.loads(aiidalab_exec(cmd))[0]
     assert info["name"] == "postgresql"
     assert parse(info["version"]).major == parse(pgsql_version).major
 
@@ -18,5 +15,4 @@ def test_rabbitmq_can_start(aiidalab_exec):
     """Test the rabbitmq-server can start, the output should be empty if
     the command is successful."""
     output = aiidalab_exec("mamba run -n aiida-core-services rabbitmq-server -detached")
-
-    assert output == b""
+    assert output == ""

tests/test-base.py

@@ -5,50 +5,61 @@
 from packaging.version import parse
 
 
-@pytest.mark.parametrize("incompatible_version", ["2.3.0"])
-def test_prevent_pip_install_of_incompatible_aiida_version(
-    aiidalab_exec, nb_user, aiida_version, incompatible_version
+@pytest.mark.parametrize("pkg_manager", ["pip", "mamba"])
+def test_prevent_installation_of_aiida(
+    aiidalab_exec, nb_user, aiida_version, pkg_manager
 ):
-    package_manager = "pip"
+    """aiida-core is pinned to the exact version in the container,
+    test that both pip and mamba refuse to install a different version"""
+
+    incompatible_version = "2.3.0"
     assert parse(aiida_version) != parse(incompatible_version)
+
     # Expected to succeed, although should be a no-op.
-    aiidalab_exec(
-        f"{package_manager} install aiida-core=={aiida_version}", user=nb_user
-    )
+    aiidalab_exec(f"{pkg_manager} install aiida-core=={aiida_version}", user=nb_user)
     with pytest.raises(Exception):
         aiidalab_exec(
-            f"{package_manager} install aiida-core=={incompatible_version}",
+            f"{pkg_manager} install aiida-core=={incompatible_version}",
             user=nb_user,
         )
 
 
 def test_correct_python_version_installed(aiidalab_exec, python_version):
-    info = json.loads(aiidalab_exec("mamba list --json --full-name python").decode())[0]
+    info = json.loads(aiidalab_exec("mamba list --json --full-name python"))[0]
     assert info["name"] == "python"
     assert parse(info["version"]) == parse(python_version)
 
 
 def test_create_conda_environment(aiidalab_exec, nb_user):
-    output = aiidalab_exec("conda create -y -n tmp", user=nb_user).decode().strip()
+    output = aiidalab_exec("conda create -y -n tmp", user=nb_user).strip()
     assert "conda activate tmp" in output
     # New conda environments should be created in ~/.conda/envs/
-    output = aiidalab_exec("conda env list", user=nb_user).decode().strip()
+    output = aiidalab_exec("conda env list", user=nb_user).strip()
     assert f"/home/{nb_user}/.conda/envs/tmp" in output
 
 
-def test_pip_check(aiidalab_exec):
-    aiidalab_exec("pip check")
-
-
 def test_correct_aiida_version_installed(aiidalab_exec, aiida_version):
-    info = json.loads(
-        aiidalab_exec("mamba list --json --full-name aiida-core").decode()
-    )[0]
+    cmd = "mamba list --json --full-name aiida-core"
+    info = json.loads(aiidalab_exec(cmd))[0]
     assert info["name"] == "aiida-core"
     assert parse(info["version"]) == parse(aiida_version)
 
 
 def test_path_local_pip(aiidalab_exec, nb_user):
     """test that the pip local bin path ~/.local/bin is added to PATH"""
-    output = aiidalab_exec("bash -c 'echo \"${PATH}\"'", user=nb_user).decode()
+    output = aiidalab_exec("bash -c 'echo \"${PATH}\"'", user=nb_user)
     assert f"/home/{nb_user}/.local/bin" in output
+
+
+def test_pip_user_install(aiidalab_exec, pip_install, nb_user):
+    """Test that pip installs packages to ~/.local/ by default"""
+    import email
+
+    # We use 'tuna' as an example of python-only package without dependencies
+    pkg = "tuna"
+    pip_install(pkg)
+    output = aiidalab_exec(f"pip show {pkg}")
+
+    # `pip show` output is in the RFC-compliant email header format
+    msg = email.message_from_string(output)
+    assert msg.get("Location").startswith(f"/home/{nb_user}/.local/")

tests/test-common.py

@@ -9,14 +9,18 @@ def test_notebook_service_available(notebook_service):
 
 
 def test_verdi_status(aiidalab_exec, nb_user):
-    output = aiidalab_exec("verdi status", user=nb_user).decode().strip()
+    output = aiidalab_exec("verdi status", user=nb_user).strip()
     assert "Connected to RabbitMQ" in output
     assert "Daemon is running" in output
 
 
 def test_ssh_agent_is_running(aiidalab_exec, nb_user):
-    output = aiidalab_exec("ps aux | grep ssh-agent", user=nb_user).decode().strip()
+    output = aiidalab_exec("ps aux | grep ssh-agent", user=nb_user).strip()
     assert "ssh-agent" in output
 
     # also check only one ssh-agent process is running
     assert len(output.splitlines()) == 1
+
+
+def test_pip_check(aiidalab_exec):
+    aiidalab_exec("pip check")

tests/test-full-stack.py

@@ -4,13 +4,10 @@
 @pytest.fixture(scope="function")
 def generate_aiidalab_install_output(aiidalab_exec, nb_user):
     def _generate_aiidalab_install_output(package_name):
-        output = (
-            aiidalab_exec(f"aiidalab install --yes --pre {package_name}", user=nb_user)
-            .decode()
-            .strip()
-        )
+        cmd = f"aiidalab install --yes --pre {package_name}"
+        output = aiidalab_exec(cmd, user=nb_user).strip()
 
-        output += aiidalab_exec(f"pip check", user=nb_user).decode().strip()
+        output += aiidalab_exec("pip check", user=nb_user).strip()
 
         # Uninstall the package to make sure the test is repeatable
         app_name = package_name.split("@")[0]

tests/test-lab.py

@@ -4,21 +4,30 @@
 
 
 def test_correct_aiidalab_version_installed(aiidalab_exec, aiidalab_version):
-    info = json.loads(aiidalab_exec("mamba list --json --full-name aiidalab").decode())[
-        0
-    ]
+    cmd = "mamba list --json --full-name aiidalab"
+    info = json.loads(aiidalab_exec(cmd))[0]
     assert info["name"] == "aiidalab"
     assert parse(info["version"]) == parse(aiidalab_version)
 
 
 def test_correct_aiidalab_home_version_installed(aiidalab_exec, aiidalab_home_version):
-    info = json.loads(
-        aiidalab_exec("mamba list --json --full-name aiidalab-home").decode()
-    )[0]
+    cmd = "mamba list --json --full-name aiidalab-home"
+    info = json.loads(aiidalab_exec(cmd))[0]
     assert info["name"] == "aiidalab-home"
     assert parse(info["version"]) == parse(aiidalab_home_version)
 
 
+def test_appmode_installed(aiidalab_exec):
+    """Test that appmode pip package is installed in correct location"""
+    import email
+
+    output = aiidalab_exec("pip show appmode")
+
+    # `pip show` output is in the RFC-compliant email header format
+    msg = email.message_from_string(output)
+    assert msg.get("Location").startswith("/opt/conda/lib/python")
+
+
 @pytest.mark.parametrize("incompatible_version", ["22.7.1"])
 def test_prevent_pip_install_of_incompatible_aiidalab_version(
     aiidalab_exec,