Merge pull request #148 from dcermak/use-id-files

Use --iidfile & --cidfile to get the ctr & img ids

pytest_container/build.py

@@ -3,15 +3,18 @@
 builds via :py:class:`MultiStageBuild`.
 
 """
+import tempfile
 from dataclasses import dataclass
-from os import path
+from os.path import basename
+from os.path import join
 from pathlib import Path
 from string import Template
 from subprocess import check_output
 from typing import Dict
 from typing import List
 from typing import Optional
 from typing import Union
+from uuid import uuid4
 
 from _pytest.config import Config
 from _pytest.mark.structures import ParameterSet
@@ -58,7 +61,7 @@ def repo_name(self) -> str:
             if repo_without_dot_git[-1] == "/"
             else repo_without_dot_git
         )
-        return path.basename(repo_without_trailing_slash)
+        return basename(repo_without_trailing_slash)
 
     @property
     def clone_command(self) -> str:
@@ -156,7 +159,7 @@ def containerfile(self) -> str:
             **{
                 k: v
                 if isinstance(v, str)
-                else str(container_from_pytest_param(v))
+                else str(container_from_pytest_param(v)._build_tag)
                 for k, v in self.containers.items()
             }
         )
@@ -195,7 +198,7 @@ def run_build_step(
         runtime: OciRuntimeBase,
         target: Optional[str] = None,
         extra_build_args: Optional[List[str]] = None,
-    ) -> bytes:
+    ) -> str:
         """Run the multistage build in the given ``tmp_path`` using the supplied
         ``runtime``. This function requires :py:meth:`prepare_build` to be run
         beforehands.
@@ -212,14 +215,24 @@ def run_build_step(
         Returns:
             Id of the final container that has been built
         """
-        cmd = (
-            runtime.build_command
-            + (extra_build_args or [])
-            + (["--target", target] if target else [])
-            + [str(tmp_path)]
-        )
-        _logger.debug("Running multistage container build: %s", cmd)
-        return check_output(cmd)
+        # This is an ugly, duplication of the launcher code
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            iidfile = join(tmp_dir, str(uuid4()))
+            cmd = (
+                runtime.build_command
+                + (extra_build_args or [])
+                + [f"--iidfile={iidfile}"]
+                + (["--target", target] if target else [])
+                + [str(tmp_path)]
+            )
+            _logger.debug("Running multistage container build: %s", cmd)
+            check_output(cmd)
+            with open(iidfile, "r") as iidfile_f:
+                img_digest_type, img_digest = (
+                    iidfile_f.read(-1).strip().split(":")
+                )
+                assert img_digest_type == "sha256"
+                return img_digest
 
     def build(
         self,
@@ -265,8 +278,6 @@ def build(
             root,
             extra_build_args,
         )
-        return runtime.get_image_id_from_stdout(
-            MultiStageBuild.run_build_step(
-                tmp_path, runtime, target, extra_build_args
-            ).decode()
+        return MultiStageBuild.run_build_step(
+            tmp_path, runtime, target, extra_build_args
         )

pytest_container/container.py

@@ -21,6 +21,9 @@
 from datetime import datetime
 from datetime import timedelta
 from hashlib import md5
+from os.path import exists
+from os.path import isabs
+from os.path import join
 from pathlib import Path
 from subprocess import call
 from subprocess import check_output
@@ -33,6 +36,7 @@
 from typing import overload
 from typing import Type
 from typing import Union
+from uuid import uuid4
 
 import pytest
 import testinfra
@@ -332,9 +336,7 @@ def __enter__(self) -> "BindMountCreator":
 
         assert self.volume.host_path
         self.volume._vol_name = self.volume.host_path
-        if os.path.isabs(self.volume.host_path) and not os.path.exists(
-            self.volume.host_path
-        ):
+        if isabs(self.volume.host_path) and not exists(self.volume.host_path):
             raise RuntimeError(
                 f"Volume with the host path '{self.volume.host_path}' "
                 "was requested but the directory does not exist"
@@ -475,6 +477,14 @@ def __post_init__(self) -> None:
     def __str__(self) -> str:
         return self.url or self.container_id
 
+    @property
+    def _build_tag(self) -> str:
+        """Internal build tag assigned to each immage, either the image url or
+        the container digest prefixed with ``pytest_container:``.
+
+        """
+        return self.url or f"pytest_container:{self.container_id}"
+
     @property
     def local_image(self) -> bool:
         """Returns true if this image has been build locally and has not been
@@ -688,13 +698,13 @@ def prepare_container(
             return
 
         with tempfile.TemporaryDirectory() as tmpdirname:
-            containerfile_path = os.path.join(tmpdirname, "Dockerfile")
+            containerfile_path = join(tmpdirname, "Dockerfile")
+            iidfile = join(tmpdirname, str(uuid4()))
             with open(containerfile_path, "w") as containerfile:
                 from_id = (
                     self.base
                     if isinstance(self.base, str)
-                    else getattr(self.base, "url", self.base.container_id)
-                    or self.base.container_id
+                    else (getattr(self.base, "url") or self.base._build_tag)
                 )
                 assert from_id
                 containerfile_contents = f"""FROM {from_id}
@@ -754,12 +764,28 @@ def prepare_container(
                     if self.add_build_tags
                     else []
                 )
-                + ["-f", containerfile_path, str(rootdir)]
+                + [
+                    f"--iidfile={iidfile}",
+                    "-f",
+                    containerfile_path,
+                    str(rootdir),
+                ]
             )
+
             _logger.debug("Building image via: %s", cmd)
-            self.container_id = runtime.get_image_id_from_stdout(
-                check_output(cmd).decode().strip()
+            check_output(cmd)
+
+            with open(iidfile, "r") as iidfile_f:
+                img_hash_type, img_id = iidfile_f.read(-1).strip().split(":")
+                assert img_hash_type == "sha256"
+                self.container_id = img_id
+
+            assert self._build_tag.startswith("pytest_container:")
+
+            check_output(
+                (runtime.runner_binary, "tag", img_id, self._build_tag)
             )
+
             _logger.debug(
                 "Successfully build the container image %s", self.container_id
             )
@@ -869,6 +895,10 @@ class ContainerLauncher:
 
     _stack: contextlib.ExitStack = field(default_factory=contextlib.ExitStack)
 
+    _cidfile: str = field(
+        default_factory=lambda: join(tempfile.gettempdir(), str(uuid4()))
+    )
+
     def __enter__(self) -> "ContainerLauncher":
         return self
 
@@ -925,6 +955,8 @@ def release_lock() -> None:
         if self.container_name:
             extra_run_args.extend(("--name", self.container_name))
 
+        extra_run_args.append(f"--cidfile={self._cidfile}")
+
         # We must perform the launches in separate branches, as containers with
         # port forwards must be launched while the lock is being held. Otherwise
         # another container could pick the same ports before this one launches.
@@ -941,14 +973,17 @@ def release_lock() -> None:
                 )
 
                 _logger.debug("Launching container via: %s", launch_cmd)
-                self._container_id = check_output(launch_cmd).decode().strip()
+                check_output(launch_cmd)
         else:
             launch_cmd = self.container.get_launch_cmd(
                 self.container_runtime, extra_run_args=extra_run_args
             )
 
             _logger.debug("Launching container via: %s", launch_cmd)
-            self._container_id = check_output(launch_cmd).decode().strip()
+            check_output(launch_cmd)
+
+        with open(self._cidfile, "r") as cidfile:
+            self._container_id = cidfile.read(-1).strip()
 
         self._wait_for_container_to_become_healthy()
 

pytest_container/runtime.py

@@ -546,13 +546,7 @@ def _runtime_error_message() -> str:
 
     def __init__(self) -> None:
         super().__init__(
-            build_command=[
-                "env",
-                "DOCKER_BUILDKIT=0",
-                "docker",
-                "build",
-                "--force-rm",
-            ],
+            build_command=["docker", "build", "--force-rm"],
             runner_binary="docker",
             _runtime_functional=self._runtime_functional,
         )

tests/test_inspect.py

@@ -5,6 +5,8 @@
 from pytest_container import DerivedContainer
 from pytest_container.container import ContainerData
 from pytest_container.inspect import VolumeMount
+from pytest_container.runtime import OciRuntimeBase
+
 
 IMAGE_WITH_EVERYTHING = DerivedContainer(
     base=LEAP,
@@ -22,7 +24,7 @@
 
 
 @pytest.mark.parametrize("container", [IMAGE_WITH_EVERYTHING], indirect=True)
-def test_inspect(container: ContainerData):
+def test_inspect(container: ContainerData, container_runtime: OciRuntimeBase):
     inspect = container.inspect
 
     assert inspect.id == container.container_id
@@ -32,7 +34,17 @@ def test_inspect(container: ContainerData):
     assert (
         "HOME" in inspect.config.env and inspect.config.env["HOME"] == "/src/"
     )
-    assert inspect.config.image == str(container.container)
+
+    # podman and docker cannot agree on what the Config.Image value is: podman
+    # prefixes it with `localhost` and the full build tag
+    # (i.e. `pytest_container:$digest`), while docker just uses the digest
+    expected_img = (
+        str(container.container)
+        if container_runtime.runner_binary == "docker"
+        else f"localhost/pytest_container:{container.container}"
+    )
+
+    assert inspect.config.image == expected_img
     assert inspect.config.cmd == ["/bin/sh"]
 
     assert (

tests/test_volumes.py

@@ -1,5 +1,7 @@
 # pylint: disable=missing-function-docstring,missing-module-docstring
 import os
+from os.path import abspath
+from os.path import join
 from typing import List
 
 import pytest
@@ -133,11 +135,11 @@ def test_container_volume_host_writing(container_per_test: ContainerData):
 
 - there is nothing to see, please carry on"""
 
-    with open(os.path.join(vol.host_path, "test"), "w") as testfile:
+    with open(join(vol.host_path, "test"), "w") as testfile:
         testfile.write(contents)
 
     testfile_in_container = container_per_test.connection.file(
-        os.path.join(vol.container_path, "test")
+        join(vol.container_path, "test")
     )
     assert (
         testfile_in_container.exists
@@ -165,7 +167,7 @@ def test_container_volumes(container_per_test: ContainerData):
     assert len(container_per_test.container.volume_mounts) == 2
     for vol in container_per_test.container.volume_mounts:
         dir_in_container = container_per_test.connection.file(
-            os.path.join("/", vol.container_path)
+            join("/", vol.container_path)
         )
         assert dir_in_container.exists and dir_in_container.is_directory
 
@@ -186,7 +188,7 @@ def test_container_volume_writeable(container_per_test: ContainerData):
     )
 
     testfile_in_container = container_per_test.connection.file(
-        os.path.join(vol.container_path, "test")
+        join(vol.container_path, "test")
     )
     assert (
         testfile_in_container.exists
@@ -234,7 +236,7 @@ def test_concurent_container_volumes(container_per_test: ContainerData):
     volume_mounts=[
         BindMount(
             "/src/",
-            host_path=os.path.join(os.path.abspath(os.getcwd()), "tests"),
+            host_path=join(abspath(os.getcwd()), "tests"),
         )
     ],
 )