From 11e002d73abb83f3783e2059b021b3df5eb44745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20=C4=8Cerm=C3=A1k?= Date: Wed, 21 Aug 2024 12:10:07 +0200 Subject: [PATCH] WIP: Add MultiStageContainer class --- CHANGELOG.rst | 8 ++- pytest_container/container.py | 113 ++++++++++++++++++++++++++++------ pytest_container/pod.py | 3 +- source/fixtures.rst | 2 +- source/usage.rst | 2 +- tests/test_container_build.py | 53 +++++++++++++--- 6 files changed, 147 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c847389..eee1424 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -71,8 +71,9 @@ Internal changes: Breaking changes: - add the parameter ``container_runtime`` to - :py:func:`~pytest_container.container.ContainerBaseABC.prepare_container` and - :py:func:`~pytest_container.build.MultiStageBuild.prepare_build`. + ``ContainerBaseABC.prepare_container`` (now called + :py:func:`~pytest_container.container.ContainerPrepareABC.prepare_container`) + and :py:func:`~pytest_container.build.MultiStageBuild.prepare_build`. - deprecate the function ``pytest_container.container_from_pytest_param``, please use @@ -225,7 +226,8 @@ Improvements and new features: parametrize this test run. - Add support to add tags to container images via - :py:attr:`~pytest_container.container.DerivedContainer.add_build_tags`. + ``DerivedContainer.add_build_tags`` (is now called + :py:attr:`~pytest_container.container._ContainerForBuild.add_build_tags`) - Lock container preparation so that only a single process is pulling & building a container image. diff --git a/pytest_container/container.py b/pytest_container/container.py index df8c784..33fa4f8 100644 --- a/pytest_container/container.py +++ b/pytest_container/container.py @@ -25,6 +25,7 @@ from os.path import isabs from os.path import join from pathlib import Path +from string import Template from subprocess import call from subprocess import check_output from types import TracebackType @@ -613,7 +614,7 @@ def filelock_filename(self) -> str: if isinstance(value, list): all_elements.append("".join([str(elem) for elem in value])) elif isinstance(value, dict): - all_elements.append("".join(value.values())) + all_elements.append("".join(str(v) for v in value.values())) else: all_elements.append(str(value)) @@ -624,7 +625,7 @@ def filelock_filename(self) -> str: return f"{sha3_256((''.join(all_elements)).encode()).hexdigest()}.lock" -class ContainerBaseABC(ABC): +class ContainerPrepareABC(ABC): """Abstract base class defining the methods that must be implemented by the classes fed to the ``*container*`` fixtures. @@ -639,6 +640,8 @@ def prepare_container( ) -> None: """Prepares the container so that it can be launched.""" + +class ContainerBaseABC(ContainerPrepareABC): @abstractmethod def get_base(self) -> "Union[Container, DerivedContainer]": """Returns the Base of this Container Image. If the container has no @@ -795,20 +798,12 @@ def _run_container_build( @dataclass(unsafe_hash=True) -class DerivedContainer(ContainerBase, ContainerBaseABC): - """Class for storing information about the Container Image under test, that - is build from a :file:`Containerfile`/:file:`Dockerfile` from a different - image (can be any image from a registry or an instance of - :py:class:`Container` or :py:class:`DerivedContainer`). +class _ContainerForBuild(ContainerBase): + """Intermediate class for adding properties to :py:class:`DerivedContainer` + and :py:class:`MultiStageContainer`. """ - base: Union[Container, "DerivedContainer", str] = "" - - #: The :file:`Containerfile` that is used to build this container derived - #: from :py:attr:`base`. - containerfile: str = "" - #: An optional image format when building images with :command:`buildah`. It #: is ignored when the container runtime is :command:`docker`. #: The ``oci`` image format is used by default. If the image format is @@ -822,6 +817,22 @@ class DerivedContainer(ContainerBase, ContainerBaseABC): #: has been built add_build_tags: List[str] = field(default_factory=list) + +@dataclass(unsafe_hash=True) +class DerivedContainer(_ContainerForBuild, ContainerBaseABC): + """Class for storing information about the Container Image under test, that + is build from a :file:`Containerfile`/:file:`Dockerfile` from a different + image (can be any image from a registry or an instance of + :py:class:`Container` or :py:class:`DerivedContainer`). + + """ + + base: Union[Container, "DerivedContainer", str] = "" + + #: The :file:`Containerfile` that is used to build this container derived + #: from :py:attr:`base`. + containerfile: str = "" + def __post_init__(self) -> None: super().__post_init__() if not self.base: @@ -895,8 +906,57 @@ def prepare_container( assert self._build_tag == internal_build_tag +@dataclass +class MultiStageContainer(_ContainerForBuild, ContainerPrepareABC): + containerfile: str = "" + + containers: Dict[str, Union[Container, DerivedContainer, str]] = field( + default_factory=dict + ) + + #: Optional stage of the multistage container build that should be built. + #: The last stage is built by default. + target_stage: str = "" + def prepare_container( + self, + container_runtime: OciRuntimeBase, + rootdir: Path, + extra_build_args: Optional[List[str]], + ) -> None: + """Prepares the container so that it can be launched.""" + template_kwargs: Dict[str, str] = {} + + for name, ctr in self.containers.items(): + if isinstance(ctr, str): + c = Container(url=ctr) + c.prepare_container( + container_runtime, rootdir, extra_build_args + ) + template_kwargs[name] = c._build_tag + else: + ctr.prepare_container( + container_runtime, rootdir, extra_build_args + ) + template_kwargs[name] = ctr._build_tag + + ctrfile = Template(self.containerfile).substitute(**template_kwargs) + + build_args = tuple(*extra_build_args) if extra_build_args else () + if self.target_stage: + build_args += ("--target", self.target_stage) + + self.container_id, internal_tag = _run_container_build( + container_runtime, + rootdir, + ctrfile, + None, + build_args, + self.image_format, + self.add_build_tags, + ) + assert self._build_tag == internal_tag @dataclass(frozen=True) @@ -915,7 +975,7 @@ class ContainerData: #: the testinfra connection to the running container connection: Any #: the container data class that has been used in this test - container: Union[Container, DerivedContainer] + container: Union[Container, DerivedContainer, MultiStageContainer] #: any ports that are exposed by this container forwarded_ports: List[PortForwarding] @@ -972,11 +1032,18 @@ def container_and_marks_from_pytest_param( ... +@overload +def container_and_marks_from_pytest_param( + ctr_or_param: MultiStageContainer, +) -> Tuple[MultiStageContainer, Literal[None]]: + ... + + @overload def container_and_marks_from_pytest_param( ctr_or_param: _pytest.mark.ParameterSet, ) -> Tuple[ - Union[Container, DerivedContainer], + Union[Container, DerivedContainer, MultiStageContainer], Optional[Collection[Union[_pytest.mark.MarkDecorator, _pytest.mark.Mark]]], ]: ... @@ -984,10 +1051,13 @@ def container_and_marks_from_pytest_param( def container_and_marks_from_pytest_param( ctr_or_param: Union[ - _pytest.mark.ParameterSet, Container, DerivedContainer + _pytest.mark.ParameterSet, + Container, + DerivedContainer, + MultiStageContainer, ], ) -> Tuple[ - Union[Container, DerivedContainer], + Union[Container, DerivedContainer, MultiStageContainer], Optional[Collection[Union[_pytest.mark.MarkDecorator, _pytest.mark.Mark]]], ]: """Extracts the :py:class:`~pytest_container.container.Container` or @@ -1001,11 +1071,14 @@ def container_and_marks_from_pytest_param( returned directly and the second return value is ``None``. """ - if isinstance(ctr_or_param, (Container, DerivedContainer)): + if isinstance( + ctr_or_param, (Container, DerivedContainer, MultiStageContainer) + ): return ctr_or_param, None if len(ctr_or_param.values) > 0 and isinstance( - ctr_or_param.values[0], (Container, DerivedContainer) + ctr_or_param.values[0], + (Container, DerivedContainer, MultiStageContainer), ): return ctr_or_param.values[0], ctr_or_param.marks @@ -1049,7 +1122,7 @@ class ContainerLauncher: """ #: The container that will be launched - container: Union[Container, DerivedContainer] + container: Union[Container, DerivedContainer, MultiStageContainer] #: The container runtime via which the container will be launched container_runtime: OciRuntimeBase diff --git a/pytest_container/pod.py b/pytest_container/pod.py index 57b9b68..bc13d6d 100644 --- a/pytest_container/pod.py +++ b/pytest_container/pod.py @@ -18,6 +18,7 @@ from pytest_container.container import create_host_port_port_forward from pytest_container.container import DerivedContainer from pytest_container.container import lock_host_port_search +from pytest_container.container import MultiStageContainer from pytest_container.inspect import PortForwarding from pytest_container.logging import _logger from pytest_container.runtime import get_selected_runtime @@ -35,7 +36,7 @@ class Pod: """ #: containers belonging to the pod - containers: List[Union[DerivedContainer, Container]] + containers: List[Union[MultiStageContainer, DerivedContainer, Container]] #: ports exposed by the pod forwarded_ports: List[PortForwarding] = field(default_factory=list) diff --git a/source/fixtures.rst b/source/fixtures.rst index 60a2d39..b726ae9 100644 --- a/source/fixtures.rst +++ b/source/fixtures.rst @@ -45,5 +45,5 @@ directive is only supported for docker images. While this is the default with :command:`docker`, :command:`buildah` will by default build images in the ``OCIv1`` format which does **not** support ``HEALTHCHECK``. To ensure that your created container includes the ``HEALTHCHECK``, set the attribute -:py:attr:`~pytest_container.container.DerivedContainer.image_format` to +:py:attr:`~pytest_container.container._ContainerForBuild.image_format` to :py:attr:`~pytest_container.container.ImageFormat.DOCKER`. diff --git a/source/usage.rst b/source/usage.rst index 651b919..8d6f002 100644 --- a/source/usage.rst +++ b/source/usage.rst @@ -8,7 +8,7 @@ Sometimes it is necessary to customize the build, run or pod create parameters of the container runtime globally, e.g. to use the host's network with docker via ``--network=host``. -The :py:meth:`~pytest_container.container.ContainerBaseABC.prepare_container` +The :py:meth:`~pytest_container.container.ContainerPrepareABC.prepare_container` and :py:meth:`~pytest_container.container.ContainerBase.get_launch_cmd` methods support passing such additional arguments/flags, but this is rather cumbersome to use in practice. The ``*container*`` and ``pod*`` fixtures will therefore diff --git a/tests/test_container_build.py b/tests/test_container_build.py index 1c7a4f1..acc0a81 100644 --- a/tests/test_container_build.py +++ b/tests/test_container_build.py @@ -11,6 +11,7 @@ from pytest_container.container import ContainerData from pytest_container.container import ContainerLauncher from pytest_container.container import EntrypointSelection +from pytest_container.container import MultiStageContainer from pytest_container.inspect import PortForwarding from pytest_container.runtime import LOCALHOST from pytest_container.runtime import OciRuntimeBase @@ -63,13 +64,8 @@ CONTAINER_IMAGES = [LEAP, LEAP_WITH_MAN, LEAP_WITH_MAN_AND_LUA] -MULTI_STAGE_BUILD = MultiStageBuild( - containers={ - "builder": LEAP_WITH_MAN, - "runner1": LEAP, - "runner2": "docker.io/alpine", - }, - containerfile_template=r"""FROM $builder as builder + +_MULTISTAGE_CTR_FILE = r"""FROM $builder as builder WORKDIR /src RUN echo $$'#!/bin/sh \n\ echo "foobar"' > test.sh && chmod +x test.sh @@ -82,9 +78,32 @@ FROM $runner2 as runner2 WORKDIR /bin COPY --from=builder /src/test.sh . -""", +""" + +_MULTISTAGE_CTRS = { + "builder": LEAP_WITH_MAN, + "runner1": LEAP, + "runner2": "docker.io/alpine", +} + +MULTI_STAGE_BUILD = MultiStageBuild( + containers=_MULTISTAGE_CTRS, + containerfile_template=_MULTISTAGE_CTR_FILE, +) + +MULTI_STAGE_CTR = MultiStageContainer( + containers=_MULTISTAGE_CTRS, + containerfile=_MULTISTAGE_CTR_FILE, +) + +MULTI_STAGE_CTR_STAGE_1 = MultiStageContainer( + containers=_MULTISTAGE_CTRS, + containerfile=_MULTISTAGE_CTR_FILE, + target_stage="runner1", + entry_point=EntrypointSelection.BASH, ) + # This container would just stop if we would launch it with -d and use the # default entrypoint. If we set the entrypoint to bash, then it should stay up. CONTAINER_THAT_STOPS = DerivedContainer( @@ -269,6 +288,24 @@ def test_multistage_build( ) +@pytest.mark.parametrize("container", [MULTI_STAGE_CTR], indirect=True) +def test_multistage_container_without_stage(container: ContainerData) -> None: + assert container.connection.file("/bin/test.sh").exists + assert ( + "Alpine" in container.connection.file("/etc/os-release").content_string + ) + + +@pytest.mark.parametrize("container", [MULTI_STAGE_CTR_STAGE_1], indirect=True) +def test_multistage_container_with_runner1_stage( + container: ContainerData, +) -> None: + assert container.connection.file("/bin/test.sh").exists + assert ( + "Leap" in container.connection.file("/etc/os-release").content_string + ) + + def test_multistage_build_target( tmp_path: Path, pytestconfig: Config, container_runtime: OciRuntimeBase ):