Add MultiStageContainer class

Author: dcermak

CHANGELOG.rst

@@ -9,6 +9,10 @@ Breaking changes:
 
 Improvements and new features:
 
+- Add the class :py:class:`~pytest_container.container.MultiStageContainer` as a
+  replacement of :py:class:`~pytest_container.build.MultiStageBuild` to handle
+  container images built from a :file:`Containerfile` with multiple stages
+
 - Add the function
   :py:func:`~pytest_container.container.ContainerData.read_container_logs` to
   get access to the logs of the running container
@@ -71,8 +75,9 @@ Internal changes:
 Breaking changes:
 
 - add the parameter ``container_runtime`` to
-  :py:func:`~pytest_container.container.ContainerBaseABC.prepare_container` and
-  :py:func:`~pytest_container.build.MultiStageBuild.prepare_build`.
+  ``ContainerBaseABC.prepare_container`` (now called
+  :py:func:`~pytest_container.container.ContainerPrepareABC.prepare_container`)
+  and :py:func:`~pytest_container.build.MultiStageBuild.prepare_build`.
 
 - deprecate the function ``pytest_container.container_from_pytest_param``,
   please use
@@ -225,7 +230,8 @@ Improvements and new features:
   parametrize this test run.
 
 - Add support to add tags to container images via
-  :py:attr:`~pytest_container.container.DerivedContainer.add_build_tags`.
+  ``DerivedContainer.add_build_tags`` (is now called
+  :py:attr:`~pytest_container.container._ContainerForBuild.add_build_tags`)
 
 - Lock container preparation so that only a single process is pulling & building
   a container image.

pytest_container/__init__.py

@@ -10,6 +10,7 @@
     "container_from_pytest_param",
     "container_to_pytest_param",
     "DerivedContainer",
+    "MultiStageContainer",
     "add_extra_run_and_build_args_options",
     "add_logging_level_options",
     "auto_container_parametrize",
@@ -31,6 +32,7 @@
 from .container import container_from_pytest_param
 from .container import container_to_pytest_param
 from .container import DerivedContainer
+from .container import MultiStageContainer
 from .helpers import add_extra_run_and_build_args_options
 from .helpers import add_logging_level_options
 from .helpers import auto_container_parametrize

pytest_container/container.py

@@ -25,6 +25,7 @@
 from os.path import isabs
 from os.path import join
 from pathlib import Path
+from string import Template
 from subprocess import call
 from subprocess import check_output
 from types import TracebackType
@@ -616,7 +617,7 @@ def filelock_filename(self) -> str:
             if isinstance(value, list):
                 all_elements.append("".join([str(elem) for elem in value]))
             elif isinstance(value, dict):
-                all_elements.append("".join(value.values()))
+                all_elements.append("".join(str(v) for v in value.values()))
             else:
                 all_elements.append(str(value))
 
@@ -627,7 +628,7 @@ def filelock_filename(self) -> str:
         return f"{sha3_256((''.join(all_elements)).encode()).hexdigest()}.lock"
 
 
-class ContainerBaseABC(ABC):
+class ContainerPrepareABC(ABC):
     """Abstract base class defining the methods that must be implemented by the
     classes fed to the ``*container*`` fixtures.
 
@@ -642,6 +643,8 @@ def prepare_container(
     ) -> None:
         """Prepares the container so that it can be launched."""
 
+
+class ContainerBaseABC(ContainerPrepareABC):
     @abstractmethod
     def get_base(self) -> "Union[Container, DerivedContainer]":
         """Returns the Base of this Container Image. If the container has no
@@ -798,20 +801,12 @@ def _run_container_build(
 
 
 @dataclass(unsafe_hash=True)
-class DerivedContainer(ContainerBase, ContainerBaseABC):
-    """Class for storing information about the Container Image under test, that
-    is build from a :file:`Containerfile`/:file:`Dockerfile` from a different
-    image (can be any image from a registry or an instance of
-    :py:class:`Container` or :py:class:`DerivedContainer`).
+class _ContainerForBuild(ContainerBase):
+    """Intermediate class for adding properties to :py:class:`DerivedContainer`
+    and :py:class:`MultiStageContainer`.
 
     """
 
-    base: Union[Container, "DerivedContainer", str] = ""
-
-    #: The :file:`Containerfile` that is used to build this container derived
-    #: from :py:attr:`base`.
-    containerfile: str = ""
-
     #: An optional image format when building images with :command:`buildah`. It
     #: is ignored when the container runtime is :command:`docker`.
     #: The ``oci`` image format is used by default. If the image format is
@@ -825,6 +820,22 @@ class DerivedContainer(ContainerBase, ContainerBaseABC):
     #: has been built
     add_build_tags: List[str] = field(default_factory=list)
 
+
+@dataclass(unsafe_hash=True)
+class DerivedContainer(_ContainerForBuild, ContainerBaseABC):
+    """Class for storing information about the Container Image under test, that
+    is build from a :file:`Containerfile`/:file:`Dockerfile` from a different
+    image (can be any image from a registry or an instance of
+    :py:class:`Container` or :py:class:`DerivedContainer`).
+
+    """
+
+    base: Union[Container, "DerivedContainer", str] = ""
+
+    #: The :file:`Containerfile` that is used to build this container derived
+    #: from :py:attr:`base`.
+    containerfile: str = ""
+
     def __post_init__(self) -> None:
         super().__post_init__()
         if not self.base:
@@ -898,8 +909,88 @@ def prepare_container(
         assert self._build_tag == internal_build_tag
 
 
+@dataclass
+class MultiStageContainer(_ContainerForBuild, ContainerPrepareABC):
+    """Class representing a container built from a :file:`Containerfile`
+    containing multiple stages. The :py:attr:`MultiStageContainer.containerfile`
+    is templated using the builtin :py:class:`string.Template`, where container
+    image IDs are inserted from the containers in
+    :py:attr:`MultiStageContainer.containers` after these have been
+    built/pulled.
+
+    """
 
+    #: :file:`Containerfile` to built the container. If any stages require
+    #: images that are defined using a :py:class:`DerivedContainer` or a
+    #: :py:class:`Container`, then insert their ids as a template name and
+    #: provide that name and the class instance as the key & value into
+    #: :py:attr:`containers`.
+    containerfile: str = ""
 
+    #: Dictionary of container stages that are used to build the final
+    #: image. The keys are the template names used in :py:attr:`containerfile`
+    #: and will be replaced with the container image ids of the respective values.
+    containers: Dict[str, Union[Container, DerivedContainer, str]] = field(
+        default_factory=dict
+    )
+
+    #: Optional stage of the multistage container build that should be built.
+    #: The last stage is built by default.
+    target_stage: str = ""
+
+    def prepare_container(
+        self,
+        container_runtime: OciRuntimeBase,
+        rootdir: Path,
+        extra_build_args: Optional[List[str]],
+    ) -> None:
+        """Builds all intermediate containers and then builds the final
+        container up to :py:attr:`target_stage` or to the last stage.
+
+        """
+
+        template_kwargs: Dict[str, str] = {}
+
+        for name, ctr in self.containers.items():
+            if isinstance(ctr, str):
+                warnings.warn(
+                    UserWarning(
+                        "Putting container URLs or scratch into the containers "
+                        "dictionary is not required, just enter them into the "
+                        "containerfile directly."
+                    )
+                )
+
+                if ctr == "scratch":
+                    template_kwargs[name] = ctr
+                else:
+                    c = Container(url=ctr)
+                    c.prepare_container(
+                        container_runtime, rootdir, extra_build_args
+                    )
+                    template_kwargs[name] = c._build_tag
+            else:
+                ctr.prepare_container(
+                    container_runtime, rootdir, extra_build_args
+                )
+                template_kwargs[name] = ctr._build_tag
+
+        ctrfile = Template(self.containerfile).substitute(**template_kwargs)
+
+        build_args = tuple(*extra_build_args) if extra_build_args else ()
+        if self.target_stage:
+            build_args += ("--target", self.target_stage)
+
+        self.container_id, internal_tag = _run_container_build(
+            container_runtime,
+            rootdir,
+            ctrfile,
+            None,
+            build_args,
+            self.image_format,
+            self.add_build_tags,
+        )
+        assert self._build_tag == internal_tag
 
 
 @dataclass(frozen=True)
@@ -918,7 +1009,7 @@ class ContainerData:
     #: the testinfra connection to the running container
     connection: Any
     #: the container data class that has been used in this test
-    container: Union[Container, DerivedContainer]
+    container: Union[Container, DerivedContainer, MultiStageContainer]
     #: any ports that are exposed by this container
     forwarded_ports: List[PortForwarding]
 
@@ -965,50 +1056,65 @@ def container_to_pytest_param(
 def container_and_marks_from_pytest_param(
     ctr_or_param: Container,
 ) -> Tuple[Container, Literal[None]]:
-    ...
+    ...  # pragma: no cover
 
 
 @overload
 def container_and_marks_from_pytest_param(
     ctr_or_param: DerivedContainer,
 ) -> Tuple[DerivedContainer, Literal[None]]:
-    ...
+    ...  # pragma: no cover
+
+
+@overload
+def container_and_marks_from_pytest_param(
+    ctr_or_param: MultiStageContainer,
+) -> Tuple[MultiStageContainer, Literal[None]]:
+    ...  # pragma: no cover
 
 
 @overload
 def container_and_marks_from_pytest_param(
     ctr_or_param: _pytest.mark.ParameterSet,
 ) -> Tuple[
-    Union[Container, DerivedContainer],
+    Union[Container, DerivedContainer, MultiStageContainer],
     Optional[Collection[Union[_pytest.mark.MarkDecorator, _pytest.mark.Mark]]],
 ]:
-    ...
+    ...  # pragma: no cover
 
 
 def container_and_marks_from_pytest_param(
     ctr_or_param: Union[
-        _pytest.mark.ParameterSet, Container, DerivedContainer
+        _pytest.mark.ParameterSet,
+        Container,
+        DerivedContainer,
+        MultiStageContainer,
     ],
 ) -> Tuple[
-    Union[Container, DerivedContainer],
+    Union[Container, DerivedContainer, MultiStageContainer],
     Optional[Collection[Union[_pytest.mark.MarkDecorator, _pytest.mark.Mark]]],
 ]:
-    """Extracts the :py:class:`~pytest_container.container.Container` or
-    :py:class:`~pytest_container.container.DerivedContainer` and the
+    """Extracts the :py:class:`~pytest_container.container.Container`,
+    :py:class:`~pytest_container.container.DerivedContainer` or
+    :py:class:`~pytest_container.container.MultiStageContainer` and the
     corresponding marks from a `pytest.param
     <https://docs.pytest.org/en/stable/reference.html?#pytest.param>`_ and
     returns both.
 
     If ``param`` is either a :py:class:`~pytest_container.container.Container`
-    or a :py:class:`~pytest_container.container.DerivedContainer`, then param is
-    returned directly and the second return value is ``None``.
+    or a :py:class:`~pytest_container.container.DerivedContainer` or a
+    :py:class:`~pytest_container.container.MultiStageContainer`, then ``param``
+    is returned directly and the second return value is ``None``.
 
     """
-    if isinstance(ctr_or_param, (Container, DerivedContainer)):
+    if isinstance(
+        ctr_or_param, (Container, DerivedContainer, MultiStageContainer)
+    ):
         return ctr_or_param, None
 
     if len(ctr_or_param.values) > 0 and isinstance(
-        ctr_or_param.values[0], (Container, DerivedContainer)
+        ctr_or_param.values[0],
+        (Container, DerivedContainer, MultiStageContainer),
     ):
         return ctr_or_param.values[0], ctr_or_param.marks
 
@@ -1052,7 +1158,7 @@ class ContainerLauncher:
     """
 
     #: The container that will be launched
-    container: Union[Container, DerivedContainer]
+    container: Union[Container, DerivedContainer, MultiStageContainer]
 
     #: The container runtime via which the container will be launched
     container_runtime: OciRuntimeBase
@@ -1082,7 +1188,7 @@ class ContainerLauncher:
 
     @staticmethod
     def from_pytestconfig(
-        container: Union[Container, DerivedContainer],
+        container: Union[Container, DerivedContainer, MultiStageContainer],
         container_runtime: OciRuntimeBase,
         pytestconfig: Config,
         container_name: str = "",

pytest_container/pod.py

@@ -19,6 +19,7 @@
 from pytest_container.container import create_host_port_port_forward
 from pytest_container.container import DerivedContainer
 from pytest_container.container import lock_host_port_search
+from pytest_container.container import MultiStageContainer
 from pytest_container.helpers import get_extra_build_args
 from pytest_container.helpers import get_extra_pod_create_args
 from pytest_container.helpers import get_extra_run_args
@@ -39,7 +40,7 @@ class Pod:
     """
 
     #: containers belonging to the pod
-    containers: List[Union[DerivedContainer, Container]]
+    containers: List[Union[MultiStageContainer, DerivedContainer, Container]]
 
     #: ports exposed by the pod
     forwarded_ports: List[PortForwarding] = field(default_factory=list)

source/api.rst

@@ -8,6 +8,8 @@ The container module
 .. automodule:: pytest_container.container
    :members:
    :undoc-members:
+   :private-members:
+   :show-inheritance:
 
 
 The pod module

source/fixtures.rst

@@ -45,5 +45,5 @@ directive is only supported for docker images. While this is the default with
 :command:`docker`, :command:`buildah` will by default build images in the
 ``OCIv1`` format which does **not** support ``HEALTHCHECK``. To ensure that your
 created container includes the ``HEALTHCHECK``, set the attribute
-:py:attr:`~pytest_container.container.DerivedContainer.image_format` to
+:py:attr:`~pytest_container.container._ContainerForBuild.image_format` to
 :py:attr:`~pytest_container.container.ImageFormat.DOCKER`.