From f5881846d6fdba58868af62a26e9357d1ee2c612 Mon Sep 17 00:00:00 2001 From: Priyanka Saggu Date: Fri, 8 Mar 2024 03:51:27 -0500 Subject: [PATCH 01/12] Add SLCC branches to gh workflows --- .github/workflows/cleanup-staging.yml | 2 ++ .github/workflows/obs_build.yml | 2 ++ .github/workflows/update-cr-project.yml | 2 ++ .github/workflows/update-deployment-branches.yml | 2 ++ 4 files changed, 8 insertions(+) diff --git a/.github/workflows/cleanup-staging.yml b/.github/workflows/cleanup-staging.yml index b7a639762..9bd109afd 100644 --- a/.github/workflows/cleanup-staging.yml +++ b/.github/workflows/cleanup-staging.yml @@ -19,6 +19,8 @@ jobs: - 6 - 5 - Tumbleweed + - SLCC-free + - SLCC-paid steps: # we need all branches for the build checks diff --git a/.github/workflows/obs_build.yml b/.github/workflows/obs_build.yml index bdd4464d4..a8f29afd7 100644 --- a/.github/workflows/obs_build.yml +++ b/.github/workflows/obs_build.yml @@ -16,6 +16,8 @@ jobs: - 6 - 5 - Tumbleweed + - SLCC-free + - SLCC-paid steps: # we need all branches for the build checks diff --git a/.github/workflows/update-cr-project.yml b/.github/workflows/update-cr-project.yml index 71437ca96..3030f8c71 100644 --- a/.github/workflows/update-cr-project.yml +++ b/.github/workflows/update-cr-project.yml @@ -21,6 +21,8 @@ jobs: - 6 - 5 - Tumbleweed + - SLCC-free + - SLCC-paid steps: # we need all branches for the build checks diff --git a/.github/workflows/update-deployment-branches.yml b/.github/workflows/update-deployment-branches.yml index a37fecd1e..8347a2442 100644 --- a/.github/workflows/update-deployment-branches.yml +++ b/.github/workflows/update-deployment-branches.yml @@ -23,6 +23,8 @@ jobs: - 6 - 5 - Tumbleweed + - SLCC-free + - SLCC-paid steps: - uses: actions/checkout@v4 From 99205569ce7be13c0de4a52d470e819a17118b82 Mon Sep 17 00:00:00 2001 From: Priyanka Saggu Date: Fri, 8 Mar 2024 05:54:54 -0500 Subject: [PATCH 02/12] enable build recipes for nginx and postgresql16 for SLCC --- src/bci_build/package/appcontainers.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/bci_build/package/appcontainers.py b/src/bci_build/package/appcontainers.py index bbea2b392..39b5e7a4f 100644 --- a/src/bci_build/package/appcontainers.py +++ b/src/bci_build/package/appcontainers.py @@ -2,6 +2,7 @@ from pathlib import Path +from bci_build.package import ALL_BASE_OS_VERSIONS from bci_build.package import ALL_NONBASE_OS_VERSIONS from bci_build.package import CAN_BE_LATEST_OS_VERSION from bci_build.package import DOCKERFILE_RUN @@ -134,7 +135,7 @@ def _envsubst_pkg_name(os_version: OsVersion) -> str: CMD /usr/lib/dirsrv/dscontainer -H """, ) - for os_version in ALL_NONBASE_OS_VERSIONS + for os_version in ALL_BASE_OS_VERSIONS ] @@ -194,7 +195,14 @@ def _envsubst_pkg_name(os_version: OsVersion) -> str: ) for ver, os_version in ( [(15, variant) for variant in (OsVersion.SP5, OsVersion.TUMBLEWEED)] - + [(16, variant) for variant in (OsVersion.SP6, OsVersion.TUMBLEWEED)] + + [ + (16, variant) + for variant in ( + OsVersion.SLE16_0, + OsVersion.SP6, + OsVersion.TUMBLEWEED, + ) + ] ) + [(pg_ver, OsVersion.TUMBLEWEED) for pg_ver in (14, 13, 12)] ] From 61ac75b4e8968176787f52fb0565f6557bd54354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20=C4=8Cerm=C3=A1k?= Date: Thu, 11 Apr 2024 17:50:28 +0200 Subject: [PATCH 03/12] Introduce SLCC_FREE & SLCC_PAID as part of SLFO - adjust build tags & labels - move distribution_base_name into OsVersion class --- src/bci_build/package/__init__.py | 143 ++++++++++++++++++++---------- src/staging/bot.py | 28 ++++-- 2 files changed, 116 insertions(+), 55 deletions(-) diff --git a/src/bci_build/package/__init__.py b/src/bci_build/package/__init__.py index 061bae540..f84557607 100644 --- a/src/bci_build/package/__init__.py +++ b/src/bci_build/package/__init__.py @@ -136,6 +136,10 @@ class OsVersion(enum.Enum): SLE16_0 = "16.0" #: openSUSE Tumbleweed TUMBLEWEED = "Tumbleweed" + #: SUSE Linux Container Collection - free stream + SLCC_FREE = "SLCC-free" + #: SUSE Linux Container Collection - paid stream + SLCC_PAID = "SLCC-paid" @staticmethod def parse(val: str) -> OsVersion: @@ -147,6 +151,17 @@ def parse(val: str) -> OsVersion: def __str__(self) -> str: return str(self.value) + @property + def distribution_base_name(self) -> str: + if self.is_tumbleweed: + return "openSUSE Tumbleweed" + elif self.is_ltss: + return "SLE LTSS" + elif self.is_sle15: + return "SLE" + + return "SUSE Linux Framework One" + @property def pretty_print(self) -> str: if self.value in (OsVersion.TUMBLEWEED.value, OsVersion.SLE16_0.value): @@ -162,6 +177,7 @@ def pretty_os_version_no_dash(self) -> str: if self.is_slfo: return "Framework One" + assert self.is_sle15 return f"15 SP{self.value}" @property @@ -200,7 +216,11 @@ def is_sle15(self) -> bool: @property def is_slfo(self) -> bool: - return self.value in (OsVersion.SLE16_0.value,) + return self.value in ( + OsVersion.SLCC_FREE.value, + OsVersion.SLCC_PAID.value, + OsVersion.SLE16_0.value, + ) @property def is_tumbleweed(self) -> bool: @@ -208,6 +228,10 @@ def is_tumbleweed(self) -> bool: @property def is_ltss(self) -> bool: + """Determines whether this OS is covered by a different EULA and is not + redistributable. + + """ return self in ALL_OS_LTSS_VERSIONS @property @@ -218,7 +242,7 @@ def os_version(self) -> str: """ if self.is_sle15: return f"15.{str(self.value)}" - if self.value == OsVersion.SLE16_0.value: + if self.is_slfo: return "16.0" # Tumbleweed rolls too fast, just use latest return "latest" @@ -231,25 +255,30 @@ def has_container_suseconnect(self) -> bool: def eula_package_names(self) -> tuple[str, ...]: if self.is_sle15: return ("skelcd-EULA-bci",) - # if self.is_slcc: - # return (f"skelcd-EULA-{str(self.value).lower()}",) + if self.is_slfo: + return (f"skelcd-EULA-{str(self.value).lower()}",) return () @property def release_package_names(self) -> tuple[str, ...]: if self.value == OsVersion.TUMBLEWEED.value: return ("openSUSE-release", "openSUSE-release-appliance-docker") - if self.value == OsVersion.SLE16_0.value: - return ("ALP-dummy-release",) if self.is_ltss: return ("sles-ltss-release",) - # if self.is_slcc: - # return (f"{str(self.value).lower()}-release",) + if self.is_slfo: + return (f"{str(self.value).lower()}-release",) assert self.is_sle15 return ("sles-release",) +SLCC_OS_VERSIONS: list[OsVersion] = [ + OsVersion.SLCC_FREE, + OsVersion.SLCC_PAID, + OsVersion.SLE16_0, +] + + #: Operating system versions that have the label ``com.suse.release-stage`` set #: to ``released``. RELEASED_OS_VERSIONS: list[OsVersion] = [ @@ -264,6 +293,8 @@ def release_package_names(self) -> tuple[str, ...]: ALL_NONBASE_OS_VERSIONS: list[OsVersion] = [ OsVersion.SP6, OsVersion.TUMBLEWEED, + OsVersion.SLCC_FREE, + OsVersion.SLCC_PAID, ] # For which versions to create Base Container Images? @@ -271,8 +302,8 @@ def release_package_names(self) -> tuple[str, ...]: OsVersion.SP5, OsVersion.SP6, OsVersion.TUMBLEWEED, - OsVersion.SLE16_0, -] +] + SLCC_OS_VERSIONS + # List of SPs that are already under LTSS ALL_OS_LTSS_VERSIONS: list[OsVersion] = [OsVersion.SP3, OsVersion.SP4] @@ -360,11 +391,16 @@ def __post_init__(self) -> None: def _build_tag_prefix(os_version: OsVersion) -> str: if os_version == OsVersion.TUMBLEWEED: return "opensuse/bci" + if os_version == OsVersion.SLE16_0: + return "suse/sle16" + if os_version == OsVersion.SLCC_PAID: + return "suse/supported" if os_version == OsVersion.SP3: return "suse/ltss/sle15.3" if os_version == OsVersion.SP4: return "suse/ltss/sle15.4" + # remaining SLE SPs + SLCC free use BCI as the prefix return "bci" @@ -595,7 +631,7 @@ def build_name(self) -> str | None: @property def build_version(self) -> str | None: - if self.os_version not in (OsVersion.TUMBLEWEED, OsVersion.SLE16_0): + if self.os_version.is_sle15: epoch = "" if self.os_epoch: epoch = f"{self.os_epoch}." @@ -613,17 +649,6 @@ def build_release(self) -> str | None: else None ) - @property - def distribution_base_name(self) -> str: - if self.os_version.is_tumbleweed: - return "openSUSE Tumbleweed" - elif self.os_version.is_ltss: - return "SLE LTSS" - elif self.os_version.is_sle15 or self.os_version.is_slfo: - return "SLE" - - raise NotImplementedError(f"Unknown os_version: {self.os_version}") - @property def eula(self) -> str: """EULA covering this image. can be ``sle-eula`` or ``sle-bci``.""" @@ -800,7 +825,7 @@ def _from_image(self) -> str | None: if self.os_version == OsVersion.TUMBLEWEED: return "opensuse/tumbleweed:latest" if self.os_version == OsVersion.SLE16_0: - return f"{_build_tag_prefix(self.os_version)}/bci-base:latest" + return f"{_build_tag_prefix(self.os_version)}/base:latest" if self.os_version in ALL_OS_LTSS_VERSIONS: return f"{_build_tag_prefix(self.os_version)}/sle15:15.{self.os_version}" if self.image_type == ImageType.APPLICATION: @@ -1060,7 +1085,7 @@ def description(self) -> str: description_formatters = { "pretty_name": self.pretty_name, "based_on_container": ( - f"based on the {self.distribution_base_name} Base Container Image" + f"based on the {self.os_version.distribution_base_name} Base Container Image" ), "podman_only": "This container is only supported with podman.", "privileged_only": "This container is only supported in privileged mode.", @@ -1077,12 +1102,12 @@ def title(self) -> str: label. It is generated from :py:attr:`BaseContainerImage.pretty_name` as - follows: ``"{distribution_base_name} BCI {self.pretty_name}"``, where - ``distribution_base_name`` is taken from - :py:attr:`~ImageProperties.distribution_base_name`. + follows: ``"{distribution_base_name}(if ! SLLC: BCI else '') + {self.pretty_name}"``, where ``distribution_base_name`` is taken from + :py:attr:`~OsVersion.distribution_base_name`. """ - return f"{self.distribution_base_name} BCI {self.pretty_name}" + return f"{self.os_version.distribution_base_name}{' BCI' if not self.os_version.is_slfo else ''} {self.pretty_name}" @property def readme_path(self) -> str: @@ -1167,22 +1192,29 @@ def labelprefix(self) -> str: :py:attr:`~BaseContainerImage.custom_labelprefix_end`. """ - labelprefix = "com.suse" - if self.os_version.is_tumbleweed: - labelprefix = "org.opensuse" - return ( - labelprefix - + "." - + ( - { - ImageType.SLE_BCI: "bci", - ImageType.APPLICATION: "application", - ImageType.LTSS: "sle", - }[self.image_type] + if self.os_version.is_opensuse: + labelprefix = "org.opensuse." + elif self.os_version.is_sle15: + labelprefix = "com.suse." + else: + assert self.os_version.is_slfo + labelprefix = ( + "com.suse.slfo." + + { + OsVersion.SLCC_FREE: "free", + OsVersion.SLCC_PAID: "supported", + OsVersion.SLE16_0: "sle16", + }[self.os_version] ) - + "." - + (self.custom_labelprefix_end or self.name) - ) + + if not self.os_version.is_slcc: + labelprefix += { + ImageType.SLE_BCI: "bci", + ImageType.APPLICATION: "application", + ImageType.LTSS: "sle", + }[self.image_type] + + return f"{labelprefix}.{(self.custom_labelprefix_end or self.name)}" @property def kiwi_version(self) -> str: @@ -1465,6 +1497,8 @@ def __post_init__(self) -> None: def _registry_prefix(self) -> str: if self.os_version.is_tumbleweed: return "opensuse" + if self.os_version.is_slfo: + return _build_tag_prefix(self.os_version) return "suse" @property @@ -1473,7 +1507,7 @@ def image_type(self) -> ImageType: @property def title(self) -> str: - return f"{self.distribution_base_name} {self.pretty_name}" + return f"{self.os_version.distribution_base_name} {self.pretty_name}" @property def eula(self) -> str: @@ -1487,7 +1521,7 @@ def eula(self) -> str: class OsContainer(BaseContainerImage): @staticmethod def version_to_container_os_version(os_version: OsVersion) -> str: - if os_version in (OsVersion.TUMBLEWEED, OsVersion.SLE16_0): + if os_version == OsVersion.TUMBLEWEED or os_version.is_slfo: return "latest" return f"15.{os_version}" @@ -1506,16 +1540,24 @@ def image_type(self) -> ImageType: return ImageType.SLE_BCI + @staticmethod + def build_tag_name_prefix(os_version: OsVersion) -> str: + """Prefix that is inserted in front of the name into the build tag""" + return "" if os_version.is_slfo else "bci-" + @property def build_tags(self) -> list[str]: tags: list[str] = [] + prefix = self.build_tag_name_prefix(self.os_version) for name in [self.name] + self.additional_names: tags += [ - f"{self._registry_prefix}/bci-{name}:%OS_VERSION_ID_SP%", - f"{self._registry_prefix}/bci-{name}:{self.image_ref_name}", + f"{self._registry_prefix}/{prefix}-{name}:%OS_VERSION_ID_SP%", + f"{self._registry_prefix}/{prefix}-{name}:{self.image_ref_name}", ] + ( - [f"{self._registry_prefix}/bci-{name}:latest"] if self.is_latest else [] + [f"{self._registry_prefix}/{prefix}{name}:latest"] + if self.is_latest + else [] ) return tags @@ -1525,7 +1567,10 @@ def image_ref_name(self) -> str: @property def reference(self) -> str: - return f"{self.registry}/{self._registry_prefix}/bci-{self.name}:{self.image_ref_name}" + return ( + f"{self.registry}/{self._registry_prefix}/" + + f"{self.build_tag_name_prefix(self.os_version)}{self.name}:{self.image_ref_name}" + ) @property def pretty_reference(self) -> str: diff --git a/src/staging/bot.py b/src/staging/bot.py index e77f3e409..ef3b9ded8 100644 --- a/src/staging/bot.py +++ b/src/staging/bot.py @@ -16,7 +16,9 @@ from functools import reduce from typing import ClassVar from typing import Literal +from typing import NoReturn from typing import TypedDict +from typing import overload import aiofiles.os import aiofiles.tempfile @@ -71,11 +73,25 @@ OS_VERSION_NEEDS_BASE_CONTAINER: tuple[OsVersion, ...] = () +@overload +def _get_base_image_prj_pkg( + os_version: Literal[ + OsVersion.SLCC_FREE, + OsVersion.SLCC_PAID, + OsVersion.SLE16_0, + ], +) -> NoReturn: ... + + +@overload +def _get_base_image_prj_pkg(os_version: OsVersion) -> tuple[str, str]: ... + + def _get_base_image_prj_pkg(os_version: OsVersion) -> tuple[str, str]: if os_version == OsVersion.TUMBLEWEED: return "openSUSE:Factory", "opensuse-tumbleweed-image" - if os_version == OsVersion.SLE16_0: - raise ValueError("The SLFO base container is provided by BCI") + if os_version.is_slfo: + raise ValueError("The SLFO base containers are provided by the project itself") return f"SUSE:SLE-15-SP{os_version}:Update", "sles15-image" @@ -83,7 +99,7 @@ def _get_base_image_prj_pkg(os_version: OsVersion) -> tuple[str, str]: def _get_bci_project_name(os_version: OsVersion) -> str: prj_suffix = ( os_version - if os_version in (OsVersion.TUMBLEWEED, OsVersion.SLE16_0) + if (os_version == OsVersion.TUMBLEWEED or os_version.is_slfo) else "SLE-15-SP" + str(os_version) ) return f"devel:BCI:{prj_suffix}" @@ -199,10 +215,10 @@ def _bcis(self) -> Generator[BaseContainerImage, None, None]: def _generate_project_name(self, prefix: str) -> str: assert self.osc_username res = f"home:{self.osc_username}:{prefix}:" - if self.os_version in (OsVersion.TUMBLEWEED, OsVersion.SLE16_0): - res += str(self.os_version) - else: + if self.os_version.is_sle15: res += f"SLE-15-SP{str(self.os_version)}" + else: + res += str(self.os_version) return res @property From 614441d464d045c04ea85f8b152a9e05d19b29a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20=C4=8Cerm=C3=A1k?= Date: Tue, 23 Apr 2024 14:09:06 +0200 Subject: [PATCH 04/12] Adjust containers recipes to new SLCC_* streams --- src/bci_build/package/__init__.py | 2 +- src/bci_build/package/appcontainers.py | 28 ++++++++++++++++-------- src/bci_build/package/basecontainers.py | 14 ++++++++++-- src/bci_build/package/gcc.py | 16 +++++++------- src/bci_build/package/golang.py | 23 ++++++++++++++++---- src/bci_build/package/openjdk.py | 3 ++- src/bci_build/package/python.py | 29 +++++++++++++++++++------ 7 files changed, 83 insertions(+), 32 deletions(-) diff --git a/src/bci_build/package/__init__.py b/src/bci_build/package/__init__.py index f84557607..d0bc6f9ce 100644 --- a/src/bci_build/package/__init__.py +++ b/src/bci_build/package/__init__.py @@ -1635,7 +1635,7 @@ def generate_disk_size_constraints(size_gb: int) -> str: f"{bci.uid}-{bci.os_version.pretty_print.lower()}": bci for bci in ( *BASE_CONTAINERS, - PYTHON_3_12_CONTAINERS, + *PYTHON_3_12_CONTAINERS, *PYTHON_3_6_CONTAINERS, *PYTHON_3_11_CONTAINERS, *PYTHON_TW_CONTAINERS, diff --git a/src/bci_build/package/appcontainers.py b/src/bci_build/package/appcontainers.py index 39b5e7a4f..9e84e8abf 100644 --- a/src/bci_build/package/appcontainers.py +++ b/src/bci_build/package/appcontainers.py @@ -6,6 +6,7 @@ from bci_build.package import ALL_NONBASE_OS_VERSIONS from bci_build.package import CAN_BE_LATEST_OS_VERSION from bci_build.package import DOCKERFILE_RUN +from bci_build.package import SLCC_OS_VERSIONS from bci_build.package import ApplicationStackContainer from bci_build.package import BuildType from bci_build.package import OsContainer @@ -42,7 +43,7 @@ def _envsubst_pkg_name(os_version: OsVersion) -> str: name="pcp", pretty_name="Performance Co-Pilot (pcp)", custom_description="{pretty_name} container {based_on_container}. {podman_only}", - from_image=f"{_build_tag_prefix(os_version)}/bci-init:{OsContainer.version_to_container_os_version(os_version)}", + from_image=f"{_build_tag_prefix(os_version)}/{OsContainer.build_tag_name_prefix(os_version)}init:{OsContainer.version_to_container_os_version(os_version)}", os_version=os_version, is_latest=os_version in CAN_BE_LATEST_OS_VERSION, support_level=SupportLevel.L3, @@ -194,11 +195,19 @@ def _envsubst_pkg_name(os_version: OsVersion) -> str: """, ) for ver, os_version in ( - [(15, variant) for variant in (OsVersion.SP5, OsVersion.TUMBLEWEED)] + [ + (15, variant) + for variant in ( + OsVersion.SP5, + OsVersion.TUMBLEWEED, + OsVersion.SLCC_PAID, + ) + ] + [ (16, variant) for variant in ( - OsVersion.SLE16_0, + OsVersion.SLCC_FREE, + OsVersion.SLCC_PAID, OsVersion.SP6, OsVersion.TUMBLEWEED, ) @@ -299,6 +308,7 @@ def _generate_prometheus_family_healthcheck(port: int) -> str: custom_end=_generate_prometheus_family_healthcheck(_BLACKBOX_PORT), ) for os_version in ALL_NONBASE_OS_VERSIONS + if not os_version.is_slfo ] _GRAFANA_FILES = {} @@ -403,7 +413,7 @@ def _get_nginx_kwargs(os_version: OsVersion): pretty_name="NGINX for SUSE RMT", **_get_nginx_kwargs(os_version), ) - for os_version in (OsVersion.SP6,) + for os_version in (OsVersion.SP6, *SLCC_OS_VERSIONS) ] + [ ApplicationStackContainer( name="nginx", @@ -421,7 +431,7 @@ def _get_nginx_kwargs(os_version: OsVersion): support_level=SupportLevel.L3, pretty_name=f"{os_version.pretty_os_version_no_dash} with Git", custom_description="A micro environment with Git {based_on_container}.", - from_image=f"{_build_tag_prefix(os_version)}/bci-micro:{OsContainer.version_to_container_os_version(os_version)}", + from_image=f"{_build_tag_prefix(os_version)}/{OsContainer.build_tag_name_prefix(os_version)}micro:{OsContainer.version_to_container_os_version(os_version)}", build_recipe_type=BuildType.KIWI, is_latest=os_version in CAN_BE_LATEST_OS_VERSION, version="%%git_version%%", @@ -440,7 +450,7 @@ def _get_nginx_kwargs(os_version: OsVersion): "git-core", "openssh-clients", ) - + (() if os_version == OsVersion.TUMBLEWEED else ("skelcd-EULA-bci",)) + + os_version.eula_package_names ], # intentionally empty config_sh_script=""" @@ -455,7 +465,7 @@ def _get_nginx_kwargs(os_version: OsVersion): name="registry", package_name="distribution-image", pretty_name="OCI Container Registry (Distribution)", - from_image=f"{_build_tag_prefix(os_version)}/bci-micro:{OsContainer.version_to_container_os_version(os_version)}", + from_image=f"{_build_tag_prefix(os_version)}/{OsContainer.build_tag_name_prefix(os_version)}micro:{OsContainer.version_to_container_os_version(os_version)}", os_version=os_version, is_latest=os_version in CAN_BE_LATEST_OS_VERSION, version="%%registry_version%%", @@ -494,7 +504,7 @@ def _get_nginx_kwargs(os_version: OsVersion): ApplicationStackContainer( name="helm", pretty_name="Kubernetes Package Manager", - from_image=f"{_build_tag_prefix(os_version)}/bci-micro:{OsContainer.version_to_container_os_version(os_version)}", + from_image=f"{_build_tag_prefix(os_version)}/{OsContainer.build_tag_name_prefix(os_version)}micro:{OsContainer.version_to_container_os_version(os_version)}", os_version=os_version, is_latest=os_version in CAN_BE_LATEST_OS_VERSION, version=get_pkg_version("helm", os_version), @@ -520,7 +530,7 @@ def _get_nginx_kwargs(os_version: OsVersion): ApplicationStackContainer( name="trivy", pretty_name="Container Vulnerability Scanner", - from_image=f"{_build_tag_prefix(os_version)}/bci-micro:{OsContainer.version_to_container_os_version(os_version)}", + from_image=f"{_build_tag_prefix(os_version)}/{OsContainer.build_tag_name_prefix(os_version)}micro:{OsContainer.version_to_container_os_version(os_version)}", os_version=os_version, is_latest=os_version in CAN_BE_LATEST_OS_VERSION, version="%%trivy_version%%", diff --git a/src/bci_build/package/basecontainers.py b/src/bci_build/package/basecontainers.py index f9edc16ab..444314958 100644 --- a/src/bci_build/package/basecontainers.py +++ b/src/bci_build/package/basecontainers.py @@ -206,7 +206,12 @@ def _get_minimal_kwargs(os_version: OsVersion): Package(name, pkg_type=PackageType.BOOTSTRAP) for name in os_version.release_package_names ] - if os_version in (OsVersion.TUMBLEWEED, OsVersion.SLE16_0): + if os_version in ( + OsVersion.TUMBLEWEED, + OsVersion.SLCC_FREE, + OsVersion.SLCC_PAID, + OsVersion.SLE16_0, + ): package_list.append(Package("rpm", pkg_type=PackageType.BOOTSTRAP)) else: # in SLE15, rpm still depends on Perl. @@ -215,8 +220,10 @@ def _get_minimal_kwargs(os_version: OsVersion): for name in ("rpm-ndb", "perl-base") ] + micro_name = "micro" if os_version.is_slfo else "bci-micro" + kwargs = { - "from_image": f"{_build_tag_prefix(os_version)}/bci-micro:{OsContainer.version_to_container_os_version(os_version)}", + "from_image": f"{_build_tag_prefix(os_version)}/{micro_name}:{OsContainer.version_to_container_os_version(os_version)}", "pretty_name": f"{os_version.pretty_os_version_no_dash} Minimal", "package_list": package_list, } @@ -291,6 +298,9 @@ def _get_minimal_kwargs(os_version: OsVersion): if os_version == OsVersion.SLE16_0: prefix = "sle16" pretty_prefix = "SLE 16" + elif os_version.is_slfo and os_version != OsVersion.SLE16_0: + prefix = "slcc" + pretty_prefix = prefix.upper() else: prefix = "sle15" pretty_prefix = "SLE 15" diff --git a/src/bci_build/package/gcc.py b/src/bci_build/package/gcc.py index a1f6155aa..f102f7a5f 100644 --- a/src/bci_build/package/gcc.py +++ b/src/bci_build/package/gcc.py @@ -28,9 +28,9 @@ def _is_latest_gcc(os_version: OsVersion, gcc_version: _GCC_VERSIONS) -> bool: return True if os_version.is_sle15 and gcc_version == 13: return True - # if os_version in (OsVersion.SLCC_DEVELOPMENT, OsVersion.SLCC_PRODUCTION): - # assert gcc_version == 13 - # return True + if os_version.is_slfo: + assert gcc_version == 13 + return True return False @@ -39,9 +39,9 @@ def _is_main_gcc(os_version: OsVersion, gcc_version: _GCC_VERSIONS) -> bool: return True if os_version.is_sle15 and gcc_version == 7: return True - # if os_version in (OsVersion.SLCC_DEVELOPMENT, OsVersion.SLCC_PRODUCTION): - # assert gcc_version == 13 - # return True + if os_version.is_slfo: + assert gcc_version == 13 + return True return False @@ -98,8 +98,8 @@ def _is_main_gcc(os_version: OsVersion, gcc_version: _GCC_VERSIONS) -> bool: for (gcc_version, os_version) in ( (7, OsVersion.SP6), (13, OsVersion.SP6), - # (13, OsVersion.SLCC_DEVELOPMENT), - # (13, OsVersion.SLCC_PRODUCTION), + (13, OsVersion.SLCC_FREE), + (13, OsVersion.SLCC_PAID), (12, OsVersion.TUMBLEWEED), (13, OsVersion.TUMBLEWEED), (14, OsVersion.TUMBLEWEED), diff --git a/src/bci_build/package/golang.py b/src/bci_build/package/golang.py index df9aea497..c235ed7d7 100644 --- a/src/bci_build/package/golang.py +++ b/src/bci_build/package/golang.py @@ -93,18 +93,33 @@ def _get_golang_kwargs( **_get_golang_kwargs(ver, govariant, sle15sp), support_level=SupportLevel.L3, ) - for ver, govariant, sle15sp in product( - _GOLANG_VERSIONS, ("",), (OsVersion.SP6,) + for ver, govariant, sle15sp in list( + product( + _GOLANG_VERSIONS, + ("",), + (OsVersion.SP6, OsVersion.SLCC_PAID), + ) ) + + [(_GOLANG_VERSIONS[-1], "", OsVersion.SLCC_FREE)] ] + [ DevelopmentContainer( **_get_golang_kwargs(ver, govariant, sle15sp), support_level=SupportLevel.L3, ) - for ver, govariant, sle15sp in product( - _GOLANG_OPENSSL_VERSIONS, ("-openssl",), (OsVersion.SP6,) + for ver, govariant, sle15sp in list( + product(_GOLANG_OPENSSL_VERSIONS, ("-openssl",), (OsVersion.SP6,)) ) + # FIXME: add the remaining golang-openssl versions here once they are released + + [("1.21", "-openssl", OsVersion.SLCC_PAID)] + # only the latest version here + + [ + ( + _GOLANG_OPENSSL_VERSIONS[-1], + "-openssl", + OsVersion.SLCC_FREE, + ) + ] ] + [ DevelopmentContainer(**_get_golang_kwargs(ver, "", OsVersion.TUMBLEWEED)) diff --git a/src/bci_build/package/openjdk.py b/src/bci_build/package/openjdk.py index 2f238e152..06c7037a5 100644 --- a/src/bci_build/package/openjdk.py +++ b/src/bci_build/package/openjdk.py @@ -91,7 +91,8 @@ def _get_openjdk_kwargs( support_level=SupportLevel.L3, ) for os_version, devel in product( - (OsVersion.SP6, OsVersion.TUMBLEWEED), (True, False) + (OsVersion.SP6, OsVersion.SLCC_PRODUCTION, OsVersion.TUMBLEWEED), + (True, False), ) ] + [ diff --git a/src/bci_build/package/python.py b/src/bci_build/package/python.py index 6d4b2e52c..faca68805 100644 --- a/src/bci_build/package/python.py +++ b/src/bci_build/package/python.py @@ -109,7 +109,7 @@ def _get_python_kwargs(py3_ver: _PYTHON_VERSIONS, os_version: OsVersion): PYTHON_TW_CONTAINERS = ( PythonDevelopmentContainer( **_get_python_kwargs(pyver, OsVersion.TUMBLEWEED), - is_latest=pyver == _PYTHON_TW_VERSIONS[-1], + is_latest=(pyver == _PYTHON_TW_VERSIONS[-1]), package_name=f"python-{pyver}-image", ) for pyver in _PYTHON_TW_VERSIONS @@ -119,12 +119,27 @@ def _get_python_kwargs(py3_ver: _PYTHON_VERSIONS, os_version: OsVersion): PythonDevelopmentContainer( **_get_python_kwargs("3.11", os_version), package_name="python-3.11-image", + is_latest=( + (os_version in CAN_BE_LATEST_OS_VERSION) + and (os_version != OsVersion.SLCC_PAID) + ), ) - for os_version in (OsVersion.SP6,) + for os_version in (OsVersion.SP6, OsVersion.SLCC_PAID) ) -PYTHON_3_12_CONTAINERS = PythonDevelopmentContainer( - **_get_python_kwargs("3.12", OsVersion.SP6), - package_name="python-3.12-image", - is_latest=OsVersion.SP6 in CAN_BE_LATEST_OS_VERSION, -) + +PYTHON_3_12_CONTAINERS = [ + PythonDevelopmentContainer( + **_get_python_kwargs("3.12", os_version), + package_name="python-3.12-image", + # Technically it is the latest but we want to prefer the long term Python + # 3.11 for SLE, SLCC Paid & TW + is_latest=os_version in CAN_BE_LATEST_OS_VERSION + and os_version != OsVersion.SLCC_PAID, + ) + for os_version in ( + OsVersion.SP6, + OsVersion.SLCC_PAID, + OsVersion.SLCC_FREE, + ) +] From 52273bbd27094519194395071bf9ecc1d8e1ab90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20=C4=8Cerm=C3=A1k?= Date: Wed, 24 Apr 2024 09:28:11 +0200 Subject: [PATCH 05/12] Set SLFO OS tag to 16.0 --- src/bci_build/package/__init__.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/bci_build/package/__init__.py b/src/bci_build/package/__init__.py index d0bc6f9ce..34db4c582 100644 --- a/src/bci_build/package/__init__.py +++ b/src/bci_build/package/__init__.py @@ -1218,8 +1218,11 @@ def labelprefix(self) -> str: @property def kiwi_version(self) -> str: - if self.os_version in (OsVersion.TUMBLEWEED, OsVersion.SLE16_0): + if self.os_version == OsVersion.TUMBLEWEED: return str(datetime.datetime.now().year) + # FIXME: both should be handled better + if self.os_version.is_slfo: + return "16.0.0" return f"15.{int(self.os_version.value)}.0" @property @@ -1521,9 +1524,13 @@ def eula(self) -> str: class OsContainer(BaseContainerImage): @staticmethod def version_to_container_os_version(os_version: OsVersion) -> str: - if os_version == OsVersion.TUMBLEWEED or os_version.is_slfo: - return "latest" - return f"15.{os_version}" + if os_version.is_sle15: + return f"15.{os_version}" + if os_version.is_slfo: + # FIXME: + # we'll probably have to find a better way here + return "16.0" + return "latest" @property def uid(self) -> str: From 7ba4fbbe5164a750e5f2a86884b670dad1b3e42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20=C4=8Cerm=C3=A1k?= Date: Thu, 2 May 2024 09:47:51 +0200 Subject: [PATCH 06/12] [bot] adjust the obs project names --- src/staging/bot.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/staging/bot.py b/src/staging/bot.py index ef3b9ded8..71726110c 100644 --- a/src/staging/bot.py +++ b/src/staging/bot.py @@ -97,11 +97,13 @@ def _get_base_image_prj_pkg(os_version: OsVersion) -> tuple[str, str]: def _get_bci_project_name(os_version: OsVersion) -> str: - prj_suffix = ( - os_version - if (os_version == OsVersion.TUMBLEWEED or os_version.is_slfo) - else "SLE-15-SP" + str(os_version) - ) + if os_version.is_sle15: + prj_suffix = f"SLE-15-SP{os_version}" + elif os_version.is_slfo: + prj_suffix = str(os_version).replace("-", ":") + else: + prj_suffix = str(os_version) + return f"devel:BCI:{prj_suffix}" From e95e385608fbba6bb418bd291b5a96a4513a567c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20=C4=8Cerm=C3=A1k?= Date: Fri, 3 May 2024 11:02:15 +0200 Subject: [PATCH 07/12] Generate staging & CR project meta from a template instead of monkey patching devel:BCI --- source/api.rst | 8 ++ src/bci_build/package/__init__.py | 15 +++ src/staging/bot.py | 156 +++++++------------------- src/staging/project_setup.py | 174 +++++++++++++++++++++++++++++ tests/test_project_setup.py | 175 ++++++++++++++++++++++++++++++ 5 files changed, 412 insertions(+), 116 deletions(-) create mode 100644 src/staging/project_setup.py create mode 100644 tests/test_project_setup.py diff --git a/source/api.rst b/source/api.rst index f6f24a4ba..0faf03ce3 100644 --- a/source/api.rst +++ b/source/api.rst @@ -62,3 +62,11 @@ API Documentation .. automodule:: staging.build_result :members: :undoc-members: + + +:py:mod:`staging.project_setup` module +-------------------------------------- + +.. automodule:: staging.project_setup + :members: + :undoc-members: diff --git a/src/bci_build/package/__init__.py b/src/bci_build/package/__init__.py index 34db4c582..3ff8950d1 100644 --- a/src/bci_build/package/__init__.py +++ b/src/bci_build/package/__init__.py @@ -180,6 +180,21 @@ def pretty_os_version_no_dash(self) -> str: assert self.is_sle15 return f"15 SP{self.value}" + @property + def full_os_name(self) -> str: + if self.is_slcc: + assert isinstance(self.value, str) + return ( + "SUSE Linux Container Collection - " + + self.value.replace("SLCC-", "").replace("-", " ") + + " Stream" + ) + if self.is_sle15: + return f"{self.distribution_base_name} {self.pretty_os_version_no_dash}" + + assert self.is_opensuse + return self.distribution_base_name + @property def deployment_branch_name(self) -> str: if self.is_tumbleweed or self.is_slfo: diff --git a/src/staging/bot.py b/src/staging/bot.py index 71726110c..0c47d5249 100644 --- a/src/staging/bot.py +++ b/src/staging/bot.py @@ -14,6 +14,8 @@ from datetime import datetime from datetime import timedelta from functools import reduce +from io import BytesIO +from pathlib import Path from typing import ClassVar from typing import Literal from typing import NoReturn @@ -36,10 +38,12 @@ from bci_build.package import OsVersion from dotnet.updater import DOTNET_IMAGES from dotnet.updater import DotNetBCI -from staging.build_result import Arch from staging.build_result import PackageBuildResult from staging.build_result import PackageStatusCode from staging.build_result import RepositoryBuildResult +from staging.project_setup import ProjectType +from staging.project_setup import generate_meta +from staging.project_setup import generate_project_name from staging.user import User from staging.util import ensure_absent from staging.util import get_obs_project_url @@ -202,6 +206,16 @@ def __post_init__(self) -> None: if not self.osc_username: raise RuntimeError("osc_username is not set, cannot continue") + def _read_file_from_branch(self, branch_name: str, file_name: str) -> bytes: + tmp = BytesIO() + try: + git.Repo(Path(__file__).parent.parent.parent).commit(branch_name).tree[ + file_name + ].stream_data(tmp) + return tmp.getvalue() + except KeyError: + raise ValueError(f"File {file_name} not found in branch {branch_name}") + @property def _bcis(self) -> Generator[BaseContainerImage, None, None]: """Generator yielding all @@ -214,19 +228,10 @@ def _bcis(self) -> Generator[BaseContainerImage, None, None]: all_bcis.sort(key=lambda bci: bci.uid) return (bci for bci in all_bcis if bci.os_version == self.os_version) - def _generate_project_name(self, prefix: str) -> str: - assert self.osc_username - res = f"home:{self.osc_username}:{prefix}:" - if self.os_version.is_sle15: - res += f"SLE-15-SP{str(self.os_version)}" - else: - res += str(self.os_version) - return res - @property def continuous_rebuild_project_name(self) -> str: """The name of the continuous rebuild project on OBS.""" - return self._generate_project_name("BCI:CR") + return generate_project_name(self.os_version, ProjectType.CR, self.osc_username) @property def staging_project_name(self) -> str: @@ -240,7 +245,9 @@ def staging_project_name(self) -> str: - ``BRANCH``: :py:attr:`branch_name` """ - return self._generate_project_name("BCI:Staging") + ":" + self.branch_name + return generate_project_name( + self.os_version, ProjectType.STAGING, self.osc_username, self.branch_name + ) @property def staging_project_url(self) -> str: @@ -582,86 +589,18 @@ def _osc(self) -> str: "osc" if not self._osc_conf_file else f"osc --config={self._osc_conf_file}" ) - async def _generate_test_project_meta(self, target_project_name: str) -> ET.Element: - bci_devel_meta = ET.fromstring( - await _fetch_bci_devel_project_config(self.os_version, "meta") - ) - - # write the same project meta as devel:BCI, but replace the 'devel:BCI:*' - # with the target project name in the main element and in all repository - # path entries - bci_devel_meta.attrib["name"] = target_project_name - - # we will remove the helmchartsrepo as we do not need it - repo_names = [] - repos_to_remove = [] - - # ppc64le & s390x are mostly busted on TW and just cause pointless - # build failures, so we don't build them - # Also, we don't use the local architecture, so drop that one always - arches_to_drop = [str(Arch.LOCAL)] - arches_to_drop.extend( - [str(Arch.PPC64LE), str(Arch.S390X)] - if self.os_version == OsVersion.TUMBLEWEED - else [] - ) - - for elem in bci_devel_meta: - if elem.tag == "repository": - if "name" in elem.attrib: - if (name := elem.attrib["name"]) in ("helmcharts", "standard"): - if name == "helmcharts": - repos_to_remove.append(elem) - continue - - repo_names.append(name) - else: - raise ValueError( - f"Invalid element, missing 'name' attribute: {ET.tostring(elem).decode()}" - ) - - for repo_elem in elem.iter(tag="path"): - if ( - "project" in repo_elem.attrib - and "devel:BCI:" in repo_elem.attrib["project"] - ): - repo_elem.attrib["project"] = target_project_name - - arch_entries = list(elem.iter(tag="arch")) - for arch_entry in arch_entries: - if arch_entry.text in arches_to_drop: - elem.remove(arch_entry) - - container_repos = ("containerfile", "images") - if name in container_repos: - for repo_name in container_repos: - (bci_devel_prj_path := ET.Element("path")).attrib["project"] = ( - _get_bci_project_name(self.os_version) - ) - bci_devel_prj_path.attrib["repository"] = repo_name - - elem.insert(0, bci_devel_prj_path) - - self.repositories = repo_names - for repo_to_remove in repos_to_remove: - bci_devel_meta.remove(repo_to_remove) - - person = ET.Element( - "person", {"userid": self.osc_username, "role": "maintainer"} - ) - bci_devel_meta.append(person) - - return bci_devel_meta - async def _send_prj_meta( - self, target_project_name: str, prj_meta: ET.Element + self, target_project_name: str, prj_meta: ET.Element | str ) -> None: """Set the meta of the project on OBS with the name ``target_project_name`` to the config ``prj_meta``. """ async with aiofiles.tempfile.NamedTemporaryFile(mode="wb") as tmp_meta: - await tmp_meta.write(ET.tostring(prj_meta)) + if isinstance(prj_meta, str): + await tmp_meta.write(prj_meta.encode()) + else: + await tmp_meta.write(ET.tostring(prj_meta)) await tmp_meta.flush() async def _send_prj_meta(): @@ -680,52 +619,37 @@ async def write_cr_project_config(self) -> None: then its configuration (= ``meta`` in OBS jargon) will be updated. """ - meta = await self._generate_test_project_meta( - self.continuous_rebuild_project_name + prj_name, meta = generate_meta( + self.os_version, ProjectType.CR, self.osc_username ) - ( - scmsync := ET.Element("scmsync") - ).text = f"https://github.com/SUSE/bci-dockerfile-generator#{self.deployment_branch_name}" - meta.append(scmsync) - await self._send_prj_meta(self.continuous_rebuild_project_name, meta) + await self._send_prj_meta(prj_name, meta) async def write_staging_project_configs(self) -> None: """Submit the ``prjconf`` and ``meta`` to the test project on OBS. - The ``prjconf`` is taken directly from the development project on OBS - (``devel:BCI:*``). + The ``meta`` is generated using a template via + py.func:`~staging.project_setup.generate_project_name`. - The ``meta`` has to be modified slightly: + The ``prjconf`` is taken from the file:`_config` file in the deployment + branch. - - we remove the ``helmcharts`` repository (we don't create anything for - that repo, so not worth creating it) - - change the path from ``devel:BCI:*`` to the staging project name - - add the bot user as the maintainer (otherwise you can't do anything in - the project anymore…) - - Then we send the ``meta`` and then the ``prjconf``. """ - confs: _ProjectConfigs = {} - - async def _fetch_prjconf(): - confs["prjconf"] = await _fetch_bci_devel_project_config( - self.os_version, "prjconf" - ) - async def _fetch_prj(): - confs["meta"] = await self._generate_test_project_meta( - self.staging_project_name - ) + prj_name, prj_meta = generate_meta( + self.os_version, ProjectType.STAGING, self.osc_username, self.branch_name + ) - await asyncio.gather(_fetch_prj(), _fetch_prjconf()) + prjconf = self._read_file_from_branch( + f"origin/{self.deployment_branch_name}", "_config" + ) # First set the project meta! This will create the project if it does not # exist already, if we do it asynchronously, then the prjconf might be # written before the project exists, which fails - await self._send_prj_meta(self.staging_project_name, confs["meta"]) + await self._send_prj_meta(prj_name, prj_meta) - async with aiofiles.tempfile.NamedTemporaryFile(mode="w") as tmp_prjconf: - await tmp_prjconf.write(confs["prjconf"]) + async with aiofiles.tempfile.NamedTemporaryFile(mode="wb") as tmp_prjconf: + await tmp_prjconf.write(prjconf) await tmp_prjconf.flush() await self._run_cmd( f"{self._osc} meta prjconf --file={tmp_prjconf.name} {self.staging_project_name}" diff --git a/src/staging/project_setup.py b/src/staging/project_setup.py new file mode 100644 index 000000000..d97eb67af --- /dev/null +++ b/src/staging/project_setup.py @@ -0,0 +1,174 @@ +from enum import Enum +from enum import auto +from enum import unique + +import jinja2 + +from bci_build.package import OsVersion + +USERS_FOR_PRODUCTION = [ + "avicenzi", + "dirkmueller", + "dancermak", + "favogt", + "fcrozat", + "pvlasin", +] + +USERS_FOR_STAGING = ["avicenzi", "dancermak"] + +META_TEMPLATE = jinja2.Template(""" + {{ project_title }} +{% if project_description %} {{ project_description }}{% else %} {% endif %} +{% for user in maintainers %} +{% endfor %}{% if extra_header %} +{{ extra_header }}{% endif %} + + + + + + + + + + +{% for prj, repo in repository_paths %} +{% endfor %} x86_64 + aarch64 +{% if with_all_arches %} s390x + ppc64le{% endif %} + + + + + x86_64 + aarch64{% if with_all_arches %} + s390x + ppc64le{% endif %} + {% if with_helmcharts_repo %} + + + x86_64 + {% endif %} + + + + x86_64 + aarch64{% if with_all_arches %} + s390x + ppc64le{% endif %} + + +""") + + +@unique +class ProjectType(Enum): + DEVEL = auto() + CR = auto() + STAGING = auto() + + +def generate_project_name( + os_version: OsVersion, + project_type: ProjectType, + osc_username: str, + branch_name: str | None = None, +) -> str: + res = { + ProjectType.DEVEL: "devel:BCI:", + ProjectType.CR: f"home:{osc_username}:BCI:CR:", + ProjectType.STAGING: f"home:{osc_username}:BCI:Staging:", + }[project_type] + + if os_version.is_sle15: + res += f"SLE-15-SP{str(os_version)}" + elif os_version.is_slcc: + res += "SLCC:" + str(os_version).split("-", 1)[1] + else: + res += str(os_version) + + if project_type == ProjectType.STAGING: + if not branch_name: + raise ValueError("staging projects need a branch name") + res += ":" + branch_name + + return res + + +def generate_meta( + os_version: OsVersion, + project_type: ProjectType, + osc_username: str, + branch_name: None | str = None, +) -> tuple[str, str]: + prj_name = generate_project_name( + os_version, project_type, osc_username, branch_name + ) + + if project_type == ProjectType.DEVEL: + users = USERS_FOR_PRODUCTION + else: + users = USERS_FOR_STAGING + [osc_username] + + with_all_arches = (os_version != OsVersion.TUMBLEWEED) or ( + project_type in (ProjectType.CR, ProjectType.DEVEL) + ) + with_helmcharts_repo = project_type == ProjectType.DEVEL and not os_version.is_slcc + + repository_paths: tuple[tuple[str, str], ...] + if os_version.is_sle15: + repository_paths = ( + ("SUSE:Registry", "standard"), + (f"SUSE:SLE-15-SP{str(os_version)}:Update", "standard"), + ) + elif os_version.is_slcc: + repository_paths = (("SUSE:ALP:Source:Standard:Core:1.0:Build", "standard"),) + if project_type in (ProjectType.CR, ProjectType.STAGING): + repository_paths += ( + (generate_project_name(os_version, ProjectType.DEVEL, ""), "standard"), + ) + else: + repository_paths = ( + ("openSUSE:Factory", "images"), + ("openSUSE:Factory:ARM", "images"), + ("openSUSE:Factory:ARM", "standard"), + ("openSUSE:Factory:PowerPC", "standard"), + ("openSUSE:Factory:zSystems", "standard"), + ("openSUSE:Factory", "snapshot"), + ) + + if project_type == ProjectType.STAGING: + assert branch_name + description = f"Staging project for https://github.com/SUSE/BCI-dockerfile-generator/tree/{branch_name}" + title = "Staging project" + else: + description = { + ProjectType.DEVEL: "Development", + ProjectType.CR: "Continuous Rebuild", + }[project_type] + " project" + title = description + + description += f" for {os_version.full_os_name}" + title += f" for {os_version.full_os_name}" + + if not os_version.is_slcc and project_type == ProjectType.DEVEL: + description = "BCI " + description + title = "BCI " + title + + extra_header = None + if project_type == ProjectType.CR: + extra_header = f" https://github.com/SUSE/bci-dockerfile-generator#{os_version.deployment_branch_name}" + + return prj_name, META_TEMPLATE.render( + project_title=title, + project_description=description, + project_name=prj_name, + maintainers=users, + with_all_arches=with_all_arches, + with_helmcharts_repo=with_helmcharts_repo, + repository_paths=repository_paths, + description=description, + extra_header=extra_header, + ) diff --git a/tests/test_project_setup.py b/tests/test_project_setup.py new file mode 100644 index 000000000..ba1d8386e --- /dev/null +++ b/tests/test_project_setup.py @@ -0,0 +1,175 @@ +import pytest + +from bci_build.package import OsVersion +from staging.project_setup import ProjectType +from staging.project_setup import generate_meta + +_OSC_USERNAME = "foobar" + + +@pytest.mark.parametrize( + "os_version, project_type, branch_name, expected_prj_name, expected_meta", + [ + ( + OsVersion.SP6, + ProjectType.DEVEL, + None, + "devel:BCI:SLE-15-SP6", + """ + BCI Development project for SLE 15 SP6 + BCI Development project for SLE 15 SP6 + + + + + + + + + + + + + + + + + + + + x86_64 + aarch64 + s390x + ppc64le + + + + + x86_64 + aarch64 + s390x + ppc64le + + + + x86_64 + + + + + x86_64 + aarch64 + s390x + ppc64le + +""", + ), + ( + OsVersion.SP5, + ProjectType.CR, + None, + (prj_name := f"home:{_OSC_USERNAME}:BCI:CR:SLE-15-SP5"), + f""" + Continuous Rebuild project for SLE 15 SP5 + Continuous Rebuild project for SLE 15 SP5 + + + + + https://github.com/SUSE/bci-dockerfile-generator#sle15-sp5 + + + + + + + + + + + + + x86_64 + aarch64 + s390x + ppc64le + + + + + x86_64 + aarch64 + s390x + ppc64le + + + + + x86_64 + aarch64 + s390x + ppc64le + +""", + ), + ( + OsVersion.SLCC_FREE, + ProjectType.STAGING, + (branch := "pr-404"), + (prj_name := f"home:{_OSC_USERNAME}:BCI:Staging:SLCC:free:{branch}"), + f""" + Staging project for SUSE Linux Container Collection - free Stream + Staging project for https://github.com/SUSE/BCI-dockerfile-generator/tree/{branch} for SUSE Linux Container Collection - free Stream + + + + + + + + + + + + + + + + + x86_64 + aarch64 + s390x + ppc64le + + + + + x86_64 + aarch64 + s390x + ppc64le + + + + + x86_64 + aarch64 + s390x + ppc64le + +""", + ), + ], +) +def test_project_meta( + os_version: OsVersion, + project_type: ProjectType, + branch_name: str | None, + expected_prj_name: str, + expected_meta: str, +) -> None: + prj_name, prj_meta = generate_meta( + os_version, project_type, _OSC_USERNAME, branch_name + ) + assert prj_name == expected_prj_name + assert prj_meta == expected_meta From 30cec6904cad5d4bce1fbd32fdbef0d003496874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20=C4=8Cerm=C3=A1k?= Date: Fri, 3 May 2024 15:12:28 +0200 Subject: [PATCH 08/12] Add bot command to configure the devel project on OBS --- src/staging/bot.py | 91 ++++++++++++++++++++++++++---------- src/staging/project_setup.py | 12 ++++- tests/test_project_setup.py | 9 ++++ 3 files changed, 87 insertions(+), 25 deletions(-) diff --git a/src/staging/bot.py b/src/staging/bot.py index 0c47d5249..9aa1ccd1e 100644 --- a/src/staging/bot.py +++ b/src/staging/bot.py @@ -13,13 +13,14 @@ from dataclasses import field from datetime import datetime from datetime import timedelta +from enum import Enum +from enum import unique from functools import reduce from io import BytesIO from pathlib import Path from typing import ClassVar from typing import Literal from typing import NoReturn -from typing import TypedDict from typing import overload import aiofiles.os @@ -124,9 +125,10 @@ async def _fetch_bci_devel_project_config( return await response.text() -class _ProjectConfigs(TypedDict): - meta: ET.Element - prjconf: str +@unique +class ProjectConfig(Enum): + PRJCONF = "prjconf" + META = "prj" @dataclass @@ -216,6 +218,16 @@ def _read_file_from_branch(self, branch_name: str, file_name: str) -> bytes: except KeyError: raise ValueError(f"File {file_name} not found in branch {branch_name}") + @property + def _devel_project_prjconf(self) -> bytes: + """Returns the saved prjconf of the corresponding ``devel:BCI:$subname`` + project from git + + """ + return self._read_file_from_branch( + f"origin/{self.deployment_branch_name}", "_config" + ) + @property def _bcis(self) -> Generator[BaseContainerImage, None, None]: """Generator yielding all @@ -589,28 +601,38 @@ def _osc(self) -> str: "osc" if not self._osc_conf_file else f"osc --config={self._osc_conf_file}" ) - async def _send_prj_meta( - self, target_project_name: str, prj_meta: ET.Element | str + async def _send_prj_config( + self, + target_project_name: str, + config: ET.Element | str | bytes, + config_type: ProjectConfig = ProjectConfig.META, ) -> None: """Set the meta of the project on OBS with the name ``target_project_name`` to the config ``prj_meta``. """ + if isinstance(config, ET.Element) and config_type == ProjectConfig.PRJCONF: + raise ValueError("Cannot set the prjconf from a XML Element") + async with aiofiles.tempfile.NamedTemporaryFile(mode="wb") as tmp_meta: - if isinstance(prj_meta, str): - await tmp_meta.write(prj_meta.encode()) + if isinstance(config, str): + data = config.encode() + elif isinstance(config, ET.Element): + data = ET.tostring(config) else: - await tmp_meta.write(ET.tostring(prj_meta)) + data = config + + await tmp_meta.write(data) await tmp_meta.flush() - async def _send_prj_meta(): + async def _send_meta(): await self._run_cmd( - f"{self._osc} meta prj --file={tmp_meta.name} {target_project_name}" + f"{self._osc} meta {config_type.value} --file={tmp_meta.name} {target_project_name}" ) # obs sometimes dies setting the project meta with SQL errors 🤯 # so we just try again… - await retry_async_run_cmd(_send_prj_meta) + await retry_async_run_cmd(_send_meta) async def write_cr_project_config(self) -> None: """Send the configuration of the continuous rebuild project to OBS. @@ -622,7 +644,7 @@ async def write_cr_project_config(self) -> None: prj_name, meta = generate_meta( self.os_version, ProjectType.CR, self.osc_username ) - await self._send_prj_meta(prj_name, meta) + await self._send_prj_config(prj_name, meta, ProjectConfig.META) async def write_staging_project_configs(self) -> None: """Submit the ``prjconf`` and ``meta`` to the test project on OBS. @@ -639,21 +661,14 @@ async def write_staging_project_configs(self) -> None: self.os_version, ProjectType.STAGING, self.osc_username, self.branch_name ) - prjconf = self._read_file_from_branch( - f"origin/{self.deployment_branch_name}", "_config" - ) - # First set the project meta! This will create the project if it does not # exist already, if we do it asynchronously, then the prjconf might be # written before the project exists, which fails - await self._send_prj_meta(prj_name, prj_meta) + await self._send_prj_config(prj_name, prj_meta, ProjectConfig.META) - async with aiofiles.tempfile.NamedTemporaryFile(mode="wb") as tmp_prjconf: - await tmp_prjconf.write(prjconf) - await tmp_prjconf.flush() - await self._run_cmd( - f"{self._osc} meta prjconf --file={tmp_prjconf.name} {self.staging_project_name}" - ) + await self._send_prj_config( + prj_name, self._devel_project_prjconf, ProjectConfig.PRJCONF + ) def _osc_fetch_results_cmd(self, extra_osc_flags: str = "") -> str: return ( @@ -1442,6 +1457,26 @@ def get_packages_without_changelog_addition( if not changelog_updated ] + async def configure_devel_bci_project(self) -> None: + """Adjust to project meta of the devel project on OBS to match the + template generated via + :py:func:`staging.project_setup.generate_meta`. Additionally set the + project meta from the file :file:`_config` in the deployment branch and + set the `OSRT:Config` attribute for pkglistgen to function as expected. + + """ + prj_name, meta = generate_meta( + self.os_version, ProjectType.DEVEL, self.osc_username + ) + await self._send_prj_config(prj_name, meta, ProjectConfig.META) + + await self._send_prj_config( + prj_name, self._devel_project_prjconf, ProjectConfig.PRJCONF + ) + + await self._run_cmd(f"""{self._osc} meta attribute {prj_name} -a OSRT:Config --set 'main-repo = standard +pkglistgen-archs = ppc64le s390x aarch64 x86_64'""") + async def configure_devel_bci_package(self, package_name: str) -> None: bci = [b for b in self._bcis if b.package_name == package_name] @@ -1507,6 +1542,7 @@ def main() -> None: "add_changelog_entry", "changelog_check", "setup_obs_package", + "setup_obs_project", "find_missing_packages", ] @@ -1681,6 +1717,10 @@ def add_commit_message_arg(p: argparse.ArgumentParser) -> None: + [dotnet_img.package_name for dotnet_img in DOTNET_IMAGES], ) + subparsers.add_parser( + "setup_obs_project", help="Configure the devel project on OBS" + ) + subparsers.add_parser( "find_missing_packages", help="Find all packages that are in the deployment branch and are missing from `devel:BCI:*` on OBS", @@ -1828,6 +1868,9 @@ async def _setup_pkg_meta(): coro = _setup_pkg_meta() + elif action == "setup_obs_project": + coro = bot.configure_devel_bci_project() + elif action == "find_missing_packages": async def _pkgs_as_str() -> str: diff --git a/src/staging/project_setup.py b/src/staging/project_setup.py index d97eb67af..a92aa3cf1 100644 --- a/src/staging/project_setup.py +++ b/src/staging/project_setup.py @@ -46,7 +46,16 @@ aarch64{% if with_all_arches %} s390x ppc64le{% endif %} - {% if with_helmcharts_repo %} + {% if with_product_repo %} + + + + + x86_64 + aarch64{% if with_all_arches %} + s390x + ppc64le{% endif %} + {% endif %}{% if with_helmcharts_repo %} x86_64 @@ -168,6 +177,7 @@ def generate_meta( maintainers=users, with_all_arches=with_all_arches, with_helmcharts_repo=with_helmcharts_repo, + with_product_repo=os_version.is_slcc, repository_paths=repository_paths, description=description, extra_header=extra_header, diff --git a/tests/test_project_setup.py b/tests/test_project_setup.py index ba1d8386e..6018d65b0 100644 --- a/tests/test_project_setup.py +++ b/tests/test_project_setup.py @@ -149,6 +149,15 @@ s390x ppc64le + + + + + x86_64 + aarch64 + s390x + ppc64le + From 20feac291fee381f868e79f816a154aa0ac6e4fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20=C4=8Cerm=C3=A1k?= Date: Fri, 3 May 2024 15:55:13 +0200 Subject: [PATCH 09/12] Add more containers to SLCC-{free,paid} --- src/bci_build/package/appcontainers.py | 6 ++++-- src/bci_build/package/node.py | 5 +++++ src/bci_build/package/openjdk.py | 7 ++++++- src/bci_build/package/php.py | 7 ++++++- src/bci_build/package/python.py | 7 +++---- src/bci_build/package/rmt.py | 1 + src/bci_build/package/ruby.py | 8 +++++++- src/bci_build/package/rust.py | 10 +++++++--- 8 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/bci_build/package/appcontainers.py b/src/bci_build/package/appcontainers.py index 9e84e8abf..77dba0ea8 100644 --- a/src/bci_build/package/appcontainers.py +++ b/src/bci_build/package/appcontainers.py @@ -2,7 +2,6 @@ from pathlib import Path -from bci_build.package import ALL_BASE_OS_VERSIONS from bci_build.package import ALL_NONBASE_OS_VERSIONS from bci_build.package import CAN_BE_LATEST_OS_VERSION from bci_build.package import DOCKERFILE_RUN @@ -136,7 +135,7 @@ def _envsubst_pkg_name(os_version: OsVersion) -> str: CMD /usr/lib/dirsrv/dscontainer -H """, ) - for os_version in ALL_BASE_OS_VERSIONS + for os_version in ALL_NONBASE_OS_VERSIONS ] @@ -252,6 +251,7 @@ def _generate_prometheus_family_healthcheck(port: int) -> str: custom_end=_generate_prometheus_family_healthcheck(_PROMETHEUS_PORT), ) for os_version in ALL_NONBASE_OS_VERSIONS + if not os_version.is_slfo ] _ALERTMANAGER_PACKAGE_NAME = "golang-github-prometheus-alertmanager" @@ -280,6 +280,7 @@ def _generate_prometheus_family_healthcheck(port: int) -> str: custom_end=_generate_prometheus_family_healthcheck(_ALERTMANAGER_PORT), ) for os_version in ALL_NONBASE_OS_VERSIONS + if not os_version.is_slfo ] _BLACKBOX_EXPORTER_PACKAGE_NAME = "prometheus-blackbox_exporter" @@ -353,6 +354,7 @@ def _generate_prometheus_family_healthcheck(port: int) -> str: """, ) for os_version in ALL_NONBASE_OS_VERSIONS + if not os_version.is_slfo ] _NGINX_FILES = {} diff --git a/src/bci_build/package/node.py b/src/bci_build/package/node.py index d18009186..7158ca037 100644 --- a/src/bci_build/package/node.py +++ b/src/bci_build/package/node.py @@ -63,4 +63,9 @@ def _get_node_kwargs(ver: _NODE_VERSIONS, os_version: OsVersion): ), DevelopmentContainer(**_get_node_kwargs(20, OsVersion.TUMBLEWEED)), DevelopmentContainer(**_get_node_kwargs(22, OsVersion.TUMBLEWEED)), + DevelopmentContainer(**_get_node_kwargs(20, OsVersion.SLCC_FREE)), + DevelopmentContainer( + **_get_node_kwargs(20, OsVersion.SLCC_PAID), + support_level=SupportLevel.L3, + ), ] diff --git a/src/bci_build/package/openjdk.py b/src/bci_build/package/openjdk.py index 06c7037a5..c595e42ee 100644 --- a/src/bci_build/package/openjdk.py +++ b/src/bci_build/package/openjdk.py @@ -91,7 +91,12 @@ def _get_openjdk_kwargs( support_level=SupportLevel.L3, ) for os_version, devel in product( - (OsVersion.SP6, OsVersion.SLCC_PRODUCTION, OsVersion.TUMBLEWEED), + ( + OsVersion.SP6, + OsVersion.SLCC_PAID, + OsVersion.SLCC_FREE, + OsVersion.TUMBLEWEED, + ), (True, False), ) ] diff --git a/src/bci_build/package/php.py b/src/bci_build/package/php.py index 876f652b2..8e407e519 100644 --- a/src/bci_build/package/php.py +++ b/src/bci_build/package/php.py @@ -193,7 +193,12 @@ def _create_php_bci( PHP_CONTAINERS = [ _create_php_bci(os_version, variant, 8) for os_version, variant in product( - (OsVersion.SP6, OsVersion.TUMBLEWEED), + ( + OsVersion.SP6, + OsVersion.TUMBLEWEED, + OsVersion.SLCC_FREE, + OsVersion.SLCC_PAID, + ), (PhpVariant.cli, PhpVariant.apache, PhpVariant.fpm), ) ] diff --git a/src/bci_build/package/python.py b/src/bci_build/package/python.py index faca68805..5dc5f9b3e 100644 --- a/src/bci_build/package/python.py +++ b/src/bci_build/package/python.py @@ -132,10 +132,9 @@ def _get_python_kwargs(py3_ver: _PYTHON_VERSIONS, os_version: OsVersion): PythonDevelopmentContainer( **_get_python_kwargs("3.12", os_version), package_name="python-3.12-image", - # Technically it is the latest but we want to prefer the long term Python - # 3.11 for SLE, SLCC Paid & TW - is_latest=os_version in CAN_BE_LATEST_OS_VERSION - and os_version != OsVersion.SLCC_PAID, + # Technically it is the latest but we want to prefer the long term + # Python 3.11 for SLE 15 & TW + is_latest=os_version.is_slfo, ) for os_version in ( OsVersion.SP6, diff --git a/src/bci_build/package/rmt.py b/src/bci_build/package/rmt.py index 474182291..e76fc6916 100644 --- a/src/bci_build/package/rmt.py +++ b/src/bci_build/package/rmt.py @@ -39,4 +39,5 @@ """, ) for os_version in ALL_NONBASE_OS_VERSIONS + if not os_version.is_slfo ] diff --git a/src/bci_build/package/ruby.py b/src/bci_build/package/ruby.py index 5d7768697..cc4666e91 100644 --- a/src/bci_build/package/ruby.py +++ b/src/bci_build/package/ruby.py @@ -11,7 +11,7 @@ from bci_build.package import generate_disk_size_constraints -def _get_ruby_kwargs(ruby_version: Literal["2.5", "3.3"], os_version: OsVersion): +def _get_ruby_kwargs(ruby_version: Literal["2.5", "3.2", "3.3"], os_version: OsVersion): ruby = f"ruby{ruby_version}" ruby_major = ruby_version.split(".")[0] @@ -66,5 +66,11 @@ def _get_ruby_kwargs(ruby_version: Literal["2.5", "3.3"], os_version: OsVersion) **_get_ruby_kwargs("2.5", OsVersion.SP6), support_level=SupportLevel.L3, ), + # FIXME: enable this for SLCC v2 + # DevelopmentContainer(**_get_ruby_kwargs("3.2", OsVersion.SLCC_DEVELOPMENT)), + # DevelopmentContainer( + # **_get_ruby_kwargs("3.2", OsVersion.SLCC_PRODUCTION), + # support_level=SupportLevel.L3, + # ), DevelopmentContainer(**_get_ruby_kwargs("3.3", OsVersion.TUMBLEWEED)), ] diff --git a/src/bci_build/package/rust.py b/src/bci_build/package/rust.py index 8f623a7f3..1c40b7044 100644 --- a/src/bci_build/package/rust.py +++ b/src/bci_build/package/rust.py @@ -6,6 +6,7 @@ from bci_build.package import ALL_NONBASE_OS_VERSIONS from bci_build.package import CAN_BE_LATEST_OS_VERSION from bci_build.package import DevelopmentContainer +from bci_build.package import OsVersion from bci_build.package import Replacement from bci_build.package import SupportLevel from bci_build.package import generate_disk_size_constraints @@ -86,8 +87,11 @@ COPY {check_fname} /etc/zypp/systemCheck.d/{check_fname} """, ) - for rust_version, os_version in product( - _RUST_VERSIONS, - ALL_NONBASE_OS_VERSIONS, + for rust_version, os_version in list( + product( + _RUST_VERSIONS, + set(ALL_NONBASE_OS_VERSIONS).difference({OsVersion.SLCC_FREE}), + ) ) + + [(_RUST_VERSIONS[-1], OsVersion.SLCC_FREE)] ] From d71be6533aacad428dd1ae72e1e7ee9a27654c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20=C4=8Cerm=C3=A1k?= Date: Tue, 7 May 2024 12:27:54 +0200 Subject: [PATCH 10/12] Add bot command to render a package list for groups.yml --- src/staging/bot.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/staging/bot.py b/src/staging/bot.py index 9aa1ccd1e..04a328745 100644 --- a/src/staging/bot.py +++ b/src/staging/bot.py @@ -37,6 +37,7 @@ from bci_build.package import ALL_CONTAINER_IMAGE_NAMES from bci_build.package import BaseContainerImage from bci_build.package import OsVersion +from bci_build.package import Package from dotnet.updater import DOTNET_IMAGES from dotnet.updater import DotNetBCI from staging.build_result import PackageBuildResult @@ -311,6 +312,29 @@ def bcis(self) -> Generator[BaseContainerImage, None, None]: ) ) + @property + def groups_yml(self) -> str: + """Generate a the ``container_packages`` YAML list for + :file:`groups.yml` in ``000package-groups`` from the container images + for this code stream. + + .. caution:: Only works for SLCC! + + """ + if not self.os_version.is_slcc: + raise ValueError("Only supported for SLCC code streams") + + res = "container_packages:" + + for bci in sorted(list(self._bcis), key=lambda b: b.uid): + res += f"\n # {bci.uid}\n - " + res += "\n - ".join( + pkg.name if isinstance(pkg, Package) else pkg + for pkg in bci.package_list + ) + + return res + @staticmethod def from_github_comment(comment_text: str, osc_username: str) -> "StagingBot": if comment_text == "": @@ -1543,6 +1567,7 @@ def main() -> None: "changelog_check", "setup_obs_package", "setup_obs_project", + "groups_yml", "find_missing_packages", ] @@ -1721,6 +1746,11 @@ def add_commit_message_arg(p: argparse.ArgumentParser) -> None: "setup_obs_project", help="Configure the devel project on OBS" ) + subparsers.add_parser( + "groups_yml", + help="Create a list of all container packages that can be inserted into groups.yml", + ) + subparsers.add_parser( "find_missing_packages", help="Find all packages that are in the deployment branch and are missing from `devel:BCI:*` on OBS", @@ -1871,6 +1901,13 @@ async def _setup_pkg_meta(): elif action == "setup_obs_project": coro = bot.configure_devel_bci_project() + elif action == "groups_yml": + + async def _groups_yml(): + return bot.groups_yml + + coro = _groups_yml() + elif action == "find_missing_packages": async def _pkgs_as_str() -> str: From 058aac3332e5166f6f310fe4ca2011c18b128920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20=C4=8Cerm=C3=A1k?= Date: Mon, 27 May 2024 21:40:51 +0200 Subject: [PATCH 11/12] [bot] Add helper to generate 000package-group & skelcd contents --- src/staging/bot.py | 495 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 468 insertions(+), 27 deletions(-) diff --git a/src/staging/bot.py b/src/staging/bot.py index 04a328745..95e5ec8b4 100644 --- a/src/staging/bot.py +++ b/src/staging/bot.py @@ -132,6 +132,417 @@ class ProjectConfig(Enum): META = "prj" +_PACKAGE_GROUP_NAME = "slcc_packages" + + +@dataclass(frozen=True) +class TripleZeroPackageGroups: + """000package-groups""" + + os_version: OsVersion + + architectures: list[str] = field( + default_factory=lambda: ["x86_64", "aarch64", "s390x", "ppc64le"] + ) + + def __post_init__(self) -> None: + if not self.os_version.is_slcc: + raise ValueError(f"Only implemented for SLCC, not for {self.os_version}") + + @property + def release_spec_in(self) -> str: + pkg_name = f"{str(self.os_version).lower()}-release" + return ( + f"""# +# spec file for package {pkg_name} +# +# Copyright (c) {datetime.now().year} SUSE LLC +# +# All modifications and additions to the file contributed by third parties +# remain the property of their copyright owners, unless otherwise agreed +# upon. The license for this file, and modifications and additions to the +# file, is the same license as for the pristine package itself (unless the +# license for the pristine package is not an Open Source License, in which +# case the license is the MIT License). An "Open Source License" is a +# license that conforms to the Open Source Definition (Version 1.9) +# published by the Open Source Initiative. + +# Please submit bugfixes or comments via https://bugzilla.suse.com/ +# + +Name: {pkg_name} +Summary: ___SUMMARY___ ___BETA_VERSION___ +License: MIT +Group: System/Fhs +Version: ___VERSION___ +Release: 0 +# FIXME? or keep this package name +BuildRequires: skelcd-EULA-{str(self.os_version).lower()} +Provides: distribution-release +""" + + """Provides: product(SUSE_SLE) = %{version}-%{release} +Provides: product(SUSE_SLE-SP___PATCH_LEVEL___) = %{version}-%{release} + +# bsc#1055299 +Conflicts: otherproviders(distribution-release) + +___PRODUCT_PROVIDES___ + +___PRODUCT_DEPENDENCIES___ + + +ExclusiveArch: """ + + " ".join(self.architectures) + + """ + +Source100: weakremovers.inc +%include %{SOURCE100} + +%description +___DESCRIPTION___ + +___FLAVOR_PACKAGES___ + +%prep + +%build + +%install +mkdir -p %buildroot/%_sysconfdir + +___CREATE_OS_RELEASE_FILE___ + +cat << EOF >> %buildroot/%_sysconfdir/os-release +DOCUMENTATION_URL="https://documentation.suse.com/" +EOF + +___CREATE_PRODUCT_FILES___ + + +%files +%defattr(644,root,root,755) +%config %_sysconfdir/os-release +%dir %_sysconfdir/products.d +%_sysconfdir/products.d/* + +%changelog +""" + ) + + @property + def product_in(self) -> str: + pool_prefix = f"{self.os_version.value}-{self.os_version.os_version}" + return ( + f""" + + + + SUSE + {str(self.os_version).lower()} + {self.os_version.os_version} + 1 + + + + {self.os_version.full_os_name} + + + + {self.os_version.pretty_os_version_no_dash} + + + + +""" + + "\n".join( + f""" + + """ + for (medium, alias), arch in itertools.product( + [("", ""), ("debug_", "-Debuginfo"), ("source_", "-Source")], + self.architectures, + ) + ) + + """ + + +""" + + "\n".join( + f""" sle-15-{arch}""" + for arch in self.architectures + ) + + """ + + + + + + {self.os_version.full_os_name} + {self.os_version.full_os_name} + {self.os_version.full_os_name} + + + + en + + + + + https://www.suse.com/releasenotes/%{{_target_cpu}}/SL-Micro/6.0/release-notes-sl-micro.rpm + + + + + {self.os_version} + + + + + + en_US + SUSE + + + + + + + + + + + + + + + + + + + + + + +""" + + "\n".join( + f""" + + """ + for arch in self.architectures + ) + + f""" + + + + + + + +""" + ) + + @property + def default_productcompose_in(self) -> str: + return f"""product_compose_schema: 0.2 + +vendor: SUSE +name: {self.os_version.value} +version: {self.os_version.os_version} +product-type: module +summary: {self.os_version.full_os_name} + +scc: + description: > + Alp Basalt ftp tree, also known as POOL. + Used for GA and maintenance update afterwards. + +build_options: +### For maintenance, otherwise only "the best" version of each package is picked: +# - take_all_available_versions +- hide_flavor_in_product_directory_name + + +source: split +debug: split + +# has only an effect during maintenance: +set_updateinfo_from: maint-coord@suse.de + +# will be extended with architecture and flavor string +# product_directory_name: "ALP-Dolomite-1.0" + +flavors: + {_PACKAGE_GROUP_NAME}_aarch64: + architectures: [ aarch64 ] + {_PACKAGE_GROUP_NAME}_ppc64le: + architectures: [ ppc64le ] + {_PACKAGE_GROUP_NAME}_s390x: + architectures: [ s390x ] + {_PACKAGE_GROUP_NAME}_x86_64: + architectures: [ x86_64 ] + +unpack: + - unpackset + +packagesets: +- name: unpackset + packages: + - skelcd-EULA-{str(self.os_version.value).lower()} + +# The following is generated by openSUSE-release-tools + +This part will get replaced by pkglistgen and the file will get written to +000productcompose sub directory. + +""" + + +@dataclass(frozen=True) +class SkelcdPackage: + os_version: OsVersion + + @property + def spec(self) -> str: + return ( + f"""# +# spec file for package skelcd +# +# Copyright (c) 2024 SUSE LLC. +# +# All modifications and additions to the file contributed by third parties +# remain the property of their copyright owners, unless otherwise agreed +# upon. The license for this file, and modifications and additions to the +# file, is the same license as for the pristine package itself (unless the +# license for the pristine package is not an Open Source License, in which +# case the license is the MIT License). An "Open Source License" is a +# license that conforms to the Open Source Definition (Version 1.9) +# published by the Open Source Initiative. + +# Please submit bugfixes or comments via http://bugs.opensuse.org/ +# +%define SLE_RELEASE 16 +# +# default replacement variables for README content +%define PRETTY_NAME {self.os_version.pretty_os_version_no_dash} +%define UNDERLINE =================================== +%define PRODUCT_LINK https://www.suse.com/sles + +%define product {str(self.os_version.value).lower()} +%define PRODUCT {str(self.os_version.value).upper()} +""" + + """ +%define dash - + +%define container_path usr/share/licenses/product/%{PRODUCT} +%define skelcd1_path usr/share/licenses/product/%{product} + +# release is a beta +%define beta 0 + +%if 0%{?beta} == 1 +%define license_dir license.beta +%else +%define license_dir license.final +%endif + +%dnl %define skelcd1_path usr/lib/skelcd/CD1 + +Name: skelcd%{?dash}%{product} + + +AutoReqProv: off +Version: 2024.05.03.1 +Release: 0 +Summary: CD skeleton for %{PRODUCT} +License: GPL-2.0-only +Group: Metapackages +BuildRoot: %{_tmppath}/%{name}-%{version}-build +Source: skelcd-%{version}.tar.xz +# please repo-checker (bsc#1089174) +Provides: skelcd = %{version} +Conflicts: otherproviders(skelcd) + +%description +Skeleton package for %{PRODUCT} + +%package -n skelcd-EULA%{?dash}%{product} +Summary: EULA for media +Group: Metapackages + +%description -n skelcd-EULA%{?dash}%{product} +Internal package only. + + +%prep +%setup -n skelcd%{?dash}%{version} -q + +%build + +%install +# +# copy the product READMEs +pushd READMEs/default +sed -i -e 's/{PRETTY_NAME}/%{PRETTY_NAME} %{SLE_RELEASE}/g' README +sed -i -e 's/{UNDERLINE}/%{UNDERLINE}/g' README +# use @ as delimiter, as the product link conflicts with the standard '/' delimiter +sed -i -e 's@{PRODUCT_LINK}@%{PRODUCT_LINK}@g' README +popd + +# +# license tarball generation +mkdir -p $RPM_BUILD_ROOT/%{skelcd1_path}/media.1 +pushd %license_dir +# touch all license files to make sure they have the most recent date +# this impacts which license is shown on the CDN to fix bsc#1186047 and bsc#1186812 +# else in case beta EULAs have a more recent date than final EULAs they won't +# get replaced +touch * +ls -1 > directory.yast # required for downloading of EULAs from SCC + +# bci doesn't have a release package, make EULA available directly +rmdir $RPM_BUILD_ROOT/%{skelcd1_path}/media.1 +mv ../BCI/*.txt $RPM_BUILD_ROOT/%{skelcd1_path}/ + +popd + +# +# skelcd-EULA +%files -n skelcd-EULA-%{product} +%defattr(644,root,root,755) +%dir %{_datadir}/licenses/product +/%{skelcd1_path} + +%changelog +""" + ) + + @dataclass class StagingBot: """Bot that creates a staging project for the BCI images in the Open Build @@ -1567,7 +1978,7 @@ def main() -> None: "changelog_check", "setup_obs_package", "setup_obs_project", - "groups_yml", + "000package-groups", "find_missing_packages", ] @@ -1756,6 +2167,23 @@ def add_commit_message_arg(p: argparse.ArgumentParser) -> None: help="Find all packages that are in the deployment branch and are missing from `devel:BCI:*` on OBS", ) + triple_zero_parser = subparsers.add_parser( + "000package-groups", help="generate 000package-groups files" + ) + triple_zero_parser.add_argument( + "--file", + nargs=1, + required=True, + type=str, + choices=[ + "groups.yml", + "release.spec.in", + "product.in", + "skelcd", + "default.productcompose.in", + ], + ) + loop = asyncio.get_event_loop() args = parser.parse_args() @@ -1793,10 +2221,10 @@ def add_commit_message_arg(p: argparse.ArgumentParser) -> None: try: action: ACTION_T = args.action - coro: Coroutine[Any, Any, Any] | None = None + coro_or_str: Coroutine[Any, Any, Any] | str | None = None if action == "rebuild": - coro = bot.force_rebuild() + coro_or_str = bot.force_rebuild() elif action == "create_staging_project": @@ -1809,17 +2237,17 @@ async def _create_staging_proj(): ) await bot.link_base_container_to_staging() - coro = _create_staging_proj() + coro_or_str = _create_staging_proj() elif action == "commit_state": - coro = bot.write_all_build_recipes_to_branch(args.commit_message[0]) + coro_or_str = bot.write_all_build_recipes_to_branch(args.commit_message[0]) elif action == "query_build_result": async def print_build_res(): return render_as_markdown(await bot.fetch_build_results()) - coro = print_build_res() + coro_or_str = print_build_res() elif action == "scratch_build": @@ -1827,10 +2255,10 @@ async def _scratch(): commit_or_none = await bot.scratch_build(args.commit_message[0]) return commit_or_none or "No changes" - coro = _scratch() + coro_or_str = _scratch() elif action == "cleanup": - coro = bot.remote_cleanup( + coro_or_str = bot.remote_cleanup( branches=not args.no_cleanup_branch, obs_project=not args.no_cleanup_project, ) @@ -1842,7 +2270,7 @@ async def _wait(): await bot.wait_for_build_to_finish(timeout_sec=args.timeout_sec[0]) ) - coro = _wait() + coro_or_str = _wait() elif action == "get_build_quality": @@ -1852,10 +2280,10 @@ async def _quality(): raise RuntimeError("Build failed!") return "Build succeded" - coro = _quality() + coro_or_str = _quality() elif action == "create_cr_project": - coro = bot.write_cr_project_config() + coro_or_str = bot.write_cr_project_config() elif action == "add_changelog_entry": changelog_entry = " ".join(args.entry) @@ -1867,7 +2295,7 @@ async def _quality(): elif packages_len > 1: pkg_names = args.packages - coro = bot.add_changelog_entry( + coro_or_str = bot.add_changelog_entry( entry=changelog_entry, username=username, package_names=pkg_names ) @@ -1886,7 +2314,7 @@ async def _error_on_pkg_without_changes(): f"{change_ref}: {', '.join(packages_without_changes)}" ) - coro = _error_on_pkg_without_changes() + coro_or_str = _error_on_pkg_without_changes() elif action == "setup_obs_package": async def _setup_pkg_meta(): @@ -1896,31 +2324,44 @@ async def _setup_pkg_meta(): ] await asyncio.gather(*tasks) - coro = _setup_pkg_meta() + coro_or_str = _setup_pkg_meta() elif action == "setup_obs_project": - coro = bot.configure_devel_bci_project() - - elif action == "groups_yml": - - async def _groups_yml(): - return bot.groups_yml - - coro = _groups_yml() + coro_or_str = bot.configure_devel_bci_project() + + elif action == "000package-groups": + if (fname := args.file[0]) == "groups.yml": + coro_or_str = bot.groups_yml + elif fname == "product.in": + coro_or_str = TripleZeroPackageGroups(bot.os_version).product_in + elif fname == "release.spec.in": + coro_or_str = TripleZeroPackageGroups(bot.os_version).release_spec_in + elif fname == "default.productcompose.in": + coro_or_str = TripleZeroPackageGroups( + bot.os_version + ).default_productcompose_in + elif fname == "skelcd": + coro_or_str = SkelcdPackage(bot.os_version).spec + else: + raise ValueError(f"Invalid file for 000package-groups: {fname}") elif action == "find_missing_packages": async def _pkgs_as_str() -> str: return ", ".join(await bot.find_missing_packages_on_obs()) - coro = _pkgs_as_str() + coro_or_str = _pkgs_as_str() else: assert False, f"invalid action: {action}" - assert coro is not None - res = loop.run_until_complete(coro) - if res: - print(res) + assert coro_or_str is not None + + if isinstance(coro_or_str, str): + print(coro_or_str) + else: + res = loop.run_until_complete(coro_or_str) + if res: + print(res) finally: loop.run_until_complete(bot.teardown()) From f3279ee686aee83bcc1e465aa894a3bf642f8ee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20=C4=8Cerm=C3=A1k?= Date: Tue, 28 May 2024 14:05:42 +0200 Subject: [PATCH 12/12] WIP: new codestream documentation --- source/index.rst | 2 ++ source/new_codestream.rst | 56 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 source/new_codestream.rst diff --git a/source/index.rst b/source/index.rst index 28c6691f9..679e2069c 100644 --- a/source/index.rst +++ b/source/index.rst @@ -11,6 +11,8 @@ Contents staging_bot + new_codestream + api Indices and tables diff --git a/source/new_codestream.rst b/source/new_codestream.rst new file mode 100644 index 000000000..6c9e0a973 --- /dev/null +++ b/source/new_codestream.rst @@ -0,0 +1,56 @@ +Creating a new codestream +========================= + +To create a new codestream follow these steps: + +1. Create a deployment branch. It must have the name + :py:attr:`~staging.bot.StagingBot.deployment_branch_name` and should contain + only a :file:`_config` file with the prjconf of the target project (usually + you can take the prjconf from the previous service pack, if applicable). + +2. Create the target project on OBS. This can be achieved via the bot command + ``setup_obs_project``: + +.. code-block:: shell + + $ export OSC_USER=$MY_USER + $ export OSC_PASSWORD=$MY_PASS + $ poetry run scratch-build-bot \ + --os-version $CODE_STREAM \ + --branch-name="doesNotMatter" \ + -vvvv setup_obs_project + +3. Add the new code stream to the github action files to the ``os_version`` + list: + :file:`.github/workflows/obs_build.yml` + :file:`.github/workflows/update-deployment-branches.yml` + :file:`.github/workflows/update-cr-project.yml` + :file:`.github/workflows/cleanup-staging.yml` + + +SLCC specific steps +------------------- + +For SLCC we need to build the FTP trees (= repositories) ourselves. For that we +must create the ``000*`` packages in the checked out project: + +.. code-block:: shell + + $ cd devel:BCI:SLCC:$stream/ + + $ osc mkpac 000product + A 000product + + $ osc mkpac 000release-packages + A 000release-packages + + $ osc mkpac 000package-groups + A 000package-groups + + +We only have to touch ``000package-groups`` directly, the remaining two are +auto-generated using `pkglistgen +`_. + + +python3 ./pkglistgen.py --verbose -A https://api.opensuse.org update_and_solve -p devel:BCI:SLCC:dynamic-developer -s target