diff --git a/.github/workflows/cleanup-staging.yml b/.github/workflows/cleanup-staging.yml index b7a639762..9bd109afd 100644 --- a/.github/workflows/cleanup-staging.yml +++ b/.github/workflows/cleanup-staging.yml @@ -19,6 +19,8 @@ jobs: - 6 - 5 - Tumbleweed + - SLCC-free + - SLCC-paid steps: # we need all branches for the build checks diff --git a/.github/workflows/obs_build.yml b/.github/workflows/obs_build.yml index bdd4464d4..a8f29afd7 100644 --- a/.github/workflows/obs_build.yml +++ b/.github/workflows/obs_build.yml @@ -16,6 +16,8 @@ jobs: - 6 - 5 - Tumbleweed + - SLCC-free + - SLCC-paid steps: # we need all branches for the build checks diff --git a/.github/workflows/update-cr-project.yml b/.github/workflows/update-cr-project.yml index 71437ca96..3030f8c71 100644 --- a/.github/workflows/update-cr-project.yml +++ b/.github/workflows/update-cr-project.yml @@ -21,6 +21,8 @@ jobs: - 6 - 5 - Tumbleweed + - SLCC-free + - SLCC-paid steps: # we need all branches for the build checks diff --git a/.github/workflows/update-deployment-branches.yml b/.github/workflows/update-deployment-branches.yml index a37fecd1e..8347a2442 100644 --- a/.github/workflows/update-deployment-branches.yml +++ b/.github/workflows/update-deployment-branches.yml @@ -23,6 +23,8 @@ jobs: - 6 - 5 - Tumbleweed + - SLCC-free + - SLCC-paid steps: - uses: actions/checkout@v4 diff --git a/source/api.rst b/source/api.rst index f6f24a4ba..0faf03ce3 100644 --- a/source/api.rst +++ b/source/api.rst @@ -62,3 +62,11 @@ API Documentation .. automodule:: staging.build_result :members: :undoc-members: + + +:py:mod:`staging.project_setup` module +-------------------------------------- + +.. automodule:: staging.project_setup + :members: + :undoc-members: diff --git a/source/index.rst b/source/index.rst index 28c6691f9..679e2069c 100644 --- a/source/index.rst +++ b/source/index.rst @@ -11,6 +11,8 @@ Contents staging_bot + new_codestream + api Indices and tables diff --git a/source/new_codestream.rst b/source/new_codestream.rst new file mode 100644 index 000000000..6c9e0a973 --- /dev/null +++ b/source/new_codestream.rst @@ -0,0 +1,56 @@ +Creating a new codestream +========================= + +To create a new codestream follow these steps: + +1. Create a deployment branch. It must have the name + :py:attr:`~staging.bot.StagingBot.deployment_branch_name` and should contain + only a :file:`_config` file with the prjconf of the target project (usually + you can take the prjconf from the previous service pack, if applicable). + +2. Create the target project on OBS. This can be achieved via the bot command + ``setup_obs_project``: + +.. code-block:: shell + + $ export OSC_USER=$MY_USER + $ export OSC_PASSWORD=$MY_PASS + $ poetry run scratch-build-bot \ + --os-version $CODE_STREAM \ + --branch-name="doesNotMatter" \ + -vvvv setup_obs_project + +3. Add the new code stream to the github action files to the ``os_version`` + list: + :file:`.github/workflows/obs_build.yml` + :file:`.github/workflows/update-deployment-branches.yml` + :file:`.github/workflows/update-cr-project.yml` + :file:`.github/workflows/cleanup-staging.yml` + + +SLCC specific steps +------------------- + +For SLCC we need to build the FTP trees (= repositories) ourselves. For that we +must create the ``000*`` packages in the checked out project: + +.. code-block:: shell + + $ cd devel:BCI:SLCC:$stream/ + + $ osc mkpac 000product + A 000product + + $ osc mkpac 000release-packages + A 000release-packages + + $ osc mkpac 000package-groups + A 000package-groups + + +We only have to touch ``000package-groups`` directly, the remaining two are +auto-generated using `pkglistgen +`_. + + +python3 ./pkglistgen.py --verbose -A https://api.opensuse.org update_and_solve -p devel:BCI:SLCC:dynamic-developer -s target diff --git a/src/bci_build/package/__init__.py b/src/bci_build/package/__init__.py index 061bae540..3ff8950d1 100644 --- a/src/bci_build/package/__init__.py +++ b/src/bci_build/package/__init__.py @@ -136,6 +136,10 @@ class OsVersion(enum.Enum): SLE16_0 = "16.0" #: openSUSE Tumbleweed TUMBLEWEED = "Tumbleweed" + #: SUSE Linux Container Collection - free stream + SLCC_FREE = "SLCC-free" + #: SUSE Linux Container Collection - paid stream + SLCC_PAID = "SLCC-paid" @staticmethod def parse(val: str) -> OsVersion: @@ -147,6 +151,17 @@ def parse(val: str) -> OsVersion: def __str__(self) -> str: return str(self.value) + @property + def distribution_base_name(self) -> str: + if self.is_tumbleweed: + return "openSUSE Tumbleweed" + elif self.is_ltss: + return "SLE LTSS" + elif self.is_sle15: + return "SLE" + + return "SUSE Linux Framework One" + @property def pretty_print(self) -> str: if self.value in (OsVersion.TUMBLEWEED.value, OsVersion.SLE16_0.value): @@ -162,8 +177,24 @@ def pretty_os_version_no_dash(self) -> str: if self.is_slfo: return "Framework One" + assert self.is_sle15 return f"15 SP{self.value}" + @property + def full_os_name(self) -> str: + if self.is_slcc: + assert isinstance(self.value, str) + return ( + "SUSE Linux Container Collection - " + + self.value.replace("SLCC-", "").replace("-", " ") + + " Stream" + ) + if self.is_sle15: + return f"{self.distribution_base_name} {self.pretty_os_version_no_dash}" + + assert self.is_opensuse + return self.distribution_base_name + @property def deployment_branch_name(self) -> str: if self.is_tumbleweed or self.is_slfo: @@ -200,7 +231,11 @@ def is_sle15(self) -> bool: @property def is_slfo(self) -> bool: - return self.value in (OsVersion.SLE16_0.value,) + return self.value in ( + OsVersion.SLCC_FREE.value, + OsVersion.SLCC_PAID.value, + OsVersion.SLE16_0.value, + ) @property def is_tumbleweed(self) -> bool: @@ -208,6 +243,10 @@ def is_tumbleweed(self) -> bool: @property def is_ltss(self) -> bool: + """Determines whether this OS is covered by a different EULA and is not + redistributable. + + """ return self in ALL_OS_LTSS_VERSIONS @property @@ -218,7 +257,7 @@ def os_version(self) -> str: """ if self.is_sle15: return f"15.{str(self.value)}" - if self.value == OsVersion.SLE16_0.value: + if self.is_slfo: return "16.0" # Tumbleweed rolls too fast, just use latest return "latest" @@ -231,25 +270,30 @@ def has_container_suseconnect(self) -> bool: def eula_package_names(self) -> tuple[str, ...]: if self.is_sle15: return ("skelcd-EULA-bci",) - # if self.is_slcc: - # return (f"skelcd-EULA-{str(self.value).lower()}",) + if self.is_slfo: + return (f"skelcd-EULA-{str(self.value).lower()}",) return () @property def release_package_names(self) -> tuple[str, ...]: if self.value == OsVersion.TUMBLEWEED.value: return ("openSUSE-release", "openSUSE-release-appliance-docker") - if self.value == OsVersion.SLE16_0.value: - return ("ALP-dummy-release",) if self.is_ltss: return ("sles-ltss-release",) - # if self.is_slcc: - # return (f"{str(self.value).lower()}-release",) + if self.is_slfo: + return (f"{str(self.value).lower()}-release",) assert self.is_sle15 return ("sles-release",) +SLCC_OS_VERSIONS: list[OsVersion] = [ + OsVersion.SLCC_FREE, + OsVersion.SLCC_PAID, + OsVersion.SLE16_0, +] + + #: Operating system versions that have the label ``com.suse.release-stage`` set #: to ``released``. RELEASED_OS_VERSIONS: list[OsVersion] = [ @@ -264,6 +308,8 @@ def release_package_names(self) -> tuple[str, ...]: ALL_NONBASE_OS_VERSIONS: list[OsVersion] = [ OsVersion.SP6, OsVersion.TUMBLEWEED, + OsVersion.SLCC_FREE, + OsVersion.SLCC_PAID, ] # For which versions to create Base Container Images? @@ -271,8 +317,8 @@ def release_package_names(self) -> tuple[str, ...]: OsVersion.SP5, OsVersion.SP6, OsVersion.TUMBLEWEED, - OsVersion.SLE16_0, -] +] + SLCC_OS_VERSIONS + # List of SPs that are already under LTSS ALL_OS_LTSS_VERSIONS: list[OsVersion] = [OsVersion.SP3, OsVersion.SP4] @@ -360,11 +406,16 @@ def __post_init__(self) -> None: def _build_tag_prefix(os_version: OsVersion) -> str: if os_version == OsVersion.TUMBLEWEED: return "opensuse/bci" + if os_version == OsVersion.SLE16_0: + return "suse/sle16" + if os_version == OsVersion.SLCC_PAID: + return "suse/supported" if os_version == OsVersion.SP3: return "suse/ltss/sle15.3" if os_version == OsVersion.SP4: return "suse/ltss/sle15.4" + # remaining SLE SPs + SLCC free use BCI as the prefix return "bci" @@ -595,7 +646,7 @@ def build_name(self) -> str | None: @property def build_version(self) -> str | None: - if self.os_version not in (OsVersion.TUMBLEWEED, OsVersion.SLE16_0): + if self.os_version.is_sle15: epoch = "" if self.os_epoch: epoch = f"{self.os_epoch}." @@ -613,17 +664,6 @@ def build_release(self) -> str | None: else None ) - @property - def distribution_base_name(self) -> str: - if self.os_version.is_tumbleweed: - return "openSUSE Tumbleweed" - elif self.os_version.is_ltss: - return "SLE LTSS" - elif self.os_version.is_sle15 or self.os_version.is_slfo: - return "SLE" - - raise NotImplementedError(f"Unknown os_version: {self.os_version}") - @property def eula(self) -> str: """EULA covering this image. can be ``sle-eula`` or ``sle-bci``.""" @@ -800,7 +840,7 @@ def _from_image(self) -> str | None: if self.os_version == OsVersion.TUMBLEWEED: return "opensuse/tumbleweed:latest" if self.os_version == OsVersion.SLE16_0: - return f"{_build_tag_prefix(self.os_version)}/bci-base:latest" + return f"{_build_tag_prefix(self.os_version)}/base:latest" if self.os_version in ALL_OS_LTSS_VERSIONS: return f"{_build_tag_prefix(self.os_version)}/sle15:15.{self.os_version}" if self.image_type == ImageType.APPLICATION: @@ -1060,7 +1100,7 @@ def description(self) -> str: description_formatters = { "pretty_name": self.pretty_name, "based_on_container": ( - f"based on the {self.distribution_base_name} Base Container Image" + f"based on the {self.os_version.distribution_base_name} Base Container Image" ), "podman_only": "This container is only supported with podman.", "privileged_only": "This container is only supported in privileged mode.", @@ -1077,12 +1117,12 @@ def title(self) -> str: label. It is generated from :py:attr:`BaseContainerImage.pretty_name` as - follows: ``"{distribution_base_name} BCI {self.pretty_name}"``, where - ``distribution_base_name`` is taken from - :py:attr:`~ImageProperties.distribution_base_name`. + follows: ``"{distribution_base_name}(if ! SLLC: BCI else '') + {self.pretty_name}"``, where ``distribution_base_name`` is taken from + :py:attr:`~OsVersion.distribution_base_name`. """ - return f"{self.distribution_base_name} BCI {self.pretty_name}" + return f"{self.os_version.distribution_base_name}{' BCI' if not self.os_version.is_slfo else ''} {self.pretty_name}" @property def readme_path(self) -> str: @@ -1167,27 +1207,37 @@ def labelprefix(self) -> str: :py:attr:`~BaseContainerImage.custom_labelprefix_end`. """ - labelprefix = "com.suse" - if self.os_version.is_tumbleweed: - labelprefix = "org.opensuse" - return ( - labelprefix - + "." - + ( - { - ImageType.SLE_BCI: "bci", - ImageType.APPLICATION: "application", - ImageType.LTSS: "sle", - }[self.image_type] + if self.os_version.is_opensuse: + labelprefix = "org.opensuse." + elif self.os_version.is_sle15: + labelprefix = "com.suse." + else: + assert self.os_version.is_slfo + labelprefix = ( + "com.suse.slfo." + + { + OsVersion.SLCC_FREE: "free", + OsVersion.SLCC_PAID: "supported", + OsVersion.SLE16_0: "sle16", + }[self.os_version] ) - + "." - + (self.custom_labelprefix_end or self.name) - ) + + if not self.os_version.is_slcc: + labelprefix += { + ImageType.SLE_BCI: "bci", + ImageType.APPLICATION: "application", + ImageType.LTSS: "sle", + }[self.image_type] + + return f"{labelprefix}.{(self.custom_labelprefix_end or self.name)}" @property def kiwi_version(self) -> str: - if self.os_version in (OsVersion.TUMBLEWEED, OsVersion.SLE16_0): + if self.os_version == OsVersion.TUMBLEWEED: return str(datetime.datetime.now().year) + # FIXME: both should be handled better + if self.os_version.is_slfo: + return "16.0.0" return f"15.{int(self.os_version.value)}.0" @property @@ -1465,6 +1515,8 @@ def __post_init__(self) -> None: def _registry_prefix(self) -> str: if self.os_version.is_tumbleweed: return "opensuse" + if self.os_version.is_slfo: + return _build_tag_prefix(self.os_version) return "suse" @property @@ -1473,7 +1525,7 @@ def image_type(self) -> ImageType: @property def title(self) -> str: - return f"{self.distribution_base_name} {self.pretty_name}" + return f"{self.os_version.distribution_base_name} {self.pretty_name}" @property def eula(self) -> str: @@ -1487,9 +1539,13 @@ def eula(self) -> str: class OsContainer(BaseContainerImage): @staticmethod def version_to_container_os_version(os_version: OsVersion) -> str: - if os_version in (OsVersion.TUMBLEWEED, OsVersion.SLE16_0): - return "latest" - return f"15.{os_version}" + if os_version.is_sle15: + return f"15.{os_version}" + if os_version.is_slfo: + # FIXME: + # we'll probably have to find a better way here + return "16.0" + return "latest" @property def uid(self) -> str: @@ -1506,16 +1562,24 @@ def image_type(self) -> ImageType: return ImageType.SLE_BCI + @staticmethod + def build_tag_name_prefix(os_version: OsVersion) -> str: + """Prefix that is inserted in front of the name into the build tag""" + return "" if os_version.is_slfo else "bci-" + @property def build_tags(self) -> list[str]: tags: list[str] = [] + prefix = self.build_tag_name_prefix(self.os_version) for name in [self.name] + self.additional_names: tags += [ - f"{self._registry_prefix}/bci-{name}:%OS_VERSION_ID_SP%", - f"{self._registry_prefix}/bci-{name}:{self.image_ref_name}", + f"{self._registry_prefix}/{prefix}-{name}:%OS_VERSION_ID_SP%", + f"{self._registry_prefix}/{prefix}-{name}:{self.image_ref_name}", ] + ( - [f"{self._registry_prefix}/bci-{name}:latest"] if self.is_latest else [] + [f"{self._registry_prefix}/{prefix}{name}:latest"] + if self.is_latest + else [] ) return tags @@ -1525,7 +1589,10 @@ def image_ref_name(self) -> str: @property def reference(self) -> str: - return f"{self.registry}/{self._registry_prefix}/bci-{self.name}:{self.image_ref_name}" + return ( + f"{self.registry}/{self._registry_prefix}/" + + f"{self.build_tag_name_prefix(self.os_version)}{self.name}:{self.image_ref_name}" + ) @property def pretty_reference(self) -> str: @@ -1590,7 +1657,7 @@ def generate_disk_size_constraints(size_gb: int) -> str: f"{bci.uid}-{bci.os_version.pretty_print.lower()}": bci for bci in ( *BASE_CONTAINERS, - PYTHON_3_12_CONTAINERS, + *PYTHON_3_12_CONTAINERS, *PYTHON_3_6_CONTAINERS, *PYTHON_3_11_CONTAINERS, *PYTHON_TW_CONTAINERS, diff --git a/src/bci_build/package/appcontainers.py b/src/bci_build/package/appcontainers.py index bbea2b392..77dba0ea8 100644 --- a/src/bci_build/package/appcontainers.py +++ b/src/bci_build/package/appcontainers.py @@ -5,6 +5,7 @@ from bci_build.package import ALL_NONBASE_OS_VERSIONS from bci_build.package import CAN_BE_LATEST_OS_VERSION from bci_build.package import DOCKERFILE_RUN +from bci_build.package import SLCC_OS_VERSIONS from bci_build.package import ApplicationStackContainer from bci_build.package import BuildType from bci_build.package import OsContainer @@ -41,7 +42,7 @@ def _envsubst_pkg_name(os_version: OsVersion) -> str: name="pcp", pretty_name="Performance Co-Pilot (pcp)", custom_description="{pretty_name} container {based_on_container}. {podman_only}", - from_image=f"{_build_tag_prefix(os_version)}/bci-init:{OsContainer.version_to_container_os_version(os_version)}", + from_image=f"{_build_tag_prefix(os_version)}/{OsContainer.build_tag_name_prefix(os_version)}init:{OsContainer.version_to_container_os_version(os_version)}", os_version=os_version, is_latest=os_version in CAN_BE_LATEST_OS_VERSION, support_level=SupportLevel.L3, @@ -193,8 +194,23 @@ def _envsubst_pkg_name(os_version: OsVersion) -> str: """, ) for ver, os_version in ( - [(15, variant) for variant in (OsVersion.SP5, OsVersion.TUMBLEWEED)] - + [(16, variant) for variant in (OsVersion.SP6, OsVersion.TUMBLEWEED)] + [ + (15, variant) + for variant in ( + OsVersion.SP5, + OsVersion.TUMBLEWEED, + OsVersion.SLCC_PAID, + ) + ] + + [ + (16, variant) + for variant in ( + OsVersion.SLCC_FREE, + OsVersion.SLCC_PAID, + OsVersion.SP6, + OsVersion.TUMBLEWEED, + ) + ] ) + [(pg_ver, OsVersion.TUMBLEWEED) for pg_ver in (14, 13, 12)] ] @@ -235,6 +251,7 @@ def _generate_prometheus_family_healthcheck(port: int) -> str: custom_end=_generate_prometheus_family_healthcheck(_PROMETHEUS_PORT), ) for os_version in ALL_NONBASE_OS_VERSIONS + if not os_version.is_slfo ] _ALERTMANAGER_PACKAGE_NAME = "golang-github-prometheus-alertmanager" @@ -263,6 +280,7 @@ def _generate_prometheus_family_healthcheck(port: int) -> str: custom_end=_generate_prometheus_family_healthcheck(_ALERTMANAGER_PORT), ) for os_version in ALL_NONBASE_OS_VERSIONS + if not os_version.is_slfo ] _BLACKBOX_EXPORTER_PACKAGE_NAME = "prometheus-blackbox_exporter" @@ -291,6 +309,7 @@ def _generate_prometheus_family_healthcheck(port: int) -> str: custom_end=_generate_prometheus_family_healthcheck(_BLACKBOX_PORT), ) for os_version in ALL_NONBASE_OS_VERSIONS + if not os_version.is_slfo ] _GRAFANA_FILES = {} @@ -335,6 +354,7 @@ def _generate_prometheus_family_healthcheck(port: int) -> str: """, ) for os_version in ALL_NONBASE_OS_VERSIONS + if not os_version.is_slfo ] _NGINX_FILES = {} @@ -395,7 +415,7 @@ def _get_nginx_kwargs(os_version: OsVersion): pretty_name="NGINX for SUSE RMT", **_get_nginx_kwargs(os_version), ) - for os_version in (OsVersion.SP6,) + for os_version in (OsVersion.SP6, *SLCC_OS_VERSIONS) ] + [ ApplicationStackContainer( name="nginx", @@ -413,7 +433,7 @@ def _get_nginx_kwargs(os_version: OsVersion): support_level=SupportLevel.L3, pretty_name=f"{os_version.pretty_os_version_no_dash} with Git", custom_description="A micro environment with Git {based_on_container}.", - from_image=f"{_build_tag_prefix(os_version)}/bci-micro:{OsContainer.version_to_container_os_version(os_version)}", + from_image=f"{_build_tag_prefix(os_version)}/{OsContainer.build_tag_name_prefix(os_version)}micro:{OsContainer.version_to_container_os_version(os_version)}", build_recipe_type=BuildType.KIWI, is_latest=os_version in CAN_BE_LATEST_OS_VERSION, version="%%git_version%%", @@ -432,7 +452,7 @@ def _get_nginx_kwargs(os_version: OsVersion): "git-core", "openssh-clients", ) - + (() if os_version == OsVersion.TUMBLEWEED else ("skelcd-EULA-bci",)) + + os_version.eula_package_names ], # intentionally empty config_sh_script=""" @@ -447,7 +467,7 @@ def _get_nginx_kwargs(os_version: OsVersion): name="registry", package_name="distribution-image", pretty_name="OCI Container Registry (Distribution)", - from_image=f"{_build_tag_prefix(os_version)}/bci-micro:{OsContainer.version_to_container_os_version(os_version)}", + from_image=f"{_build_tag_prefix(os_version)}/{OsContainer.build_tag_name_prefix(os_version)}micro:{OsContainer.version_to_container_os_version(os_version)}", os_version=os_version, is_latest=os_version in CAN_BE_LATEST_OS_VERSION, version="%%registry_version%%", @@ -486,7 +506,7 @@ def _get_nginx_kwargs(os_version: OsVersion): ApplicationStackContainer( name="helm", pretty_name="Kubernetes Package Manager", - from_image=f"{_build_tag_prefix(os_version)}/bci-micro:{OsContainer.version_to_container_os_version(os_version)}", + from_image=f"{_build_tag_prefix(os_version)}/{OsContainer.build_tag_name_prefix(os_version)}micro:{OsContainer.version_to_container_os_version(os_version)}", os_version=os_version, is_latest=os_version in CAN_BE_LATEST_OS_VERSION, version=get_pkg_version("helm", os_version), @@ -512,7 +532,7 @@ def _get_nginx_kwargs(os_version: OsVersion): ApplicationStackContainer( name="trivy", pretty_name="Container Vulnerability Scanner", - from_image=f"{_build_tag_prefix(os_version)}/bci-micro:{OsContainer.version_to_container_os_version(os_version)}", + from_image=f"{_build_tag_prefix(os_version)}/{OsContainer.build_tag_name_prefix(os_version)}micro:{OsContainer.version_to_container_os_version(os_version)}", os_version=os_version, is_latest=os_version in CAN_BE_LATEST_OS_VERSION, version="%%trivy_version%%", diff --git a/src/bci_build/package/basecontainers.py b/src/bci_build/package/basecontainers.py index f9edc16ab..444314958 100644 --- a/src/bci_build/package/basecontainers.py +++ b/src/bci_build/package/basecontainers.py @@ -206,7 +206,12 @@ def _get_minimal_kwargs(os_version: OsVersion): Package(name, pkg_type=PackageType.BOOTSTRAP) for name in os_version.release_package_names ] - if os_version in (OsVersion.TUMBLEWEED, OsVersion.SLE16_0): + if os_version in ( + OsVersion.TUMBLEWEED, + OsVersion.SLCC_FREE, + OsVersion.SLCC_PAID, + OsVersion.SLE16_0, + ): package_list.append(Package("rpm", pkg_type=PackageType.BOOTSTRAP)) else: # in SLE15, rpm still depends on Perl. @@ -215,8 +220,10 @@ def _get_minimal_kwargs(os_version: OsVersion): for name in ("rpm-ndb", "perl-base") ] + micro_name = "micro" if os_version.is_slfo else "bci-micro" + kwargs = { - "from_image": f"{_build_tag_prefix(os_version)}/bci-micro:{OsContainer.version_to_container_os_version(os_version)}", + "from_image": f"{_build_tag_prefix(os_version)}/{micro_name}:{OsContainer.version_to_container_os_version(os_version)}", "pretty_name": f"{os_version.pretty_os_version_no_dash} Minimal", "package_list": package_list, } @@ -291,6 +298,9 @@ def _get_minimal_kwargs(os_version: OsVersion): if os_version == OsVersion.SLE16_0: prefix = "sle16" pretty_prefix = "SLE 16" + elif os_version.is_slfo and os_version != OsVersion.SLE16_0: + prefix = "slcc" + pretty_prefix = prefix.upper() else: prefix = "sle15" pretty_prefix = "SLE 15" diff --git a/src/bci_build/package/gcc.py b/src/bci_build/package/gcc.py index a1f6155aa..f102f7a5f 100644 --- a/src/bci_build/package/gcc.py +++ b/src/bci_build/package/gcc.py @@ -28,9 +28,9 @@ def _is_latest_gcc(os_version: OsVersion, gcc_version: _GCC_VERSIONS) -> bool: return True if os_version.is_sle15 and gcc_version == 13: return True - # if os_version in (OsVersion.SLCC_DEVELOPMENT, OsVersion.SLCC_PRODUCTION): - # assert gcc_version == 13 - # return True + if os_version.is_slfo: + assert gcc_version == 13 + return True return False @@ -39,9 +39,9 @@ def _is_main_gcc(os_version: OsVersion, gcc_version: _GCC_VERSIONS) -> bool: return True if os_version.is_sle15 and gcc_version == 7: return True - # if os_version in (OsVersion.SLCC_DEVELOPMENT, OsVersion.SLCC_PRODUCTION): - # assert gcc_version == 13 - # return True + if os_version.is_slfo: + assert gcc_version == 13 + return True return False @@ -98,8 +98,8 @@ def _is_main_gcc(os_version: OsVersion, gcc_version: _GCC_VERSIONS) -> bool: for (gcc_version, os_version) in ( (7, OsVersion.SP6), (13, OsVersion.SP6), - # (13, OsVersion.SLCC_DEVELOPMENT), - # (13, OsVersion.SLCC_PRODUCTION), + (13, OsVersion.SLCC_FREE), + (13, OsVersion.SLCC_PAID), (12, OsVersion.TUMBLEWEED), (13, OsVersion.TUMBLEWEED), (14, OsVersion.TUMBLEWEED), diff --git a/src/bci_build/package/golang.py b/src/bci_build/package/golang.py index df9aea497..c235ed7d7 100644 --- a/src/bci_build/package/golang.py +++ b/src/bci_build/package/golang.py @@ -93,18 +93,33 @@ def _get_golang_kwargs( **_get_golang_kwargs(ver, govariant, sle15sp), support_level=SupportLevel.L3, ) - for ver, govariant, sle15sp in product( - _GOLANG_VERSIONS, ("",), (OsVersion.SP6,) + for ver, govariant, sle15sp in list( + product( + _GOLANG_VERSIONS, + ("",), + (OsVersion.SP6, OsVersion.SLCC_PAID), + ) ) + + [(_GOLANG_VERSIONS[-1], "", OsVersion.SLCC_FREE)] ] + [ DevelopmentContainer( **_get_golang_kwargs(ver, govariant, sle15sp), support_level=SupportLevel.L3, ) - for ver, govariant, sle15sp in product( - _GOLANG_OPENSSL_VERSIONS, ("-openssl",), (OsVersion.SP6,) + for ver, govariant, sle15sp in list( + product(_GOLANG_OPENSSL_VERSIONS, ("-openssl",), (OsVersion.SP6,)) ) + # FIXME: add the remaining golang-openssl versions here once they are released + + [("1.21", "-openssl", OsVersion.SLCC_PAID)] + # only the latest version here + + [ + ( + _GOLANG_OPENSSL_VERSIONS[-1], + "-openssl", + OsVersion.SLCC_FREE, + ) + ] ] + [ DevelopmentContainer(**_get_golang_kwargs(ver, "", OsVersion.TUMBLEWEED)) diff --git a/src/bci_build/package/node.py b/src/bci_build/package/node.py index d18009186..7158ca037 100644 --- a/src/bci_build/package/node.py +++ b/src/bci_build/package/node.py @@ -63,4 +63,9 @@ def _get_node_kwargs(ver: _NODE_VERSIONS, os_version: OsVersion): ), DevelopmentContainer(**_get_node_kwargs(20, OsVersion.TUMBLEWEED)), DevelopmentContainer(**_get_node_kwargs(22, OsVersion.TUMBLEWEED)), + DevelopmentContainer(**_get_node_kwargs(20, OsVersion.SLCC_FREE)), + DevelopmentContainer( + **_get_node_kwargs(20, OsVersion.SLCC_PAID), + support_level=SupportLevel.L3, + ), ] diff --git a/src/bci_build/package/openjdk.py b/src/bci_build/package/openjdk.py index 2f238e152..c595e42ee 100644 --- a/src/bci_build/package/openjdk.py +++ b/src/bci_build/package/openjdk.py @@ -91,7 +91,13 @@ def _get_openjdk_kwargs( support_level=SupportLevel.L3, ) for os_version, devel in product( - (OsVersion.SP6, OsVersion.TUMBLEWEED), (True, False) + ( + OsVersion.SP6, + OsVersion.SLCC_PAID, + OsVersion.SLCC_FREE, + OsVersion.TUMBLEWEED, + ), + (True, False), ) ] + [ diff --git a/src/bci_build/package/php.py b/src/bci_build/package/php.py index 876f652b2..8e407e519 100644 --- a/src/bci_build/package/php.py +++ b/src/bci_build/package/php.py @@ -193,7 +193,12 @@ def _create_php_bci( PHP_CONTAINERS = [ _create_php_bci(os_version, variant, 8) for os_version, variant in product( - (OsVersion.SP6, OsVersion.TUMBLEWEED), + ( + OsVersion.SP6, + OsVersion.TUMBLEWEED, + OsVersion.SLCC_FREE, + OsVersion.SLCC_PAID, + ), (PhpVariant.cli, PhpVariant.apache, PhpVariant.fpm), ) ] diff --git a/src/bci_build/package/python.py b/src/bci_build/package/python.py index 6d4b2e52c..5dc5f9b3e 100644 --- a/src/bci_build/package/python.py +++ b/src/bci_build/package/python.py @@ -109,7 +109,7 @@ def _get_python_kwargs(py3_ver: _PYTHON_VERSIONS, os_version: OsVersion): PYTHON_TW_CONTAINERS = ( PythonDevelopmentContainer( **_get_python_kwargs(pyver, OsVersion.TUMBLEWEED), - is_latest=pyver == _PYTHON_TW_VERSIONS[-1], + is_latest=(pyver == _PYTHON_TW_VERSIONS[-1]), package_name=f"python-{pyver}-image", ) for pyver in _PYTHON_TW_VERSIONS @@ -119,12 +119,26 @@ def _get_python_kwargs(py3_ver: _PYTHON_VERSIONS, os_version: OsVersion): PythonDevelopmentContainer( **_get_python_kwargs("3.11", os_version), package_name="python-3.11-image", + is_latest=( + (os_version in CAN_BE_LATEST_OS_VERSION) + and (os_version != OsVersion.SLCC_PAID) + ), ) - for os_version in (OsVersion.SP6,) + for os_version in (OsVersion.SP6, OsVersion.SLCC_PAID) ) -PYTHON_3_12_CONTAINERS = PythonDevelopmentContainer( - **_get_python_kwargs("3.12", OsVersion.SP6), - package_name="python-3.12-image", - is_latest=OsVersion.SP6 in CAN_BE_LATEST_OS_VERSION, -) + +PYTHON_3_12_CONTAINERS = [ + PythonDevelopmentContainer( + **_get_python_kwargs("3.12", os_version), + package_name="python-3.12-image", + # Technically it is the latest but we want to prefer the long term + # Python 3.11 for SLE 15 & TW + is_latest=os_version.is_slfo, + ) + for os_version in ( + OsVersion.SP6, + OsVersion.SLCC_PAID, + OsVersion.SLCC_FREE, + ) +] diff --git a/src/bci_build/package/rmt.py b/src/bci_build/package/rmt.py index 474182291..e76fc6916 100644 --- a/src/bci_build/package/rmt.py +++ b/src/bci_build/package/rmt.py @@ -39,4 +39,5 @@ """, ) for os_version in ALL_NONBASE_OS_VERSIONS + if not os_version.is_slfo ] diff --git a/src/bci_build/package/ruby.py b/src/bci_build/package/ruby.py index 5d7768697..cc4666e91 100644 --- a/src/bci_build/package/ruby.py +++ b/src/bci_build/package/ruby.py @@ -11,7 +11,7 @@ from bci_build.package import generate_disk_size_constraints -def _get_ruby_kwargs(ruby_version: Literal["2.5", "3.3"], os_version: OsVersion): +def _get_ruby_kwargs(ruby_version: Literal["2.5", "3.2", "3.3"], os_version: OsVersion): ruby = f"ruby{ruby_version}" ruby_major = ruby_version.split(".")[0] @@ -66,5 +66,11 @@ def _get_ruby_kwargs(ruby_version: Literal["2.5", "3.3"], os_version: OsVersion) **_get_ruby_kwargs("2.5", OsVersion.SP6), support_level=SupportLevel.L3, ), + # FIXME: enable this for SLCC v2 + # DevelopmentContainer(**_get_ruby_kwargs("3.2", OsVersion.SLCC_DEVELOPMENT)), + # DevelopmentContainer( + # **_get_ruby_kwargs("3.2", OsVersion.SLCC_PRODUCTION), + # support_level=SupportLevel.L3, + # ), DevelopmentContainer(**_get_ruby_kwargs("3.3", OsVersion.TUMBLEWEED)), ] diff --git a/src/bci_build/package/rust.py b/src/bci_build/package/rust.py index 8f623a7f3..1c40b7044 100644 --- a/src/bci_build/package/rust.py +++ b/src/bci_build/package/rust.py @@ -6,6 +6,7 @@ from bci_build.package import ALL_NONBASE_OS_VERSIONS from bci_build.package import CAN_BE_LATEST_OS_VERSION from bci_build.package import DevelopmentContainer +from bci_build.package import OsVersion from bci_build.package import Replacement from bci_build.package import SupportLevel from bci_build.package import generate_disk_size_constraints @@ -86,8 +87,11 @@ COPY {check_fname} /etc/zypp/systemCheck.d/{check_fname} """, ) - for rust_version, os_version in product( - _RUST_VERSIONS, - ALL_NONBASE_OS_VERSIONS, + for rust_version, os_version in list( + product( + _RUST_VERSIONS, + set(ALL_NONBASE_OS_VERSIONS).difference({OsVersion.SLCC_FREE}), + ) ) + + [(_RUST_VERSIONS[-1], OsVersion.SLCC_FREE)] ] diff --git a/src/staging/bot.py b/src/staging/bot.py index e77f3e409..95e5ec8b4 100644 --- a/src/staging/bot.py +++ b/src/staging/bot.py @@ -13,10 +13,15 @@ from dataclasses import field from datetime import datetime from datetime import timedelta +from enum import Enum +from enum import unique from functools import reduce +from io import BytesIO +from pathlib import Path from typing import ClassVar from typing import Literal -from typing import TypedDict +from typing import NoReturn +from typing import overload import aiofiles.os import aiofiles.tempfile @@ -32,12 +37,15 @@ from bci_build.package import ALL_CONTAINER_IMAGE_NAMES from bci_build.package import BaseContainerImage from bci_build.package import OsVersion +from bci_build.package import Package from dotnet.updater import DOTNET_IMAGES from dotnet.updater import DotNetBCI -from staging.build_result import Arch from staging.build_result import PackageBuildResult from staging.build_result import PackageStatusCode from staging.build_result import RepositoryBuildResult +from staging.project_setup import ProjectType +from staging.project_setup import generate_meta +from staging.project_setup import generate_project_name from staging.user import User from staging.util import ensure_absent from staging.util import get_obs_project_url @@ -71,21 +79,37 @@ OS_VERSION_NEEDS_BASE_CONTAINER: tuple[OsVersion, ...] = () +@overload +def _get_base_image_prj_pkg( + os_version: Literal[ + OsVersion.SLCC_FREE, + OsVersion.SLCC_PAID, + OsVersion.SLE16_0, + ], +) -> NoReturn: ... + + +@overload +def _get_base_image_prj_pkg(os_version: OsVersion) -> tuple[str, str]: ... + + def _get_base_image_prj_pkg(os_version: OsVersion) -> tuple[str, str]: if os_version == OsVersion.TUMBLEWEED: return "openSUSE:Factory", "opensuse-tumbleweed-image" - if os_version == OsVersion.SLE16_0: - raise ValueError("The SLFO base container is provided by BCI") + if os_version.is_slfo: + raise ValueError("The SLFO base containers are provided by the project itself") return f"SUSE:SLE-15-SP{os_version}:Update", "sles15-image" def _get_bci_project_name(os_version: OsVersion) -> str: - prj_suffix = ( - os_version - if os_version in (OsVersion.TUMBLEWEED, OsVersion.SLE16_0) - else "SLE-15-SP" + str(os_version) - ) + if os_version.is_sle15: + prj_suffix = f"SLE-15-SP{os_version}" + elif os_version.is_slfo: + prj_suffix = str(os_version).replace("-", ":") + else: + prj_suffix = str(os_version) + return f"devel:BCI:{prj_suffix}" @@ -102,9 +126,421 @@ async def _fetch_bci_devel_project_config( return await response.text() -class _ProjectConfigs(TypedDict): - meta: ET.Element - prjconf: str +@unique +class ProjectConfig(Enum): + PRJCONF = "prjconf" + META = "prj" + + +_PACKAGE_GROUP_NAME = "slcc_packages" + + +@dataclass(frozen=True) +class TripleZeroPackageGroups: + """000package-groups""" + + os_version: OsVersion + + architectures: list[str] = field( + default_factory=lambda: ["x86_64", "aarch64", "s390x", "ppc64le"] + ) + + def __post_init__(self) -> None: + if not self.os_version.is_slcc: + raise ValueError(f"Only implemented for SLCC, not for {self.os_version}") + + @property + def release_spec_in(self) -> str: + pkg_name = f"{str(self.os_version).lower()}-release" + return ( + f"""# +# spec file for package {pkg_name} +# +# Copyright (c) {datetime.now().year} SUSE LLC +# +# All modifications and additions to the file contributed by third parties +# remain the property of their copyright owners, unless otherwise agreed +# upon. The license for this file, and modifications and additions to the +# file, is the same license as for the pristine package itself (unless the +# license for the pristine package is not an Open Source License, in which +# case the license is the MIT License). An "Open Source License" is a +# license that conforms to the Open Source Definition (Version 1.9) +# published by the Open Source Initiative. + +# Please submit bugfixes or comments via https://bugzilla.suse.com/ +# + +Name: {pkg_name} +Summary: ___SUMMARY___ ___BETA_VERSION___ +License: MIT +Group: System/Fhs +Version: ___VERSION___ +Release: 0 +# FIXME? or keep this package name +BuildRequires: skelcd-EULA-{str(self.os_version).lower()} +Provides: distribution-release +""" + + """Provides: product(SUSE_SLE) = %{version}-%{release} +Provides: product(SUSE_SLE-SP___PATCH_LEVEL___) = %{version}-%{release} + +# bsc#1055299 +Conflicts: otherproviders(distribution-release) + +___PRODUCT_PROVIDES___ + +___PRODUCT_DEPENDENCIES___ + + +ExclusiveArch: """ + + " ".join(self.architectures) + + """ + +Source100: weakremovers.inc +%include %{SOURCE100} + +%description +___DESCRIPTION___ + +___FLAVOR_PACKAGES___ + +%prep + +%build + +%install +mkdir -p %buildroot/%_sysconfdir + +___CREATE_OS_RELEASE_FILE___ + +cat << EOF >> %buildroot/%_sysconfdir/os-release +DOCUMENTATION_URL="https://documentation.suse.com/" +EOF + +___CREATE_PRODUCT_FILES___ + + +%files +%defattr(644,root,root,755) +%config %_sysconfdir/os-release +%dir %_sysconfdir/products.d +%_sysconfdir/products.d/* + +%changelog +""" + ) + + @property + def product_in(self) -> str: + pool_prefix = f"{self.os_version.value}-{self.os_version.os_version}" + return ( + f""" + + + + SUSE + {str(self.os_version).lower()} + {self.os_version.os_version} + 1 + + + + {self.os_version.full_os_name} + + + + {self.os_version.pretty_os_version_no_dash} + + + + +""" + + "\n".join( + f""" + + """ + for (medium, alias), arch in itertools.product( + [("", ""), ("debug_", "-Debuginfo"), ("source_", "-Source")], + self.architectures, + ) + ) + + """ + + +""" + + "\n".join( + f""" sle-15-{arch}""" + for arch in self.architectures + ) + + """ + + + + + + {self.os_version.full_os_name} + {self.os_version.full_os_name} + {self.os_version.full_os_name} + + + + en + + + + + https://www.suse.com/releasenotes/%{{_target_cpu}}/SL-Micro/6.0/release-notes-sl-micro.rpm + + + + + {self.os_version} + + + + + + en_US + SUSE + + + + + + + + + + + + + + + + + + + + + + +""" + + "\n".join( + f""" + + """ + for arch in self.architectures + ) + + f""" + + + + + + + +""" + ) + + @property + def default_productcompose_in(self) -> str: + return f"""product_compose_schema: 0.2 + +vendor: SUSE +name: {self.os_version.value} +version: {self.os_version.os_version} +product-type: module +summary: {self.os_version.full_os_name} + +scc: + description: > + Alp Basalt ftp tree, also known as POOL. + Used for GA and maintenance update afterwards. + +build_options: +### For maintenance, otherwise only "the best" version of each package is picked: +# - take_all_available_versions +- hide_flavor_in_product_directory_name + + +source: split +debug: split + +# has only an effect during maintenance: +set_updateinfo_from: maint-coord@suse.de + +# will be extended with architecture and flavor string +# product_directory_name: "ALP-Dolomite-1.0" + +flavors: + {_PACKAGE_GROUP_NAME}_aarch64: + architectures: [ aarch64 ] + {_PACKAGE_GROUP_NAME}_ppc64le: + architectures: [ ppc64le ] + {_PACKAGE_GROUP_NAME}_s390x: + architectures: [ s390x ] + {_PACKAGE_GROUP_NAME}_x86_64: + architectures: [ x86_64 ] + +unpack: + - unpackset + +packagesets: +- name: unpackset + packages: + - skelcd-EULA-{str(self.os_version.value).lower()} + +# The following is generated by openSUSE-release-tools + +This part will get replaced by pkglistgen and the file will get written to +000productcompose sub directory. + +""" + + +@dataclass(frozen=True) +class SkelcdPackage: + os_version: OsVersion + + @property + def spec(self) -> str: + return ( + f"""# +# spec file for package skelcd +# +# Copyright (c) 2024 SUSE LLC. +# +# All modifications and additions to the file contributed by third parties +# remain the property of their copyright owners, unless otherwise agreed +# upon. The license for this file, and modifications and additions to the +# file, is the same license as for the pristine package itself (unless the +# license for the pristine package is not an Open Source License, in which +# case the license is the MIT License). An "Open Source License" is a +# license that conforms to the Open Source Definition (Version 1.9) +# published by the Open Source Initiative. + +# Please submit bugfixes or comments via http://bugs.opensuse.org/ +# +%define SLE_RELEASE 16 +# +# default replacement variables for README content +%define PRETTY_NAME {self.os_version.pretty_os_version_no_dash} +%define UNDERLINE =================================== +%define PRODUCT_LINK https://www.suse.com/sles + +%define product {str(self.os_version.value).lower()} +%define PRODUCT {str(self.os_version.value).upper()} +""" + + """ +%define dash - + +%define container_path usr/share/licenses/product/%{PRODUCT} +%define skelcd1_path usr/share/licenses/product/%{product} + +# release is a beta +%define beta 0 + +%if 0%{?beta} == 1 +%define license_dir license.beta +%else +%define license_dir license.final +%endif + +%dnl %define skelcd1_path usr/lib/skelcd/CD1 + +Name: skelcd%{?dash}%{product} + + +AutoReqProv: off +Version: 2024.05.03.1 +Release: 0 +Summary: CD skeleton for %{PRODUCT} +License: GPL-2.0-only +Group: Metapackages +BuildRoot: %{_tmppath}/%{name}-%{version}-build +Source: skelcd-%{version}.tar.xz +# please repo-checker (bsc#1089174) +Provides: skelcd = %{version} +Conflicts: otherproviders(skelcd) + +%description +Skeleton package for %{PRODUCT} + +%package -n skelcd-EULA%{?dash}%{product} +Summary: EULA for media +Group: Metapackages + +%description -n skelcd-EULA%{?dash}%{product} +Internal package only. + + +%prep +%setup -n skelcd%{?dash}%{version} -q + +%build + +%install +# +# copy the product READMEs +pushd READMEs/default +sed -i -e 's/{PRETTY_NAME}/%{PRETTY_NAME} %{SLE_RELEASE}/g' README +sed -i -e 's/{UNDERLINE}/%{UNDERLINE}/g' README +# use @ as delimiter, as the product link conflicts with the standard '/' delimiter +sed -i -e 's@{PRODUCT_LINK}@%{PRODUCT_LINK}@g' README +popd + +# +# license tarball generation +mkdir -p $RPM_BUILD_ROOT/%{skelcd1_path}/media.1 +pushd %license_dir +# touch all license files to make sure they have the most recent date +# this impacts which license is shown on the CDN to fix bsc#1186047 and bsc#1186812 +# else in case beta EULAs have a more recent date than final EULAs they won't +# get replaced +touch * +ls -1 > directory.yast # required for downloading of EULAs from SCC + +# bci doesn't have a release package, make EULA available directly +rmdir $RPM_BUILD_ROOT/%{skelcd1_path}/media.1 +mv ../BCI/*.txt $RPM_BUILD_ROOT/%{skelcd1_path}/ + +popd + +# +# skelcd-EULA +%files -n skelcd-EULA-%{product} +%defattr(644,root,root,755) +%dir %{_datadir}/licenses/product +/%{skelcd1_path} + +%changelog +""" + ) @dataclass @@ -184,6 +620,26 @@ def __post_init__(self) -> None: if not self.osc_username: raise RuntimeError("osc_username is not set, cannot continue") + def _read_file_from_branch(self, branch_name: str, file_name: str) -> bytes: + tmp = BytesIO() + try: + git.Repo(Path(__file__).parent.parent.parent).commit(branch_name).tree[ + file_name + ].stream_data(tmp) + return tmp.getvalue() + except KeyError: + raise ValueError(f"File {file_name} not found in branch {branch_name}") + + @property + def _devel_project_prjconf(self) -> bytes: + """Returns the saved prjconf of the corresponding ``devel:BCI:$subname`` + project from git + + """ + return self._read_file_from_branch( + f"origin/{self.deployment_branch_name}", "_config" + ) + @property def _bcis(self) -> Generator[BaseContainerImage, None, None]: """Generator yielding all @@ -196,19 +652,10 @@ def _bcis(self) -> Generator[BaseContainerImage, None, None]: all_bcis.sort(key=lambda bci: bci.uid) return (bci for bci in all_bcis if bci.os_version == self.os_version) - def _generate_project_name(self, prefix: str) -> str: - assert self.osc_username - res = f"home:{self.osc_username}:{prefix}:" - if self.os_version in (OsVersion.TUMBLEWEED, OsVersion.SLE16_0): - res += str(self.os_version) - else: - res += f"SLE-15-SP{str(self.os_version)}" - return res - @property def continuous_rebuild_project_name(self) -> str: """The name of the continuous rebuild project on OBS.""" - return self._generate_project_name("BCI:CR") + return generate_project_name(self.os_version, ProjectType.CR, self.osc_username) @property def staging_project_name(self) -> str: @@ -222,7 +669,9 @@ def staging_project_name(self) -> str: - ``BRANCH``: :py:attr:`branch_name` """ - return self._generate_project_name("BCI:Staging") + ":" + self.branch_name + return generate_project_name( + self.os_version, ProjectType.STAGING, self.osc_username, self.branch_name + ) @property def staging_project_url(self) -> str: @@ -274,6 +723,29 @@ def bcis(self) -> Generator[BaseContainerImage, None, None]: ) ) + @property + def groups_yml(self) -> str: + """Generate a the ``container_packages`` YAML list for + :file:`groups.yml` in ``000package-groups`` from the container images + for this code stream. + + .. caution:: Only works for SLCC! + + """ + if not self.os_version.is_slcc: + raise ValueError("Only supported for SLCC code streams") + + res = "container_packages:" + + for bci in sorted(list(self._bcis), key=lambda b: b.uid): + res += f"\n # {bci.uid}\n - " + res += "\n - ".join( + pkg.name if isinstance(pkg, Package) else pkg + for pkg in bci.package_list + ) + + return res + @staticmethod def from_github_comment(comment_text: str, osc_username: str) -> "StagingBot": if comment_text == "": @@ -564,96 +1036,38 @@ def _osc(self) -> str: "osc" if not self._osc_conf_file else f"osc --config={self._osc_conf_file}" ) - async def _generate_test_project_meta(self, target_project_name: str) -> ET.Element: - bci_devel_meta = ET.fromstring( - await _fetch_bci_devel_project_config(self.os_version, "meta") - ) - - # write the same project meta as devel:BCI, but replace the 'devel:BCI:*' - # with the target project name in the main element and in all repository - # path entries - bci_devel_meta.attrib["name"] = target_project_name - - # we will remove the helmchartsrepo as we do not need it - repo_names = [] - repos_to_remove = [] - - # ppc64le & s390x are mostly busted on TW and just cause pointless - # build failures, so we don't build them - # Also, we don't use the local architecture, so drop that one always - arches_to_drop = [str(Arch.LOCAL)] - arches_to_drop.extend( - [str(Arch.PPC64LE), str(Arch.S390X)] - if self.os_version == OsVersion.TUMBLEWEED - else [] - ) - - for elem in bci_devel_meta: - if elem.tag == "repository": - if "name" in elem.attrib: - if (name := elem.attrib["name"]) in ("helmcharts", "standard"): - if name == "helmcharts": - repos_to_remove.append(elem) - continue - - repo_names.append(name) - else: - raise ValueError( - f"Invalid element, missing 'name' attribute: {ET.tostring(elem).decode()}" - ) - - for repo_elem in elem.iter(tag="path"): - if ( - "project" in repo_elem.attrib - and "devel:BCI:" in repo_elem.attrib["project"] - ): - repo_elem.attrib["project"] = target_project_name - - arch_entries = list(elem.iter(tag="arch")) - for arch_entry in arch_entries: - if arch_entry.text in arches_to_drop: - elem.remove(arch_entry) - - container_repos = ("containerfile", "images") - if name in container_repos: - for repo_name in container_repos: - (bci_devel_prj_path := ET.Element("path")).attrib["project"] = ( - _get_bci_project_name(self.os_version) - ) - bci_devel_prj_path.attrib["repository"] = repo_name - - elem.insert(0, bci_devel_prj_path) - - self.repositories = repo_names - for repo_to_remove in repos_to_remove: - bci_devel_meta.remove(repo_to_remove) - - person = ET.Element( - "person", {"userid": self.osc_username, "role": "maintainer"} - ) - bci_devel_meta.append(person) - - return bci_devel_meta - - async def _send_prj_meta( - self, target_project_name: str, prj_meta: ET.Element + async def _send_prj_config( + self, + target_project_name: str, + config: ET.Element | str | bytes, + config_type: ProjectConfig = ProjectConfig.META, ) -> None: """Set the meta of the project on OBS with the name ``target_project_name`` to the config ``prj_meta``. """ + if isinstance(config, ET.Element) and config_type == ProjectConfig.PRJCONF: + raise ValueError("Cannot set the prjconf from a XML Element") + async with aiofiles.tempfile.NamedTemporaryFile(mode="wb") as tmp_meta: - await tmp_meta.write(ET.tostring(prj_meta)) + if isinstance(config, str): + data = config.encode() + elif isinstance(config, ET.Element): + data = ET.tostring(config) + else: + data = config + + await tmp_meta.write(data) await tmp_meta.flush() - async def _send_prj_meta(): + async def _send_meta(): await self._run_cmd( - f"{self._osc} meta prj --file={tmp_meta.name} {target_project_name}" + f"{self._osc} meta {config_type.value} --file={tmp_meta.name} {target_project_name}" ) # obs sometimes dies setting the project meta with SQL errors 🤯 # so we just try again… - await retry_async_run_cmd(_send_prj_meta) + await retry_async_run_cmd(_send_meta) async def write_cr_project_config(self) -> None: """Send the configuration of the continuous rebuild project to OBS. @@ -662,56 +1076,34 @@ async def write_cr_project_config(self) -> None: then its configuration (= ``meta`` in OBS jargon) will be updated. """ - meta = await self._generate_test_project_meta( - self.continuous_rebuild_project_name + prj_name, meta = generate_meta( + self.os_version, ProjectType.CR, self.osc_username ) - ( - scmsync := ET.Element("scmsync") - ).text = f"https://github.com/SUSE/bci-dockerfile-generator#{self.deployment_branch_name}" - meta.append(scmsync) - await self._send_prj_meta(self.continuous_rebuild_project_name, meta) + await self._send_prj_config(prj_name, meta, ProjectConfig.META) async def write_staging_project_configs(self) -> None: """Submit the ``prjconf`` and ``meta`` to the test project on OBS. - The ``prjconf`` is taken directly from the development project on OBS - (``devel:BCI:*``). + The ``meta`` is generated using a template via + py.func:`~staging.project_setup.generate_project_name`. - The ``meta`` has to be modified slightly: + The ``prjconf`` is taken from the file:`_config` file in the deployment + branch. - - we remove the ``helmcharts`` repository (we don't create anything for - that repo, so not worth creating it) - - change the path from ``devel:BCI:*`` to the staging project name - - add the bot user as the maintainer (otherwise you can't do anything in - the project anymore…) - - Then we send the ``meta`` and then the ``prjconf``. """ - confs: _ProjectConfigs = {} - - async def _fetch_prjconf(): - confs["prjconf"] = await _fetch_bci_devel_project_config( - self.os_version, "prjconf" - ) - - async def _fetch_prj(): - confs["meta"] = await self._generate_test_project_meta( - self.staging_project_name - ) - await asyncio.gather(_fetch_prj(), _fetch_prjconf()) + prj_name, prj_meta = generate_meta( + self.os_version, ProjectType.STAGING, self.osc_username, self.branch_name + ) # First set the project meta! This will create the project if it does not # exist already, if we do it asynchronously, then the prjconf might be # written before the project exists, which fails - await self._send_prj_meta(self.staging_project_name, confs["meta"]) + await self._send_prj_config(prj_name, prj_meta, ProjectConfig.META) - async with aiofiles.tempfile.NamedTemporaryFile(mode="w") as tmp_prjconf: - await tmp_prjconf.write(confs["prjconf"]) - await tmp_prjconf.flush() - await self._run_cmd( - f"{self._osc} meta prjconf --file={tmp_prjconf.name} {self.staging_project_name}" - ) + await self._send_prj_config( + prj_name, self._devel_project_prjconf, ProjectConfig.PRJCONF + ) def _osc_fetch_results_cmd(self, extra_osc_flags: str = "") -> str: return ( @@ -1500,6 +1892,26 @@ def get_packages_without_changelog_addition( if not changelog_updated ] + async def configure_devel_bci_project(self) -> None: + """Adjust to project meta of the devel project on OBS to match the + template generated via + :py:func:`staging.project_setup.generate_meta`. Additionally set the + project meta from the file :file:`_config` in the deployment branch and + set the `OSRT:Config` attribute for pkglistgen to function as expected. + + """ + prj_name, meta = generate_meta( + self.os_version, ProjectType.DEVEL, self.osc_username + ) + await self._send_prj_config(prj_name, meta, ProjectConfig.META) + + await self._send_prj_config( + prj_name, self._devel_project_prjconf, ProjectConfig.PRJCONF + ) + + await self._run_cmd(f"""{self._osc} meta attribute {prj_name} -a OSRT:Config --set 'main-repo = standard +pkglistgen-archs = ppc64le s390x aarch64 x86_64'""") + async def configure_devel_bci_package(self, package_name: str) -> None: bci = [b for b in self._bcis if b.package_name == package_name] @@ -1565,6 +1977,8 @@ def main() -> None: "add_changelog_entry", "changelog_check", "setup_obs_package", + "setup_obs_project", + "000package-groups", "find_missing_packages", ] @@ -1739,11 +2153,37 @@ def add_commit_message_arg(p: argparse.ArgumentParser) -> None: + [dotnet_img.package_name for dotnet_img in DOTNET_IMAGES], ) + subparsers.add_parser( + "setup_obs_project", help="Configure the devel project on OBS" + ) + + subparsers.add_parser( + "groups_yml", + help="Create a list of all container packages that can be inserted into groups.yml", + ) + subparsers.add_parser( "find_missing_packages", help="Find all packages that are in the deployment branch and are missing from `devel:BCI:*` on OBS", ) + triple_zero_parser = subparsers.add_parser( + "000package-groups", help="generate 000package-groups files" + ) + triple_zero_parser.add_argument( + "--file", + nargs=1, + required=True, + type=str, + choices=[ + "groups.yml", + "release.spec.in", + "product.in", + "skelcd", + "default.productcompose.in", + ], + ) + loop = asyncio.get_event_loop() args = parser.parse_args() @@ -1781,10 +2221,10 @@ def add_commit_message_arg(p: argparse.ArgumentParser) -> None: try: action: ACTION_T = args.action - coro: Coroutine[Any, Any, Any] | None = None + coro_or_str: Coroutine[Any, Any, Any] | str | None = None if action == "rebuild": - coro = bot.force_rebuild() + coro_or_str = bot.force_rebuild() elif action == "create_staging_project": @@ -1797,17 +2237,17 @@ async def _create_staging_proj(): ) await bot.link_base_container_to_staging() - coro = _create_staging_proj() + coro_or_str = _create_staging_proj() elif action == "commit_state": - coro = bot.write_all_build_recipes_to_branch(args.commit_message[0]) + coro_or_str = bot.write_all_build_recipes_to_branch(args.commit_message[0]) elif action == "query_build_result": async def print_build_res(): return render_as_markdown(await bot.fetch_build_results()) - coro = print_build_res() + coro_or_str = print_build_res() elif action == "scratch_build": @@ -1815,10 +2255,10 @@ async def _scratch(): commit_or_none = await bot.scratch_build(args.commit_message[0]) return commit_or_none or "No changes" - coro = _scratch() + coro_or_str = _scratch() elif action == "cleanup": - coro = bot.remote_cleanup( + coro_or_str = bot.remote_cleanup( branches=not args.no_cleanup_branch, obs_project=not args.no_cleanup_project, ) @@ -1830,7 +2270,7 @@ async def _wait(): await bot.wait_for_build_to_finish(timeout_sec=args.timeout_sec[0]) ) - coro = _wait() + coro_or_str = _wait() elif action == "get_build_quality": @@ -1840,10 +2280,10 @@ async def _quality(): raise RuntimeError("Build failed!") return "Build succeded" - coro = _quality() + coro_or_str = _quality() elif action == "create_cr_project": - coro = bot.write_cr_project_config() + coro_or_str = bot.write_cr_project_config() elif action == "add_changelog_entry": changelog_entry = " ".join(args.entry) @@ -1855,7 +2295,7 @@ async def _quality(): elif packages_len > 1: pkg_names = args.packages - coro = bot.add_changelog_entry( + coro_or_str = bot.add_changelog_entry( entry=changelog_entry, username=username, package_names=pkg_names ) @@ -1874,7 +2314,7 @@ async def _error_on_pkg_without_changes(): f"{change_ref}: {', '.join(packages_without_changes)}" ) - coro = _error_on_pkg_without_changes() + coro_or_str = _error_on_pkg_without_changes() elif action == "setup_obs_package": async def _setup_pkg_meta(): @@ -1884,21 +2324,44 @@ async def _setup_pkg_meta(): ] await asyncio.gather(*tasks) - coro = _setup_pkg_meta() + coro_or_str = _setup_pkg_meta() + + elif action == "setup_obs_project": + coro_or_str = bot.configure_devel_bci_project() + + elif action == "000package-groups": + if (fname := args.file[0]) == "groups.yml": + coro_or_str = bot.groups_yml + elif fname == "product.in": + coro_or_str = TripleZeroPackageGroups(bot.os_version).product_in + elif fname == "release.spec.in": + coro_or_str = TripleZeroPackageGroups(bot.os_version).release_spec_in + elif fname == "default.productcompose.in": + coro_or_str = TripleZeroPackageGroups( + bot.os_version + ).default_productcompose_in + elif fname == "skelcd": + coro_or_str = SkelcdPackage(bot.os_version).spec + else: + raise ValueError(f"Invalid file for 000package-groups: {fname}") elif action == "find_missing_packages": async def _pkgs_as_str() -> str: return ", ".join(await bot.find_missing_packages_on_obs()) - coro = _pkgs_as_str() + coro_or_str = _pkgs_as_str() else: assert False, f"invalid action: {action}" - assert coro is not None - res = loop.run_until_complete(coro) - if res: - print(res) + assert coro_or_str is not None + + if isinstance(coro_or_str, str): + print(coro_or_str) + else: + res = loop.run_until_complete(coro_or_str) + if res: + print(res) finally: loop.run_until_complete(bot.teardown()) diff --git a/src/staging/project_setup.py b/src/staging/project_setup.py new file mode 100644 index 000000000..a92aa3cf1 --- /dev/null +++ b/src/staging/project_setup.py @@ -0,0 +1,184 @@ +from enum import Enum +from enum import auto +from enum import unique + +import jinja2 + +from bci_build.package import OsVersion + +USERS_FOR_PRODUCTION = [ + "avicenzi", + "dirkmueller", + "dancermak", + "favogt", + "fcrozat", + "pvlasin", +] + +USERS_FOR_STAGING = ["avicenzi", "dancermak"] + +META_TEMPLATE = jinja2.Template(""" + {{ project_title }} +{% if project_description %} {{ project_description }}{% else %} {% endif %} +{% for user in maintainers %} +{% endfor %}{% if extra_header %} +{{ extra_header }}{% endif %} + + + + + + + + + + +{% for prj, repo in repository_paths %} +{% endfor %} x86_64 + aarch64 +{% if with_all_arches %} s390x + ppc64le{% endif %} + + + + + x86_64 + aarch64{% if with_all_arches %} + s390x + ppc64le{% endif %} + {% if with_product_repo %} + + + + + x86_64 + aarch64{% if with_all_arches %} + s390x + ppc64le{% endif %} + {% endif %}{% if with_helmcharts_repo %} + + + x86_64 + {% endif %} + + + + x86_64 + aarch64{% if with_all_arches %} + s390x + ppc64le{% endif %} + + +""") + + +@unique +class ProjectType(Enum): + DEVEL = auto() + CR = auto() + STAGING = auto() + + +def generate_project_name( + os_version: OsVersion, + project_type: ProjectType, + osc_username: str, + branch_name: str | None = None, +) -> str: + res = { + ProjectType.DEVEL: "devel:BCI:", + ProjectType.CR: f"home:{osc_username}:BCI:CR:", + ProjectType.STAGING: f"home:{osc_username}:BCI:Staging:", + }[project_type] + + if os_version.is_sle15: + res += f"SLE-15-SP{str(os_version)}" + elif os_version.is_slcc: + res += "SLCC:" + str(os_version).split("-", 1)[1] + else: + res += str(os_version) + + if project_type == ProjectType.STAGING: + if not branch_name: + raise ValueError("staging projects need a branch name") + res += ":" + branch_name + + return res + + +def generate_meta( + os_version: OsVersion, + project_type: ProjectType, + osc_username: str, + branch_name: None | str = None, +) -> tuple[str, str]: + prj_name = generate_project_name( + os_version, project_type, osc_username, branch_name + ) + + if project_type == ProjectType.DEVEL: + users = USERS_FOR_PRODUCTION + else: + users = USERS_FOR_STAGING + [osc_username] + + with_all_arches = (os_version != OsVersion.TUMBLEWEED) or ( + project_type in (ProjectType.CR, ProjectType.DEVEL) + ) + with_helmcharts_repo = project_type == ProjectType.DEVEL and not os_version.is_slcc + + repository_paths: tuple[tuple[str, str], ...] + if os_version.is_sle15: + repository_paths = ( + ("SUSE:Registry", "standard"), + (f"SUSE:SLE-15-SP{str(os_version)}:Update", "standard"), + ) + elif os_version.is_slcc: + repository_paths = (("SUSE:ALP:Source:Standard:Core:1.0:Build", "standard"),) + if project_type in (ProjectType.CR, ProjectType.STAGING): + repository_paths += ( + (generate_project_name(os_version, ProjectType.DEVEL, ""), "standard"), + ) + else: + repository_paths = ( + ("openSUSE:Factory", "images"), + ("openSUSE:Factory:ARM", "images"), + ("openSUSE:Factory:ARM", "standard"), + ("openSUSE:Factory:PowerPC", "standard"), + ("openSUSE:Factory:zSystems", "standard"), + ("openSUSE:Factory", "snapshot"), + ) + + if project_type == ProjectType.STAGING: + assert branch_name + description = f"Staging project for https://github.com/SUSE/BCI-dockerfile-generator/tree/{branch_name}" + title = "Staging project" + else: + description = { + ProjectType.DEVEL: "Development", + ProjectType.CR: "Continuous Rebuild", + }[project_type] + " project" + title = description + + description += f" for {os_version.full_os_name}" + title += f" for {os_version.full_os_name}" + + if not os_version.is_slcc and project_type == ProjectType.DEVEL: + description = "BCI " + description + title = "BCI " + title + + extra_header = None + if project_type == ProjectType.CR: + extra_header = f" https://github.com/SUSE/bci-dockerfile-generator#{os_version.deployment_branch_name}" + + return prj_name, META_TEMPLATE.render( + project_title=title, + project_description=description, + project_name=prj_name, + maintainers=users, + with_all_arches=with_all_arches, + with_helmcharts_repo=with_helmcharts_repo, + with_product_repo=os_version.is_slcc, + repository_paths=repository_paths, + description=description, + extra_header=extra_header, + ) diff --git a/tests/test_project_setup.py b/tests/test_project_setup.py new file mode 100644 index 000000000..6018d65b0 --- /dev/null +++ b/tests/test_project_setup.py @@ -0,0 +1,184 @@ +import pytest + +from bci_build.package import OsVersion +from staging.project_setup import ProjectType +from staging.project_setup import generate_meta + +_OSC_USERNAME = "foobar" + + +@pytest.mark.parametrize( + "os_version, project_type, branch_name, expected_prj_name, expected_meta", + [ + ( + OsVersion.SP6, + ProjectType.DEVEL, + None, + "devel:BCI:SLE-15-SP6", + """ + BCI Development project for SLE 15 SP6 + BCI Development project for SLE 15 SP6 + + + + + + + + + + + + + + + + + + + + x86_64 + aarch64 + s390x + ppc64le + + + + + x86_64 + aarch64 + s390x + ppc64le + + + + x86_64 + + + + + x86_64 + aarch64 + s390x + ppc64le + +""", + ), + ( + OsVersion.SP5, + ProjectType.CR, + None, + (prj_name := f"home:{_OSC_USERNAME}:BCI:CR:SLE-15-SP5"), + f""" + Continuous Rebuild project for SLE 15 SP5 + Continuous Rebuild project for SLE 15 SP5 + + + + + https://github.com/SUSE/bci-dockerfile-generator#sle15-sp5 + + + + + + + + + + + + + x86_64 + aarch64 + s390x + ppc64le + + + + + x86_64 + aarch64 + s390x + ppc64le + + + + + x86_64 + aarch64 + s390x + ppc64le + +""", + ), + ( + OsVersion.SLCC_FREE, + ProjectType.STAGING, + (branch := "pr-404"), + (prj_name := f"home:{_OSC_USERNAME}:BCI:Staging:SLCC:free:{branch}"), + f""" + Staging project for SUSE Linux Container Collection - free Stream + Staging project for https://github.com/SUSE/BCI-dockerfile-generator/tree/{branch} for SUSE Linux Container Collection - free Stream + + + + + + + + + + + + + + + + + x86_64 + aarch64 + s390x + ppc64le + + + + + x86_64 + aarch64 + s390x + ppc64le + + + + + + x86_64 + aarch64 + s390x + ppc64le + + + + + x86_64 + aarch64 + s390x + ppc64le + +""", + ), + ], +) +def test_project_meta( + os_version: OsVersion, + project_type: ProjectType, + branch_name: str | None, + expected_prj_name: str, + expected_meta: str, +) -> None: + prj_name, prj_meta = generate_meta( + os_version, project_type, _OSC_USERNAME, branch_name + ) + assert prj_name == expected_prj_name + assert prj_meta == expected_meta