From c0cd9c2f509f4538a357b1f3b6dce38945753610 Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Mon, 9 Oct 2023 17:05:33 +0800 Subject: [PATCH 1/5] feat: add support to run build for recipes with additional-platforms --- bioconda_utils/build.py | 15 +++++++++++++++ bioconda_utils/recipe.py | 7 +++++++ 2 files changed, 22 insertions(+) diff --git a/bioconda_utils/build.py b/bioconda_utils/build.py index 3101394350..a501ae85be 100644 --- a/bioconda_utils/build.py +++ b/bioconda_utils/build.py @@ -27,6 +27,7 @@ from . import upload from . import lint from . import graph +from . import recipe as _recipe logger = logging.getLogger(__name__) @@ -355,7 +356,21 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str], skipped_recipes = [] failed_uploads = [] + def skip_additional_platform(): + recipe_obj = _recipe.Recipe.from_file(recipe_folder, recipe) + native_platform = utils.RepoData().native_platform() + # On linux-aarch64 env, only build recipe with linux-aarch64 additional_platforms + if native_platform == "linux-aarch64": + if "linux-aarch64" not in recipe_obj.extra_additional_platforms: + logger.info("BUILD SKIP: skipping %s for %s platform", recipe, native_platform) + return True + return False + for recipe, name in recipes: + # If not force, skip recipes that are not for this platform + if not force and skip_additional_platform(): + continue + if name in skip_dependent: logger.info('BUILD SKIP: skipping %s because it depends on %s ' 'which had a failed build.', diff --git a/bioconda_utils/recipe.py b/bioconda_utils/recipe.py index ca955e6b1f..2eae36f2c7 100644 --- a/bioconda_utils/recipe.py +++ b/bioconda_utils/recipe.py @@ -406,6 +406,13 @@ def maintainers(self): return utils.ensure_list(self.meta['extra']['recipe-maintainers']) return [] + @property + def extra_additional_platforms(self) -> list: + """The extra additional-platforms list""" + if 'extra' in self.meta and 'additional-platforms' in self.meta['extra']: + return list(self.meta["extra"]["additional-platforms"]) + return [] + @property def name(self) -> str: """The name of the toplevel package built by this recipe""" From 3c68c8db3971f69a5c25725b9090146e3a21d258 Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Fri, 13 Oct 2023 12:37:11 +0800 Subject: [PATCH 2/5] Add unit test Signed-off-by: Yikun Jiang --- bioconda_utils/build.py | 33 ++++++++++++++++++++++----------- test/test_recipe.py | 37 +++++++++++++++++++++++++++++++++++++ test/test_utils.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 11 deletions(-) diff --git a/bioconda_utils/build.py b/bioconda_utils/build.py index a501ae85be..b2db8df8ad 100644 --- a/bioconda_utils/build.py +++ b/bioconda_utils/build.py @@ -260,6 +260,27 @@ def get_subdags(dag, n_workers, worker_offset): return subdags +def check_native_platform_skippable(recipe_folder: str, recipe: str): + """ + Given a recipe, check this recipe should skip in current platform or not. + + Arguments: + recipe_folder: Directory containing possibly many, and possibly nested, recipes. + recipe: recipe name + + Returns: + Return True if current native platform are not included in recipe's additional platforms (no need to build). + """ + recipe_obj = _recipe.Recipe.from_file(recipe_folder, recipe) + native_platform = utils.RepoData().native_platform() + # On linux-aarch64 env, only build recipe with linux-aarch64 extra_additional_platforms + if native_platform == "linux-aarch64": + if "linux-aarch64" not in recipe_obj.extra_additional_platforms: + logger.info("BUILD SKIP: skipping %s for %s platform", recipe, native_platform) + return True + return False + + def build_recipes(recipe_folder: str, config_path: str, recipes: List[str], mulled_test: bool = True, testonly: bool = False, force: bool = False, @@ -356,19 +377,9 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str], skipped_recipes = [] failed_uploads = [] - def skip_additional_platform(): - recipe_obj = _recipe.Recipe.from_file(recipe_folder, recipe) - native_platform = utils.RepoData().native_platform() - # On linux-aarch64 env, only build recipe with linux-aarch64 additional_platforms - if native_platform == "linux-aarch64": - if "linux-aarch64" not in recipe_obj.extra_additional_platforms: - logger.info("BUILD SKIP: skipping %s for %s platform", recipe, native_platform) - return True - return False - for recipe, name in recipes: # If not force, skip recipes that are not for this platform - if not force and skip_additional_platform(): + if not force and check_native_platform_skippable(recipe_folder, recipe): continue if name in skip_dependent: diff --git a/test/test_recipe.py b/test/test_recipe.py index e7e296379e..08c0f944ea 100644 --- a/test/test_recipe.py +++ b/test/test_recipe.py @@ -56,6 +56,33 @@ license: BSD home: https://elsewhere summary: the_summary +three: + folder: three + meta.yaml: + #{% set version="0.1" %} + package: + name: three + version: #{{version}} + source: + - url: https://somewhere # [osx] + sha256: 123 # [osx[ + - url: https://somewhere # [linux] + sha256: 456 # [linux] + build: + number: 0 + outputs: + - name: libthree + - name: three-tools + test: + commands: + - do nothing + about: + license: BSD + home: https://elsewhere + summary: the_summary + extra: + additional-platforms: + - linux-aarch64 """ RECIPES = yaml.load(RECIPE_DATA) @@ -258,6 +285,16 @@ def test_recipe_package_names(recipe): assert recipe.package_names == expected +@with_recipes +def test_recipe_extra_additional_platforms(recipe): + expected = { + 'one': [], + 'two': [], + 'three': ["linux-aarch64"] + }[recipe.name] + assert recipe.extra_additional_platforms == expected + + @with_recipes def test_get_deps_dict(recipe): recipe.meta_yaml.extend([ diff --git a/test/test_utils.py b/test/test_utils.py index 09c4abb9eb..e0f21f2cfb 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -6,6 +6,7 @@ import tempfile import requests import uuid +import unittest.mock import contextlib import tarfile import logging @@ -889,6 +890,42 @@ def test_load_meta_skipping(): assert utils.load_all_meta(recipe) == [] +@unittest.mock.patch('bioconda_utils.utils.RepoData.native_platform') +def test_native_platform_skipping(func_mock): + expections = [ + # Don't skip linux-x86 for any recipes + ["one", "linux", False], + ["two", "linux", False], + # Skip recipe without linux aarch64 enable on linux-aarch64 platform + ["one", "linux-aarch64", True], + # Don't skip recipe with linux aarch64 enable on linux-aarch64 platform + ["two", "linux-aarch64", False], + ] + r = Recipes( + """ + one: + meta.yaml: | + package: + name: one + version: "0.1" + two: + meta.yaml: | + package: + name: one + version: "0.1" + extra: + additional-platforms: + - linux-aarch64 + """, from_string=True) + r.write_recipes() + # Make sure RepoData singleton init + utils.RepoData.register_config(config_fixture) + for recipe_name, native_platform, result in expections: + recipe_folder = os.path.dirname(r.recipe_dirs[recipe_name]) + func_mock.return_value = native_platform + assert build.check_native_platform_skippable(recipe_folder, r.recipe_dirs[recipe_name]) == result + + def test_variants(): """ Multiple variants should return multiple metadata From 335b3478323f0c6a68de8ae9c34e11f1c1e96fff Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Fri, 13 Oct 2023 16:01:38 +0800 Subject: [PATCH 3/5] Fix ambiguous argument 'origin/master...HEAD' Signed-off-by: Yikun Jiang --- .github/workflows/GithubActionTests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/GithubActionTests.yml b/.github/workflows/GithubActionTests.yml index 091b4d5e64..bb319d1a28 100644 --- a/.github/workflows/GithubActionTests.yml +++ b/.github/workflows/GithubActionTests.yml @@ -17,6 +17,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: set path run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH @@ -52,6 +54,8 @@ jobs: runs-on: macos-latest steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: set path run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH From 961116afb9be943e58dccdce08a41eb756604466 Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Fri, 13 Oct 2023 17:06:49 +0800 Subject: [PATCH 4/5] Fix test --- test/test_recipe.py | 41 ++++++++--------------------------------- 1 file changed, 8 insertions(+), 33 deletions(-) diff --git a/test/test_recipe.py b/test/test_recipe.py index 08c0f944ea..5e1b84559c 100644 --- a/test/test_recipe.py +++ b/test/test_recipe.py @@ -56,33 +56,6 @@ license: BSD home: https://elsewhere summary: the_summary -three: - folder: three - meta.yaml: - #{% set version="0.1" %} - package: - name: three - version: #{{version}} - source: - - url: https://somewhere # [osx] - sha256: 123 # [osx[ - - url: https://somewhere # [linux] - sha256: 456 # [linux] - build: - number: 0 - outputs: - - name: libthree - - name: three-tools - test: - commands: - - do nothing - about: - license: BSD - home: https://elsewhere - summary: the_summary - extra: - additional-platforms: - - linux-aarch64 """ RECIPES = yaml.load(RECIPE_DATA) @@ -287,12 +260,14 @@ def test_recipe_package_names(recipe): @with_recipes def test_recipe_extra_additional_platforms(recipe): - expected = { - 'one': [], - 'two': [], - 'three': ["linux-aarch64"] - }[recipe.name] - assert recipe.extra_additional_platforms == expected + assert recipe.extra_additional_platforms == [] + recipe.meta_yaml += [ + 'extra:', + ' additional-platforms:', + ' - linux-aarch64' + ] + recipe.render() + assert recipe.extra_additional_platforms == ["linux-aarch64"] @with_recipes From 046dbbfe637ebb9dc23848bacba86a0b559d17ab Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Fri, 13 Oct 2023 20:13:50 +0800 Subject: [PATCH 5/5] Address comments Signed-off-by: Yikun Jiang --- bioconda_utils/build.py | 14 +++++++------- test/test_utils.py | 11 +++++------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/bioconda_utils/build.py b/bioconda_utils/build.py index b2db8df8ad..2dd6b2efd2 100644 --- a/bioconda_utils/build.py +++ b/bioconda_utils/build.py @@ -260,23 +260,22 @@ def get_subdags(dag, n_workers, worker_offset): return subdags -def check_native_platform_skippable(recipe_folder: str, recipe: str): +def do_not_consider_for_additional_platform(recipe_folder: str, recipe: str, platform: str): """ Given a recipe, check this recipe should skip in current platform or not. Arguments: recipe_folder: Directory containing possibly many, and possibly nested, recipes. - recipe: recipe name + recipe: Relative path to recipe + platform: current native platform Returns: Return True if current native platform are not included in recipe's additional platforms (no need to build). """ recipe_obj = _recipe.Recipe.from_file(recipe_folder, recipe) - native_platform = utils.RepoData().native_platform() # On linux-aarch64 env, only build recipe with linux-aarch64 extra_additional_platforms - if native_platform == "linux-aarch64": + if platform == "linux-aarch64": if "linux-aarch64" not in recipe_obj.extra_additional_platforms: - logger.info("BUILD SKIP: skipping %s for %s platform", recipe, native_platform) return True return False @@ -378,8 +377,9 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str], failed_uploads = [] for recipe, name in recipes: - # If not force, skip recipes that are not for this platform - if not force and check_native_platform_skippable(recipe_folder, recipe): + platform = utils.RepoData().native_platform() + if not force and do_not_consider_for_additional_platform(recipe_folder, recipe, platform): + logger.info("BUILD SKIP: skipping %s for additional platform %s", recipe, platform) continue if name in skip_dependent: diff --git a/test/test_utils.py b/test/test_utils.py index e0f21f2cfb..916fb10894 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -6,7 +6,6 @@ import tempfile import requests import uuid -import unittest.mock import contextlib import tarfile import logging @@ -890,8 +889,7 @@ def test_load_meta_skipping(): assert utils.load_all_meta(recipe) == [] -@unittest.mock.patch('bioconda_utils.utils.RepoData.native_platform') -def test_native_platform_skipping(func_mock): +def test_native_platform_skipping(): expections = [ # Don't skip linux-x86 for any recipes ["one", "linux", False], @@ -920,10 +918,11 @@ def test_native_platform_skipping(func_mock): r.write_recipes() # Make sure RepoData singleton init utils.RepoData.register_config(config_fixture) - for recipe_name, native_platform, result in expections: + for recipe_name, platform, result in expections: recipe_folder = os.path.dirname(r.recipe_dirs[recipe_name]) - func_mock.return_value = native_platform - assert build.check_native_platform_skippable(recipe_folder, r.recipe_dirs[recipe_name]) == result + assert build.do_not_consider_for_additional_platform(recipe_folder, + r.recipe_dirs[recipe_name], + platform) == result def test_variants():