Merge 24.9.x back into main

.authors.yml

@@ -8,7 +8,7 @@
   aliases:
   - Mike Sarahan
   - Michael Sarahan
-  num_commits: 2001
+  num_commits: 2002
   first_commit: 2015-09-04 21:31:08
 - name: Jonathan J. Helmus
   email: jjhelmus@gmail.com
@@ -162,7 +162,7 @@
   aliases:
   - MinRK
   github: minrk
-  num_commits: 17
+  num_commits: 18
   first_commit: 2014-02-13 19:43:59
 - name: Matty G
   email: meawoppl@gmail.com
@@ -1121,7 +1121,7 @@
   alternate_emails:
   - becker.mr@gmail.com
   - beckermr@users.noreply.github.com
-  num_commits: 21
+  num_commits: 22
   first_commit: 2019-10-17 23:05:16
   github: beckermr
 - name: Jinzhe Zeng
@@ -1171,7 +1171,7 @@
   github: pradghos
 - name: James Lamb
   email: jaylamb20@gmail.com
-  num_commits: 1
+  num_commits: 2
   first_commit: 2020-04-26 13:41:22
   github: jameslamb
 - name: Oleg Alexandrov
@@ -1202,7 +1202,7 @@
   alternate_emails:
   - clee@anaconda.com
 - name: Ken Odegard
-  num_commits: 213
+  num_commits: 220
   email: kodegard@anaconda.com
   first_commit: 2020-09-08 19:53:41
   github: kenodegard
@@ -1240,7 +1240,7 @@
   github: pre-commit-ci[bot]
   aliases:
   - pre-commit-ci[bot]
-  num_commits: 79
+  num_commits: 86
   first_commit: 2021-11-20 01:47:17
 - name: Jacob Walls
   email: jacobtylerwalls@gmail.com
@@ -1251,7 +1251,7 @@
   github: beeankha
   alternate_emails:
   - beeankha@gmail.com
-  num_commits: 34
+  num_commits: 43
   first_commit: 2022-01-19 16:40:06
 - name: Conda Bot
   email: 18747875+conda-bot@users.noreply.github.com
@@ -1262,7 +1262,7 @@
   alternate_emails:
   - ad-team+condabot@anaconda.com
   - 18747875+conda-bot@users.noreply.github.com
-  num_commits: 58
+  num_commits: 62
   first_commit: 2022-01-17 18:09:22
 - name: Uwe L. Korn
   email: xhochy@users.noreply.github.com
@@ -1310,7 +1310,7 @@
 - name: dependabot[bot]
   email: 49699333+dependabot[bot]@users.noreply.github.com
   github: dependabot[bot]
-  num_commits: 34
+  num_commits: 35
   first_commit: 2022-05-31 04:34:40
 - name: Serhii Kupriienko
   email: 79282962+skupr-anaconda@users.noreply.github.com

CHANGELOG.md

@@ -1,5 +1,39 @@
 [//]: # (current developments)
 
+## 24.9.0 (2024-09-18)
+
+### Enhancements
+
+* Reduce render time when there is a large number of unused variants. (#5392)
+
+### Bug fixes
+
+* Ensure variables mentioned in `script_env` are undefined in the multi-output build environment
+  if they are undefined in the environment that `conda-build` is invoked from. (#5322)
+* Variables used in single-line jinja2 `for` and `set` statements are now properly included in the variant
+  matrix for some edge cases. (#5447)
+* Allow undefined jinja variables when a particular metadata block evaluates as skipped. (#5458)
+
+### Deprecations
+
+* Remove `conda_build.build.check_external`. `patchelf` is an explicit conda-build dependency on Linux, so it will always be installed. (#5441)
+* Remove `conda_build.metadata._get_env_path`. Use `conda.base.context.locate_prefix_by_name` instead. (#5441)
+* Remove `conda_build.build._construct_metadata_for_test_from_recipe`. Test built packages instead, not recipes (e.g., `conda build --test package` instead of `conda build --test recipe/`). (#5478)
+
+### Contributors
+
+* @beeankha
+* @conda-bot
+* @jameslamb
+* @kenodegard
+* @beckermr
+* @msarahan
+* @minrk
+* @dependabot[bot]
+* @pre-commit-ci[bot]
+
+
+
 ## 24.7.1 (2024-07-30)
 
 ### Bug fixes

conda_build/config.py

@@ -31,7 +31,9 @@
 
 if TYPE_CHECKING:
     from pathlib import Path
-    from typing import Any
+    from typing import Any, TypeVar
+
+    T = TypeVar("T")
 
 invocation_time = ""
 
@@ -821,14 +823,23 @@ def clean_pkgs(self):
 
     def copy(self) -> Config:
         new = copy.copy(self)
-        # Use picke.loads(pickle.dumps(...) as a faster copy.deepcopy alternative.
-        new.variant = pickle.loads(pickle.dumps(self.variant, pickle.HIGHEST_PROTOCOL))
+        new.variant = self._copy_variants(self.variant)
         if hasattr(self, "variants"):
-            new.variants = pickle.loads(
-                pickle.dumps(self.variants, pickle.HIGHEST_PROTOCOL)
-            )
+            new.variants = self.copy_variants()
         return new
 
+    def _copy_variants(self, variant_or_list: T) -> T:
+        """Efficient deep copy used for variant dicts and lists"""
+        # Use pickle.loads(pickle.dumps(...) as a faster copy.deepcopy alternative.
+        return pickle.loads(pickle.dumps(variant_or_list, pickle.HIGHEST_PROTOCOL))
+
+    def copy_variants(self) -> list[dict] | None:
+        """Return deep copy of the variants list, if any"""
+        if getattr(self, "variants", None) is not None:
+            return self._copy_variants(self.variants)
+        else:
+            return None
+
     # context management - automatic cleanup if self.dirty or self.keep_old_work is not True
     def __enter__(self):
         pass

conda_build/metadata.py

@@ -2288,7 +2288,6 @@ def validate_features(self):
     def copy(self: Self) -> MetaData:
         new = copy.copy(self)
         new.config = self.config.copy()
-        new.config.variant = copy.deepcopy(self.config.variant)
         new.meta = copy.deepcopy(self.meta)
         new.type = getattr(
             self, "type", "conda_v2" if self.config.conda_pkg_format == "2" else "conda"
@@ -2672,15 +2671,16 @@ def get_output_metadata_set(
         _check_run_constrained(output_tuples)
         return output_tuples
 
-    def get_loop_vars(self):
-        return get_vars(getattr(self.config, "input_variants", self.config.variants))
+    def get_loop_vars(self, subset=None):
+        return get_vars(
+            getattr(self.config, "input_variants", self.config.variants), subset=subset
+        )
 
     def get_used_loop_vars(self, force_top_level=False, force_global=False):
-        loop_vars = self.get_loop_vars()
         used_vars = self.get_used_vars(
             force_top_level=force_top_level, force_global=force_global
         )
-        return set(loop_vars).intersection(used_vars)
+        return self.get_loop_vars(subset=used_vars)
 
     def get_rendered_recipe_text(
         self, permit_undefined_jinja=False, extract_pattern=None

conda_build/render.py

@@ -835,12 +835,20 @@ def distribute_variants(
     used_variables = metadata.get_used_loop_vars(force_global=False)
     top_loop = metadata.get_reduced_variant_set(used_variables)
 
+    # defer potentially expensive copy of input variants list
+    # until after reduction of the list for each variant
+    # since the initial list can be very long
+    all_variants = metadata.config.variants
+    metadata.config.variants = []
+
     for variant in top_loop:
         from .build import get_all_replacements
 
         get_all_replacements(variant)
         mv = metadata.copy()
         mv.config.variant = variant
+        # start with shared list:
+        mv.config.variants = all_variants
 
         pin_run_as_build = variant.get("pin_run_as_build", {})
         if mv.numpy_xx and "numpy" not in pin_run_as_build:
@@ -860,6 +868,9 @@ def distribute_variants(
                 )
                 or mv.config.variants
             )
+        # copy variants before we start modifying them,
+        # but after we've reduced the list via the conform_dict filter
+        mv.config.variants = mv.config.copy_variants()
         get_all_replacements(mv.config.variants)
         pin_run_as_build = variant.get("pin_run_as_build", {})
         if mv.numpy_xx and "numpy" not in pin_run_as_build:

conda_build/variants.py

@@ -700,7 +700,10 @@ def get_package_variants(recipedir_or_metadata, config=None, variants=None):
     return filter_combined_spec_to_used_keys(combined_spec, specs=specs)
 
 
-def get_vars(variants: Iterable[dict[str, Any]]) -> set[str]:
+def get_vars(
+    variants: Iterable[dict[str, Any]],
+    subset: set[str] | None = None,
+) -> set[str]:
     """For purposes of naming/identifying, provide a way of identifying which variables contribute
     to the matrix dimensionality"""
     first, *others = variants
@@ -710,10 +713,12 @@ def get_vars(variants: Iterable[dict[str, Any]]) -> set[str]:
         "ignore_version",
         *ensure_list(first.get("extend_keys")),
     }
+    to_consider = set(first)
+    if subset is not None:
+        to_consider.intersection_update(subset)
+    to_consider.difference_update(special_keys)
     return {
-        var
-        for var in set(first) - special_keys
-        if any(first[var] != other[var] for other in others)
+        var for var in to_consider if any(first[var] != other[var] for other in others)
     }
 
 

tests/requirements-ci.txt

@@ -15,4 +15,5 @@ pytest-mock
 pytest-rerunfailures
 pytest-xdist
 ruamel.yaml
+setuptools <75.1.0  # temporary, see https://github.com/conda/conda-build/issues/5493
 tomli  # [py<3.11] for coverage pyproject.toml