From 1ba5760c46bdfbf6f14063295f40b888f0952c3a Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 18 Sep 2024 06:32:39 -0700 Subject: [PATCH 1/3] reduce cost of large variant matrix (#5392) * discard unused variants before copying metadata when variant matrix is large and mostly unused (as in conda-forge), the length of input_variants may be several thousand when only a few are actually used. This causes `get_loop_vars` and `metadata.copy()` to become very expensive. * try reducing with all used vars instead of loop vars should reduce less * perf: copy distributed variants list after subsetting vastly reduces the number of copies computed for large variant matrices * perf: pass used_vars subset to get_loop_vars rather than computing all loop vars and then intersecting, only consider relevant keys when computing loop vars reduces get_used_loop_vars from O(n_vars * n_variants) to O(n_used_vars * n_variants) * remove redundant deepcopy of config.variant config.copy already copies this, no need to do it twice in metadata.copy * add config.copy_variants method to avoid calling pickle in too many places * Update news/5392-variant-copy * Add benchmark test for `render_recipe` (#5490) --------- Co-authored-by: Matthew R. Becker Co-authored-by: Bianca Henderson Co-authored-by: Ken Odegard --- conda_build/config.py | 23 +- conda_build/metadata.py | 10 +- conda_build/render.py | 11 + conda_build/variants.py | 13 +- news/5392-variant-copy | 3 + .../_render_recipe/conda_build_config.yaml | 941 ++++++++++++++++++ .../metadata/_render_recipe/meta.yaml | 72 ++ tests/test_render.py | 16 +- 8 files changed, 1073 insertions(+), 16 deletions(-) create mode 100644 news/5392-variant-copy create mode 100644 tests/test-recipes/metadata/_render_recipe/conda_build_config.yaml create mode 100644 tests/test-recipes/metadata/_render_recipe/meta.yaml diff --git a/conda_build/config.py b/conda_build/config.py index 465058701f..719451a2d9 100644 --- a/conda_build/config.py +++ b/conda_build/config.py @@ -31,7 +31,9 @@ if TYPE_CHECKING: from pathlib import Path - from typing import Any + from typing import Any, TypeVar + + T = TypeVar("T") invocation_time = "" @@ -821,14 +823,23 @@ def clean_pkgs(self): def copy(self) -> Config: new = copy.copy(self) - # Use picke.loads(pickle.dumps(...) as a faster copy.deepcopy alternative. - new.variant = pickle.loads(pickle.dumps(self.variant, pickle.HIGHEST_PROTOCOL)) + new.variant = self._copy_variants(self.variant) if hasattr(self, "variants"): - new.variants = pickle.loads( - pickle.dumps(self.variants, pickle.HIGHEST_PROTOCOL) - ) + new.variants = self.copy_variants() return new + def _copy_variants(self, variant_or_list: T) -> T: + """Efficient deep copy used for variant dicts and lists""" + # Use pickle.loads(pickle.dumps(...) as a faster copy.deepcopy alternative. + return pickle.loads(pickle.dumps(variant_or_list, pickle.HIGHEST_PROTOCOL)) + + def copy_variants(self) -> list[dict] | None: + """Return deep copy of the variants list, if any""" + if getattr(self, "variants", None) is not None: + return self._copy_variants(self.variants) + else: + return None + # context management - automatic cleanup if self.dirty or self.keep_old_work is not True def __enter__(self): pass diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 071036bd0c..b744a8b3c9 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -2288,7 +2288,6 @@ def validate_features(self): def copy(self: Self) -> MetaData: new = copy.copy(self) new.config = self.config.copy() - new.config.variant = copy.deepcopy(self.config.variant) new.meta = copy.deepcopy(self.meta) new.type = getattr( self, "type", "conda_v2" if self.config.conda_pkg_format == "2" else "conda" @@ -2672,15 +2671,16 @@ def get_output_metadata_set( _check_run_constrained(output_tuples) return output_tuples - def get_loop_vars(self): - return get_vars(getattr(self.config, "input_variants", self.config.variants)) + def get_loop_vars(self, subset=None): + return get_vars( + getattr(self.config, "input_variants", self.config.variants), subset=subset + ) def get_used_loop_vars(self, force_top_level=False, force_global=False): - loop_vars = self.get_loop_vars() used_vars = self.get_used_vars( force_top_level=force_top_level, force_global=force_global ) - return set(loop_vars).intersection(used_vars) + return self.get_loop_vars(subset=used_vars) def get_rendered_recipe_text( self, permit_undefined_jinja=False, extract_pattern=None diff --git a/conda_build/render.py b/conda_build/render.py index 0c80df0005..353706f99b 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -835,12 +835,20 @@ def distribute_variants( used_variables = metadata.get_used_loop_vars(force_global=False) top_loop = metadata.get_reduced_variant_set(used_variables) + # defer potentially expensive copy of input variants list + # until after reduction of the list for each variant + # since the initial list can be very long + all_variants = metadata.config.variants + metadata.config.variants = [] + for variant in top_loop: from .build import get_all_replacements get_all_replacements(variant) mv = metadata.copy() mv.config.variant = variant + # start with shared list: + mv.config.variants = all_variants pin_run_as_build = variant.get("pin_run_as_build", {}) if mv.numpy_xx and "numpy" not in pin_run_as_build: @@ -860,6 +868,9 @@ def distribute_variants( ) or mv.config.variants ) + # copy variants before we start modifying them, + # but after we've reduced the list via the conform_dict filter + mv.config.variants = mv.config.copy_variants() get_all_replacements(mv.config.variants) pin_run_as_build = variant.get("pin_run_as_build", {}) if mv.numpy_xx and "numpy" not in pin_run_as_build: diff --git a/conda_build/variants.py b/conda_build/variants.py index 3eef82266d..8a23f27405 100644 --- a/conda_build/variants.py +++ b/conda_build/variants.py @@ -700,7 +700,10 @@ def get_package_variants(recipedir_or_metadata, config=None, variants=None): return filter_combined_spec_to_used_keys(combined_spec, specs=specs) -def get_vars(variants: Iterable[dict[str, Any]]) -> set[str]: +def get_vars( + variants: Iterable[dict[str, Any]], + subset: set[str] | None = None, +) -> set[str]: """For purposes of naming/identifying, provide a way of identifying which variables contribute to the matrix dimensionality""" first, *others = variants @@ -710,10 +713,12 @@ def get_vars(variants: Iterable[dict[str, Any]]) -> set[str]: "ignore_version", *ensure_list(first.get("extend_keys")), } + to_consider = set(first) + if subset is not None: + to_consider.intersection_update(subset) + to_consider.difference_update(special_keys) return { - var - for var in set(first) - special_keys - if any(first[var] != other[var] for other in others) + var for var in to_consider if any(first[var] != other[var] for other in others) } diff --git a/news/5392-variant-copy b/news/5392-variant-copy new file mode 100644 index 0000000000..83c8f30eed --- /dev/null +++ b/news/5392-variant-copy @@ -0,0 +1,3 @@ +### Enhancements + +* Reduce render time when there is a large number of unused variants. (#5392) diff --git a/tests/test-recipes/metadata/_render_recipe/conda_build_config.yaml b/tests/test-recipes/metadata/_render_recipe/conda_build_config.yaml new file mode 100644 index 0000000000..b781a551a2 --- /dev/null +++ b/tests/test-recipes/metadata/_render_recipe/conda_build_config.yaml @@ -0,0 +1,941 @@ +# from https://github.com/conda-forge/petsc4py-feedstock/blob/ccb2a3aed8e07d554b5f8e1dee47508a8d81cac4/recipe/conda_build_config.yaml +mpi: + - mpich + - openmpi + +scalar: + - real + - complex + +device: + - host + - cuda11 # [linux] + - cuda12 # [linux] + +# from https://github.com/conda-forge/conda-forge-pinning-feedstock/blob/9eed6c22e88a0c853e0781ebb1f14589b65c6c36/recipe/conda_build_config.yaml +c_compiler: + - gcc # [linux] + - clang # [osx] + - vs2019 # [win and x86_64] + - vs2022 # [win and arm64] +c_compiler_version: # [unix] + - 13 # [linux] + - 17 # [osx] + - 11 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux] + - 12 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux] +c_stdlib: + - sysroot # [linux] + - macosx_deployment_target # [osx] + - vs # [win] +m2w64_c_stdlib: # [win] + - m2w64-toolchain # [win] +c_stdlib_version: # [unix] + - 2.17 # [linux] + - 2.17 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 2.17 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 10.13 # [osx and x86_64] + - 11.0 # [osx and arm64] +cxx_compiler: + - gxx # [linux] + - clangxx # [osx] + - vs2019 # [win and x86_64] + - vs2022 # [win and arm64] +cxx_compiler_version: # [unix] + - 13 # [linux] + - 17 # [osx] + - 11 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux] + - 12 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux] +llvm_openmp: # [osx] + - 17 # [osx] +fortran_compiler: # [unix or win64] + - gfortran # [linux64 or (osx and x86_64)] + - gfortran # [aarch64 or ppc64le or armv7l or s390x] + - flang # [win64] +fortran_compiler_version: # [unix or win64] + - 13 # [linux] + - 13 # [osx] + - 5 # [win64] + - 11 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux] + - 12 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux] +m2w64_c_compiler: # [win] + - m2w64-toolchain # [win] +m2w64_cxx_compiler: # [win] + - m2w64-toolchain # [win] +m2w64_fortran_compiler: # [win] + - m2w64-toolchain # [win] + +cuda_compiler: + - None + - nvcc # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cuda-nvcc # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] +cuda_compiler_version: + - None + - 11.8 # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 12.0 # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] +cuda_compiler_version_min: + - None # [osx] + - 11.8 # [linux or win64] + +arm_variant_type: # [aarch64] + - sbsa # [aarch64] + +_libgcc_mutex: + - 0.1 conda_forge +# +# Go Compiler Options +# + +# The basic go-compiler with CGO disabled, +# It generates fat binaries without libc dependencies +# The activation scripts will set your CC,CXX and related flags +# to invalid values. +go_compiler: + - go-nocgo +# The go compiler build with CGO enabled. +# It can generate fat binaries that depend on conda's libc. +# You should use this compiler if the underlying +# program needs to link against other C libraries, in which +# case make sure to add 'c,cpp,fortran_compiler' for unix +# and the m2w64 equivalent for windows. +cgo_compiler: + - go-cgo +# The following are helpful variables to simplify go meta.yaml files. +target_goos: + - linux # [linux] + - darwin # [osx] + - windows # [win] +target_goarch: + - amd64 # [x86_64] + - arm64 # [arm64 or aarch64] + - ppc64le # [ppc64le] +target_goexe: + - # [unix] + - .exe # [win] +target_gobin: + - '${PREFIX}/bin/' # [unix] + - '%PREFIX%\bin\' # [win] + +# Rust Compiler Options +rust_compiler: + - rust + +# the numbers here are the Darwin Kernel version for macOS 10.9 & 11.0; +# this is used to form our target triple on osx, and nothing else. After +# we bumped the minimum macOS version to 10.13, this was left unchanged, +# since it is not essential, and long-term we'd like to remove the version. +macos_machine: # [osx] + - x86_64-apple-darwin13.4.0 # [osx and x86_64] + - arm64-apple-darwin20.0.0 # [osx and arm64] +MACOSX_DEPLOYMENT_TARGET: # [osx] + - 11.0 # [osx and arm64] + - 10.13 # [osx and x86_64] +VERBOSE_AT: + - V=1 +VERBOSE_CM: + - VERBOSE=1 + +channel_sources: + - conda-forge + +channel_targets: + - conda-forge main + +cdt_name: # [linux] + - cos7 # [linux] + + - cos7 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cos7 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +docker_image: # [os.environ.get("BUILD_PLATFORM", "").startswith("linux-")] + # Native builds + - quay.io/condaforge/linux-anvil-cos7-x86_64 # [os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-aarch64 # [os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] + - quay.io/condaforge/linux-anvil-ppc64le # [os.environ.get("BUILD_PLATFORM") == "linux-ppc64le"] + - quay.io/condaforge/linux-anvil-armv7l # [os.environ.get("BUILD_PLATFORM") == "linux-armv7l"] + + # CUDA 11.8 + - quay.io/condaforge/linux-anvil-cuda:11.8 # [linux64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-64"] + # CUDA 11.8 arch: native compilation (build == target) + - quay.io/condaforge/linux-anvil-ppc64le-cuda:11.8 # [ppc64le and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-ppc64le"] + - quay.io/condaforge/linux-anvil-aarch64-cuda:11.8 # [aarch64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] + # CUDA 11.8 arch: cross-compilation (build != target) + - quay.io/condaforge/linux-anvil-cuda:11.8 # [ppc64le and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:11.8 # [aarch64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-64"] + + # CUDA 12 + - quay.io/condaforge/linux-anvil-cos7-x86_64 # [linux64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-64"] + # case: native compilation (build == target) + - quay.io/condaforge/linux-anvil-ppc64le # [ppc64le and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-ppc64le"] + - quay.io/condaforge/linux-anvil-aarch64 # [aarch64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] + # case: cross-compilation (build != target) + - quay.io/condaforge/linux-anvil-cos7-x86_64 # [ppc64le and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cos7-x86_64 # [aarch64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-64"] + +zip_keys: + # For CUDA, c_stdlib_version/cdt_name is zipped below with the compilers. + - # [linux and os.environ.get("CF_CUDA_ENABLED", "False") != "True"] + - c_stdlib_version # [linux and os.environ.get("CF_CUDA_ENABLED", "False") != "True"] + - cdt_name # [linux and os.environ.get("CF_CUDA_ENABLED", "False") != "True"] + - # [unix] + - c_compiler_version # [unix] + - cxx_compiler_version # [unix] + - fortran_compiler_version # [unix] + - c_stdlib_version # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cdt_name # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cuda_compiler # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cuda_compiler_version # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - docker_image # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM", "").startswith("linux-")] + - # [win64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cuda_compiler # [win64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cuda_compiler_version # [win64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - + - python + - numpy + - python_impl + - + - libarrow + - libarrow_all + + +# aarch64 specifics because conda-build sets many things to centos 6 +# this can probably be removed when conda-build gets updated defaults +# for aarch64 +cdt_arch: aarch64 # [aarch64] +BUILD: aarch64-conda_cos7-linux-gnu # [aarch64] + +# armv7l specifics because conda-build sets many things to centos 6 +# this can probably be removed when conda-build gets updated defaults +# for aarch64 +cdt_arch: armv7l # [armv7l] +BUILD: armv7-conda_cos7-linux-gnueabihf # [armv7l] + +pin_run_as_build: + # TODO: add run_exports to the following feedstocks + libsvm: + max_pin: x + netcdf-cxx4: + max_pin: x.x + vlfeat: + max_pin: x.x.x + +# Pinning packages + +# blas +libblas: + - 3.9 *netlib +libcblas: + - 3.9 *netlib +liblapack: + - 3.9 *netlib +liblapacke: + - 3.9 *netlib +blas_impl: + - openblas + - mkl # [x86 or x86_64] + - blis # [x86 or x86_64] + +# this output was dropped as of libabseil 20230125 +abseil_cpp: + - '20220623.0' +alsa_lib: + - '1.2' +antic: + - 0.2 +aom: + - '3.9' +arb: + - '2.23' +arpack: + - '3.9' +assimp: + - 5.4.1 +attr: + - 2.5 +aws_c_auth: + - 0.7.31 +aws_c_cal: + - 0.7.4 +aws_c_common: + - 0.9.28 +aws_c_compression: + - 0.2.19 +# coupled to aws_c_common version bump, see +# https://github.com/conda-forge/aws-c-http-feedstock/pull/109 +aws_c_event_stream: + - 0.4.3 +aws_c_http: + - 0.8.10 +# the builds got coupled because 0.2.4 landed before the this migrator finished +aws_c_io: + - 0.14.18 +# the builds got coupled because 0.2.4 landed before the io migrator +aws_c_mqtt: + - 0.10.5 +aws_c_s3: + - 0.6.5 +aws_c_sdkutils: + - 0.1.19 +aws_checksums: + - 0.1.20 +aws_crt_cpp: + - 0.28.2 +aws_sdk_cpp: + - 1.11.379 +azure_core_cpp: + - 1.13.0 +azure_identity_cpp: + - 1.8.0 +azure_storage_blobs_cpp: + - 12.12.0 +azure_storage_common_cpp: + - 12.7.0 +azure_storage_files_datalake_cpp: + - 12.11.0 +azure_storage_files_shares_cpp: + - 12.10.0 +azure_storage_queues_cpp: + - 12.3.0 +bullet_cpp: + - 3.25 +bzip2: + - 1 +c_ares: + - 1 +c_blosc2: + - '2.15' +cairo: + - 1 +capnproto: + - 1.0.2 +ccr: + - 1.3 +cfitsio: + - 4.3.1 +coin_or_cbc: + - 2.10 +coincbc: + - 2.10 +coin_or_cgl: + - 0.60 +coin_or_clp: + - 1.17 +coin_or_osi: + - 0.108 +coin_or_utils: + - 2.11 +console_bridge: + - 1.0 +cudnn: + - 8 +cutensor: + - 2 +curl: + - 8 +dav1d: + - 1.2.1 +davix: + - '0.8' +dbus: + - 1 +dcap: + - 2.47 +eclib: + - '20231211' +elfutils: + - '0.191' +exiv2: + - '0.28' +expat: + - 2 +ffmpeg: + - '6' +fftw: + - 3 +flann: + - 1.9.2 +flatbuffers: + - 24.3.25 +fmt: + - '10' +fontconfig: + - 2 +freetype: + - 2 +gct: + - 6.2.1705709074 +gf2x: + - '1.3' +gdk_pixbuf: + - 2 +gnuradio_core: + - 3.10.11 +gnutls: + - '3.8' +gsl: + - 2.7 +gsoap: + - 2.8.123 +gstreamer: + - '1.22' +gst_plugins_base: + - '1.22' +gdal: + - '3.9' +libgdal: + - '3.9' +libgdal_core: + - '3.9' +geos: + - 3.12.2 +geotiff: + - 1.7.3 +gfal2: + - '2.23' +gflags: + - 2.2 +giflib: + - 5.2 +glew: + - 2.1 +glib: + - '2' +glog: + - '0.7' +glpk: + - '5.0' +gmp: + - 6 +# keep google_cloud_cpp in sync with libgoogle_cloud_* +google_cloud_cpp: + - '2.29' +google_cloud_cpp_common: + - 0.25.0 +googleapis_cpp: + - '0.10' +graphviz: + - '12' +# this has been renamed to libgrpc as of 1.49; dropped as of 1.52. +# IOW, this version is unavailable; makes the renaming more obvious +grpc_cpp: + - '1.52' +harfbuzz: + - '9' +hdf4: + - 4.2.15 +hdf5: + - 1.14.3 +icu: + - '73' +idyntree: + - '12' +imath: + - 3.1.11 +ipopt: + - 3.14.16 +isl: + - '0.26' +jasper: + - 4 +jpeg: + - 9 +lcms2: + - 2 +lerc: + - '4' +libjpeg_turbo: + - '3' +libev: + - 4.33 +json_c: + - '0.17' +jsoncpp: + - 1.9.5 +kealib: + - '1.5' +krb5: + - '1.21' +ldas_tools_framecpp: + - '2.9' +libabseil: + - '20240116' +libabseil_static: + - '20220623.0' +libaec: + - '1' +libarchive: + - '3.7' +libarrow: + - '17.0' + - '16.1' + - 15 + - 14 +libarrow_all: + - '17.0' + - '16.1' + - 15 + - 14 +libavif: + - 1 +libblitz: + - 1.0.2 +libboost_devel: + - '1.84' +libboost_headers: + - '1.84' +libboost_python_devel: + - '1.84' +libcint: + - '6.1' +libcurl: + - 8 +libcrc32c: + - 1.1 +libdap4: + - 3.20.6 +libdeflate: + - '1.21' +libduckdb_devel: + - '1' +libeantic: + - '2' +libevent: + - 2.1.12 +libexactreal: + - '4' +libffi: + - '3.4' +libflatsurf: + - 3 +libflint: + - '3.0' +libframel: + - '8.41' +libgit2: + - '1.8' +# Keep in sync with google_cloud_cpp +libgoogle_cloud: + - '2.29' +libgoogle_cloud_devel: + - '2.29' +libgoogle_cloud_all_devel: + - '2.29' +libgoogle_cloud_aiplatform_devel: + - '2.29' +libgoogle_cloud_automl_devel: + - '2.29' +libgoogle_cloud_bigquery_devel: + - '2.29' +libgoogle_cloud_bigtable_devel: + - '2.29' +libgoogle_cloud_compute_devel: + - '2.29' +libgoogle_cloud_dialogflow_cx_devel: + - '2.29' +libgoogle_cloud_dialogflow_es_devel: + - '2.29' +libgoogle_cloud_discoveryengine_devel: + - '2.29' +libgoogle_cloud_dlp_devel: + - '2.29' +libgoogle_cloud_iam_devel: + - '2.29' +libgoogle_cloud_oauth2_devel: + - '2.29' +libgoogle_cloud_policytroubleshooter_devel: + - '2.29' +libgoogle_cloud_pubsub_devel: + - '2.29' +libgoogle_cloud_spanner_devel: + - '2.29' +libgoogle_cloud_speech_devel: + - '2.29' +libgoogle_cloud_storage_devel: + - '2.29' +libgrpc: + - "1.62" +libhugetlbfs: + - 2 +libhwloc: + - 2.11.1 +libhwy: + - '1.1' +libiconv: + - 1 +libidn2: + - 2 +libintervalxt: + - 3 +libitk_devel: + - 5.4 +libkml: + - 1.3 +libkml_devel: + - 1.3 +libiio: + - 0 +libitk_devel: + - 5.4 +libmed: + - '4.1' +libmatio: + - 1.5.27 +libmatio_cpp: + - 0.2.5 +libmicrohttpd: + - '1.0' +libnetcdf: + - 4.9.2 +libode: + - 0.16.2 +libopencv: + - 4.10.0 +libopentelemetry_cpp: + - '1.16' +libosqp: + - 0.6.3 +libopenvino_dev: + - 2024.3.0 +libpcap: + - '1.10' +libpng: + - 1.6 +libprotobuf: + - 4.25.3 +libpq: + - '16' +libraw: + - '0.21' +librdkafka: + - '2.4' +librsvg: + - 2 +libsecret: + - 0.18 +libsentencepiece: + - 0.2.0 +libsndfile: + - '1.2' +libsoup: + - 3 +libspatialindex: + - 2.0.0 +libssh: + - 0.10 +libssh2: + - 1 +libsvm: + - '332' +# keep libsqlite in sync with sqlite +libsqlite: + - 3 +libtensorflow: + - "2.16" +libtensorflow_cc: + - "2.16" +libthrift: + - 0.20.0 +libtiff: + - '4.6' +# keep in synch with pytorch +libtorch: + - '2.3' +libunwind: + - '1.6' +libv8: + - 8.9.83 +libvigra: + - '1.12' +libvips: + - 8 +libwebp: + - 1 +libwebp_base: + - 1 +libxcb: + - '1.16' +libxml2: + - 2 +libxsmm: + - 1 +libuuid: + - 2 +libyarp: + - 3.9.0 +libzip: + - 1 +lmdb: + - 0.9.29 +log4cxx: + - 1.2.0 +lz4_c: + - '1.9.3' +lzo: + - 2 +metis: + - 5.1.0 +mimalloc: + - 2.1.7 +mkl: + - '2023' +mkl_devel: + - 2023 +mpg123: + - '1.32' +mpich: + - 4 +mpfr: + - 4 +msgpack_c: + - 6 +msgpack_cxx: + - 6 +mumps_mpi: + - 5.7.2 +mumps_seq: + - 5.7.2 +nccl: + - 2 +ncurses: + - 6 +netcdf_cxx4: + - 4.3 +netcdf_fortran: + - '4.6' +nettle: + - '3.9' +nodejs: + - '20' + - '18' +nss: + - 3 +nspr: + - 4 +nlopt: + - '2.7' +ntl: + - '11.4.3' +# we build for the oldest version possible of numpy for forward compatibility +# we roughly follow NEP29 in choosing the oldest version +numpy: + # part of a zip_keys: python, python_impl, numpy + - 1.22 + - 1.22 + - 1.23 + - 1.26 +occt: + - 7.7.2 +openblas: + - 0.3.* +openexr: + - '3.2' +openh264: + - 2.4.1 +openjpeg: + - '2' +openmpi: + - 4 +openslide: + - 4 +openssl: + - '3' +orc: + - 2.0.2 +pango: + - '1' +pari: + - 2.15.* *_pthread +pcl: + - 1.14.1 +perl: + - 5.32.1 +petsc: + - '3.21' +petsc4py: + - '3.21' +pugixml: + - '1.14' +slepc: + - '3.21' +slepc4py: + - '3.21' +svt_av1: + - 2.2.1 +p11_kit: + - '0.24' +pcre: + - '8' +pcre2: + - '10.44' +pdal: + - '2.7' +libpdal: + - '2.7' +libpdal_core: + - '2.7' +pixman: + - 0 +poco: + - 1.13.3 +poppler: + - '24.08' +postgresql: + - '16' +postgresql_plpython: + - '16' +proj: + - '9.4' +pulseaudio: + - '17.0' +pulseaudio_client: + - '17.0' +pulseaudio_daemon: + - '17.0' +pybind11_abi: + - 4 +python: + # part of a zip_keys: python, python_impl, numpy + - 3.9.* *_cpython + - 3.10.* *_cpython + - 3.11.* *_cpython + - 3.12.* *_cpython +python_impl: + # part of a zip_keys: python, python_impl, numpy + - cpython + - cpython + - cpython + - cpython +is_freethreading: + - false +# Keep in sync with libtorch +pytorch: + - '2.3' +pyqt: + - 5.15 +pyqtwebengine: + - 5.15 +pyqtchart: + - 5.15 +qhull: + - 2020.2 +qpdf: + - 11 +qt: + - 5.15 +qt_main: + - 5.15 +qt6_main: + - '6.7' +qtkeychain: + - '0.14' +rdma_core: + - '53' +re2: + - 2023.09.01 +readline: + - "8" +rocksdb: + - '9.1' +root_base: + - 6.28.12 + - 6.30.4 +ruby: + - 2.5 + - 2.6 +r_base: + - 4.1 # [win] + - 4.3 # [not win] +libscotch: + - 7.0.4 +libptscotch: + - 7.0.4 +scotch: + - 7.0.4 +ptscotch: + - 7.0.4 +s2n: + - 1.5.2 +sdl2: + - '2' +sdl2_image: + - '2' +sdl2_mixer: + - '2' +sdl2_net: + - '2' +sdl2_ttf: + - '2' +singular: + - 4.3.2.p8 +snappy: + - 1.2 +soapysdr: + - '0.8' +sox: + - 14.4.2 +spdlog: + - '1.12' +# keep sqlite in sync with libsqlite +sqlite: + - 3 +srm_ifce: + - 1.24.6 +starlink_ast: + - 9.2.11 +suitesparse: + - '7' +superlu_dist: + - '9' +tbb: + - '2021' +tbb_devel: + - '2021' +tensorflow: + - "2.16" +thrift_cpp: + - 0.20.0 +tinyxml2: + - '10' +tk: + - 8.6 # [not ppc64le] +tiledb: + - '2.25' +ucx: + - '1.16.0' +uhd: + - 4.7.0 +urdfdom: + - '4.0' +vc: # [win] + - 14 # [win] +vigra: + - '1.12' +vlfeat: + - 0.9.21 +volk: + - '3.1' +vtk: + - 9.2.6 +wcslib: + - '8' +wxwidgets: + - '3.2' +x264: + - '1!164.*' +x265: + - '3.5' +xerces_c: + - 3.2 +xrootd: + - '5' +xz: + - 5 +zeromq: + - '4.3.5' +zfp: + - 1.0 +zlib: + - 1 +zlib_ng: + - '2.2' +zstd: + - '1.5' diff --git a/tests/test-recipes/metadata/_render_recipe/meta.yaml b/tests/test-recipes/metadata/_render_recipe/meta.yaml new file mode 100644 index 0000000000..c9f171b4bb --- /dev/null +++ b/tests/test-recipes/metadata/_render_recipe/meta.yaml @@ -0,0 +1,72 @@ +# from https://github.com/conda-forge/petsc4py-feedstock/blob/ccb2a3aed8e07d554b5f8e1dee47508a8d81cac4/recipe/meta.yaml +{% set name = "petsc4py" %} +{% set version = "3.21.5" %} +{% set sha256 = "70e6fa795e9abd8014faec0203cd0cc3efd79f4647c97cafc33776421c9ab1e8" %} +{% set build = 1 %} + +{% set version_xy = version.rsplit('.', 1)[0] %} + +package: + name: petsc4py + version: {{ version }} + +source: + url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz + sha256: {{ sha256 }} + +build: + number: {{ build }} + script: + - {{ PYTHON }} conf/cythonize.py + - export PETSC_DIR=$PREFIX + - {{ PYTHON }} -m pip -v install --no-deps . + skip: true # [win] + track_features: + - petsc4py_complex # [scalar == "complex"] + - petsc4py_cuda # [device != "host"] + +requirements: + build: + - {{ compiler('fortran') }} + - {{ compiler('c') }} + - {{ stdlib("c") }} + - python # [build_platform != target_platform] + - cython # [build_platform != target_platform] + - cross-python_{{ target_platform }} # [build_platform != target_platform] + - numpy # [build_platform != target_platform] + - {{ mpi }} # [mpi == 'openmpi' and build_platform != target_platform] + host: + - python + - cython + - pip + - setuptools + - numpy + - {{ mpi }} + - petsc {{ version_xy }}.* {{ scalar }}_* # [device == 'host'] + - petsc {{ version_xy }}.* {{ device }}_{{ scalar }}_* # [device != 'host'] + run: + - python + - {{ mpi }} + - petsc # pinned by petsc run_exports + run_constrained: + - mpi4py >=3.0.1 + +test: + requires: + - pip + commands: + - pip check + +about: + home: https://bitbucket.org/petsc/petsc4py + summary: Python bindings for PETSc + license: BSD-2-Clause + license_file: LICENSE.rst + +extra: + recipe-maintainers: + - dalcinl + - minrk + - davidbrochart + - SylvainCorlay + - stefanozampini diff --git a/tests/test_render.py b/tests/test_render.py index 940d090781..c755110232 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -16,12 +16,16 @@ find_pkg_dir_or_file_in_pkgs_dirs, get_pin_from_build, open_recipe, + render_recipe, ) -from conda_build.utils import CONDA_PACKAGE_EXTENSION_V1 +from conda_build.utils import CONDA_PACKAGE_EXTENSION_V1, on_linux + +from .utils import metadata_path if TYPE_CHECKING: from pathlib import Path + from conda_build.config import Config from conda_build.metadata import MetaData @@ -134,3 +138,13 @@ def test_open_recipe(tmp_path: Path): ): with open_recipe(path): pass + + +@pytest.mark.benchmark +def test_render_recipe(testing_config: Config) -> None: + recipes = render_recipe(metadata_path / "_render_recipe", config=testing_config) + + if on_linux: + assert len(recipes) == 48 + else: + assert len(recipes) == 16 From 9133820c20e9ee33ba877da93c558ef7055ff6a8 Mon Sep 17 00:00:00 2001 From: Bianca Henderson Date: Wed, 18 Sep 2024 12:08:20 -0400 Subject: [PATCH 2/3] Changelog 24.9.0 (#5484) * Update .authors.yml * Update news * Updated authorship for 24.9.0 * Updated CHANGELOG for 24.9.0 --------- Co-authored-by: Ken Odegard Co-authored-by: Katherine Kinnaman --- .authors.yml | 18 ++++++------ CHANGELOG.md | 34 ++++++++++++++++++++++ news/5322-undefine-build-vars | 20 ------------- news/5392-variant-copy | 3 -- news/5441-24.9-removals | 20 ------------- news/5447-jinja2-for-set-vars | 20 ------------- news/5458-avoid-undefined-jinja-in-skipped | 19 ------------ news/5478-24.9-removals | 19 ------------ 8 files changed, 43 insertions(+), 110 deletions(-) delete mode 100644 news/5322-undefine-build-vars delete mode 100644 news/5392-variant-copy delete mode 100644 news/5441-24.9-removals delete mode 100644 news/5447-jinja2-for-set-vars delete mode 100644 news/5458-avoid-undefined-jinja-in-skipped delete mode 100644 news/5478-24.9-removals diff --git a/.authors.yml b/.authors.yml index f4b769483a..eeb7d860b4 100644 --- a/.authors.yml +++ b/.authors.yml @@ -8,7 +8,7 @@ aliases: - Mike Sarahan - Michael Sarahan - num_commits: 2001 + num_commits: 2002 first_commit: 2015-09-04 21:31:08 - name: Jonathan J. Helmus email: jjhelmus@gmail.com @@ -162,7 +162,7 @@ aliases: - MinRK github: minrk - num_commits: 17 + num_commits: 18 first_commit: 2014-02-13 19:43:59 - name: Matty G email: meawoppl@gmail.com @@ -1121,7 +1121,7 @@ alternate_emails: - becker.mr@gmail.com - beckermr@users.noreply.github.com - num_commits: 21 + num_commits: 22 first_commit: 2019-10-17 23:05:16 github: beckermr - name: Jinzhe Zeng @@ -1171,7 +1171,7 @@ github: pradghos - name: James Lamb email: jaylamb20@gmail.com - num_commits: 1 + num_commits: 2 first_commit: 2020-04-26 13:41:22 github: jameslamb - name: Oleg Alexandrov @@ -1202,7 +1202,7 @@ alternate_emails: - clee@anaconda.com - name: Ken Odegard - num_commits: 213 + num_commits: 220 email: kodegard@anaconda.com first_commit: 2020-09-08 19:53:41 github: kenodegard @@ -1240,7 +1240,7 @@ github: pre-commit-ci[bot] aliases: - pre-commit-ci[bot] - num_commits: 79 + num_commits: 86 first_commit: 2021-11-20 01:47:17 - name: Jacob Walls email: jacobtylerwalls@gmail.com @@ -1251,7 +1251,7 @@ github: beeankha alternate_emails: - beeankha@gmail.com - num_commits: 34 + num_commits: 43 first_commit: 2022-01-19 16:40:06 - name: Conda Bot email: 18747875+conda-bot@users.noreply.github.com @@ -1262,7 +1262,7 @@ alternate_emails: - ad-team+condabot@anaconda.com - 18747875+conda-bot@users.noreply.github.com - num_commits: 58 + num_commits: 62 first_commit: 2022-01-17 18:09:22 - name: Uwe L. Korn email: xhochy@users.noreply.github.com @@ -1310,7 +1310,7 @@ - name: dependabot[bot] email: 49699333+dependabot[bot]@users.noreply.github.com github: dependabot[bot] - num_commits: 34 + num_commits: 35 first_commit: 2022-05-31 04:34:40 - name: Serhii Kupriienko email: 79282962+skupr-anaconda@users.noreply.github.com diff --git a/CHANGELOG.md b/CHANGELOG.md index a088d2f953..80e4756c77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ [//]: # (current developments) +## 24.9.0 (2024-09-18) + +### Enhancements + +* Reduce render time when there is a large number of unused variants. (#5392) + +### Bug fixes + +* Ensure variables mentioned in `script_env` are undefined in the multi-output build environment + if they are undefined in the environment that `conda-build` is invoked from. (#5322) +* Variables used in single-line jinja2 `for` and `set` statements are now properly included in the variant + matrix for some edge cases. (#5447) +* Allow undefined jinja variables when a particular metadata block evaluates as skipped. (#5458) + +### Deprecations + +* Remove `conda_build.build.check_external`. `patchelf` is an explicit conda-build dependency on Linux, so it will always be installed. (#5441) +* Remove `conda_build.metadata._get_env_path`. Use `conda.base.context.locate_prefix_by_name` instead. (#5441) +* Remove `conda_build.build._construct_metadata_for_test_from_recipe`. Test built packages instead, not recipes (e.g., `conda build --test package` instead of `conda build --test recipe/`). (#5478) + +### Contributors + +* @beeankha +* @conda-bot +* @jameslamb +* @kenodegard +* @beckermr +* @msarahan +* @minrk +* @dependabot[bot] +* @pre-commit-ci[bot] + + + ## 24.7.1 (2024-07-30) ### Bug fixes diff --git a/news/5322-undefine-build-vars b/news/5322-undefine-build-vars deleted file mode 100644 index 67c47373b5..0000000000 --- a/news/5322-undefine-build-vars +++ /dev/null @@ -1,20 +0,0 @@ -### Enhancements - -* - -### Bug fixes - -* Ensures that variables mentioned in `script_env` are undefined in multi-output build environment - if undefined in the environment `conda-build` is invoked from. - -### Deprecations - -* - -### Docs - -* - -### Other - -* diff --git a/news/5392-variant-copy b/news/5392-variant-copy deleted file mode 100644 index 83c8f30eed..0000000000 --- a/news/5392-variant-copy +++ /dev/null @@ -1,3 +0,0 @@ -### Enhancements - -* Reduce render time when there is a large number of unused variants. (#5392) diff --git a/news/5441-24.9-removals b/news/5441-24.9-removals deleted file mode 100644 index 1613579d16..0000000000 --- a/news/5441-24.9-removals +++ /dev/null @@ -1,20 +0,0 @@ -### Enhancements - -* - -### Bug fixes - -* - -### Deprecations - -* Remove `conda_build.build.check_external`. `patchelf` is an explicit conda-build dependency on Linux so it will always be installed. (#5441) -* Remove `conda_build.metadata._get_env_path`. Use `conda.base.context.locate_prefix_by_name` instead. (#5441) - -### Docs - -* - -### Other - -* diff --git a/news/5447-jinja2-for-set-vars b/news/5447-jinja2-for-set-vars deleted file mode 100644 index fbca651f89..0000000000 --- a/news/5447-jinja2-for-set-vars +++ /dev/null @@ -1,20 +0,0 @@ -### Enhancements - -* - -### Bug fixes - -* Variables used in single-line jinja2 `for` and `set` statements are now properly included in the variant - matrix for some edge cases. (#5447) - -### Deprecations - -* - -### Docs - -* - -### Other - -* diff --git a/news/5458-avoid-undefined-jinja-in-skipped b/news/5458-avoid-undefined-jinja-in-skipped deleted file mode 100644 index e1c232a65e..0000000000 --- a/news/5458-avoid-undefined-jinja-in-skipped +++ /dev/null @@ -1,19 +0,0 @@ -### Enhancements - -* - -### Bug fixes - -* Allow undefined jinja variables when a particular metadata block evaluates as skipped. (#5458) - -### Deprecations - -* - -### Docs - -* - -### Other - -* diff --git a/news/5478-24.9-removals b/news/5478-24.9-removals deleted file mode 100644 index 5d19e52eda..0000000000 --- a/news/5478-24.9-removals +++ /dev/null @@ -1,19 +0,0 @@ -### Enhancements - -* - -### Bug fixes - -* - -### Deprecations - -* Remove `conda_build.build._construct_metadata_for_test_from_recipe`. Test built packages instead, not recipes (e.g., `conda build --test package` instead of `conda build --test recipe/`). (#5478) - -### Docs - -* - -### Other - -* From c7df14774e62a809ac24c0f3dba1054d188770b2 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Thu, 19 Sep 2024 13:12:45 -0500 Subject: [PATCH 3/3] Require `setuptools <71.1.0` (#5496) --- tests/requirements-ci.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements-ci.txt b/tests/requirements-ci.txt index 0d3b9a1fee..bb41f87fd7 100644 --- a/tests/requirements-ci.txt +++ b/tests/requirements-ci.txt @@ -15,4 +15,5 @@ pytest-mock pytest-rerunfailures pytest-xdist ruamel.yaml +setuptools <75.1.0 # temporary, see https://github.com/conda/conda-build/issues/5493 tomli # [py<3.11] for coverage pyproject.toml