From 5c612617a0fe3d56fd758a1a996c0959e65be61f Mon Sep 17 00:00:00 2001 From: nichmor Date: Thu, 1 Aug 2024 15:05:44 +0300 Subject: [PATCH] fix: omit also jinja functions (#37) --- .../jinja/__init__.py | 0 .../jinja/filters.py | 28 ++++ .../{ => jinja}/jinja.py | 63 ++++++-- .../jinja/objects.py | 35 ++++ src/rattler_build_conda_compat/jinja/utils.py | 11 ++ tests/__snapshots__/test_jinja.ambr | 144 ++++++++++++++++- tests/data/context.yaml | 11 -- tests/data/mamba_recipe.yaml | 150 ++++++++++++++++++ tests/test_jinja.py | 15 +- 9 files changed, 424 insertions(+), 33 deletions(-) create mode 100644 src/rattler_build_conda_compat/jinja/__init__.py create mode 100644 src/rattler_build_conda_compat/jinja/filters.py rename src/rattler_build_conda_compat/{ => jinja}/jinja.py (56%) create mode 100644 src/rattler_build_conda_compat/jinja/objects.py create mode 100644 src/rattler_build_conda_compat/jinja/utils.py delete mode 100644 tests/data/context.yaml create mode 100644 tests/data/mamba_recipe.yaml diff --git a/src/rattler_build_conda_compat/jinja/__init__.py b/src/rattler_build_conda_compat/jinja/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/rattler_build_conda_compat/jinja/filters.py b/src/rattler_build_conda_compat/jinja/filters.py new file mode 100644 index 0000000..b28e455 --- /dev/null +++ b/src/rattler_build_conda_compat/jinja/filters.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from rattler_build_conda_compat.jinja.utils import _MissingUndefined + + +def _version_to_build_string(some_string: str | _MissingUndefined) -> str: + """ + Converts some version by removing the . character and returning only the first two elements of the version. + If piped value is undefined, it returns the undefined value as is. + """ + if isinstance(some_string, _MissingUndefined): + return f"{some_string._undefined_name}_version_to_build_string" # noqa: SLF001 + # We first split the string by whitespace and take the first part + split = some_string.split()[0] if some_string.split() else some_string + # We then split the string by . and take the first two parts + parts = split.split(".") + major = parts[0] if len(parts) > 0 else "" + minor = parts[1] if len(parts) > 1 else "" + return f"{major}{minor}" + + +def _bool(value: str) -> bool: + return bool(value) + + +def _split(s: str, sep: str = " ") -> list[str]: + """Filter that split a string by a separator""" + return s.split(sep) diff --git a/src/rattler_build_conda_compat/jinja.py b/src/rattler_build_conda_compat/jinja/jinja.py similarity index 56% rename from src/rattler_build_conda_compat/jinja.py rename to src/rattler_build_conda_compat/jinja/jinja.py index 456849c..fc2db37 100644 --- a/src/rattler_build_conda_compat/jinja.py +++ b/src/rattler_build_conda_compat/jinja/jinja.py @@ -4,8 +4,18 @@ import jinja2 import yaml -from jinja2 import DebugUndefined +from rattler_build_conda_compat.jinja.filters import _bool, _split, _version_to_build_string +from rattler_build_conda_compat.jinja.objects import ( + _stub_compatible_pin, + _stub_is_linux, + _stub_is_unix, + _stub_is_win, + _stub_match, + _stub_subpackage_pin, + _StubEnv, +) +from rattler_build_conda_compat.jinja.utils import _MissingUndefined from rattler_build_conda_compat.loader import load_yaml @@ -13,33 +23,59 @@ class RecipeWithContext(TypedDict, total=False): context: dict[str, str] -class _MissingUndefined(DebugUndefined): - def __str__(self) -> str: - """ - By default, `DebugUndefined` return values in the form `{{ value }}`. - `rattler-build` has a different syntax, so we need to override this method, - and return the value in the form `${{ value }}`. - """ - return f"${super().__str__()}" - - def jinja_env() -> jinja2.Environment: """ Create a `rattler-build` specific Jinja2 environment with modified syntax. + Target platform, build platform, and mpi are set to linux-64 by default. """ - return jinja2.Environment( + + env = jinja2.Environment( variable_start_string="${{", variable_end_string="}}", trim_blocks=True, lstrip_blocks=True, - autoescape=True, + autoescape=jinja2.select_autoescape(default_for_string=False), undefined=_MissingUndefined, ) + env_obj = _StubEnv() + + # inject rattler-build recipe functions in jinja environment + env.globals.update( + { + "compiler": lambda x: x + "_compiler_stub", + "stdlib": lambda x: x + "_stdlib_stub", + "pin_subpackage": _stub_subpackage_pin, + "pin_compatible": _stub_compatible_pin, + "cdt": lambda *args, **kwargs: "cdt_stub", # noqa: ARG005 + "env": env_obj, + "match": _stub_match, + "is_unix": _stub_is_unix, + "is_win": _stub_is_win, + "is_linux": _stub_is_linux, + "unix": True, + "linux": True, + "target_platform": "linux-64", + "build_platform": "linux-64", + "mpi": "mpi", + } + ) + + # inject rattler-build recipe filters in jinja environment + env.filters.update( + { + "version_to_buildstring": _version_to_build_string, + "split": _split, + "bool": _bool, + } + ) + return env + def load_recipe_context(context: dict[str, str], jinja_env: jinja2.Environment) -> dict[str, str]: """ Load all string values from the context dictionary as Jinja2 templates. + Use linux-64 as default target_platform, build_platform, and mpi. """ # Process each key-value pair in the dictionary for key, value in context.items(): @@ -56,6 +92,7 @@ def render_recipe_with_context(recipe_content: RecipeWithContext) -> dict[str, A """ Render the recipe using known values from context section. Unknown values are not evaluated and are kept as it is. + Target platform, build platform, and mpi are set to linux-64 by default. Examples: --- diff --git a/src/rattler_build_conda_compat/jinja/objects.py b/src/rattler_build_conda_compat/jinja/objects.py new file mode 100644 index 0000000..32e64ba --- /dev/null +++ b/src/rattler_build_conda_compat/jinja/objects.py @@ -0,0 +1,35 @@ +from __future__ import annotations + + +class _StubEnv: + """A class to represent the env object used in rattler-build recipe.""" + + def get(self, env_var: str, default: str | None = None) -> str: # noqa: ARG002 + return f"""env_"{env_var}" """ + + def exists(self, env_var: str) -> str: + return f"""env_exists_"{env_var}" """ + + +def _stub_compatible_pin(*args, **kwargs) -> str: # noqa: ARG001, ANN003, ANN002 + return f"compatible_pin {args[0]}" + + +def _stub_subpackage_pin(*args, **kwargs) -> str: # noqa: ARG001, ANN003, ANN002 + return f"subpackage_pin {args[0]}" + + +def _stub_match(*args, **kwargs) -> str: # noqa: ARG001, ANN003, ANN002 + return f"match {args[0]}" + + +def _stub_is_unix(platform: str) -> str: + return f"is_unix {platform}" + + +def _stub_is_win(platform: str) -> str: + return f"is_win {platform}" + + +def _stub_is_linux(platform: str) -> str: + return f"is_linux {platform}" diff --git a/src/rattler_build_conda_compat/jinja/utils.py b/src/rattler_build_conda_compat/jinja/utils.py new file mode 100644 index 0000000..dc688e9 --- /dev/null +++ b/src/rattler_build_conda_compat/jinja/utils.py @@ -0,0 +1,11 @@ +from jinja2 import DebugUndefined + + +class _MissingUndefined(DebugUndefined): + def __str__(self) -> str: + """ + By default, `DebugUndefined` return values in the form `{{ value }}`. + `rattler-build` has a different syntax, so we need to override this method, + and return the value in the form `${{ value }}`. + """ + return f"${super().__str__()}" diff --git a/tests/__snapshots__/test_jinja.ambr b/tests/__snapshots__/test_jinja.ambr index f3e6259..18d5008 100644 --- a/tests/__snapshots__/test_jinja.ambr +++ b/tests/__snapshots__/test_jinja.ambr @@ -1,15 +1,145 @@ # serializer version: 1 # name: test_render_recipe_with_context ''' + about: + description: '# Mamba, the Fast Cross-Platform Package Manager + + env_"MY_ENV_VAR" + + env_"MY_ENV_VAR" + + env_exists_"MY_ENV_VAR" + + ' + homepage: https://github.com/mamba-org/mamba + license: BSD-3-Clause + license_family: BSD + license_file: LICENSE + repository: https://github.com/mamba-org/mamba + summary: A fast drop-in alternative to conda, using libsolv for dependency resolution build: - string: ${{ blas_variant }}${{ hash }}_foo-bla + number: '2' context: - name: foo - name_version: foo-bla - version: bla - package: - name: foo - version: bla + build_number: '2' + libmamba_version: 1.5.8 + libmambapy_version: 1.5.8 + mamba_version: 1.5.8 + name: mamba + release: 2024.03.25 + outputs: + - build: + script: + - build_mamba.sh + - '' + package: + name: libmamba + version: 1.5.8 + requirements: + build: + - cxx_compiler_stub + - cmake + - ninja + - '' + host: + - libsolv >=0.7.23 + - libcurl >=8.4.0 + - fmt + - '' + ignore_run_exports: + by_name: + - spdlog + - python + run: + - libsolv >=0.7.23 + run_exports: + - subpackage_pin libmamba + tests: + - script: + - else: + - if not exist %LIBRARY_PREFIX%\include\mamba\version.hpp (exit 1) + if: unix + then: + - test -d ${PREFIX}/include/mamba + - build: + script: + - build_mamba.sh + - '' + package: + name: libmambapy + version: 1.5.8 + requirements: + build: + - cxx_compiler_stub + - cmake + - ninja + - if: build_platform != target_platform + then: + - python + - cross-python_linux-64 + - pybind11 + - pybind11-abi + host: + - python + - nlohmann_json + - subpackage_pin libmamba + ignore_run_exports: + by_name: + - spdlog + run: + - python + - subpackage_pin libmamba + run_exports: + - subpackage_pin libmambapy + tests: + - python: + imports: + - libmambapy + - libmambapy.bindings + - script: + - python -c "import libmambapy._version; assert libmambapy._version.__version__ + == '1.5.8'" + - build: + python: + entry_points: + - mamba = mamba.mamba:main + script: + - build_mamba.sh + - '' + - '' + string: pypython_version_to_build_stringh${{ hash }}_2 + package: + name: mamba + version: 1.5.8 + requirements: + build: + - if: build_platform != target_platform + then: + - python + - cross-python_linux-64 + run: + - python + - conda >=23.9,<24 + - subpackage_pin libmambapy + tests: + - python: + imports: + - mamba + - script: + - mamba --help + - python -c "import mamba._version; assert mamba._version.__version__ == '1.5.8'" + - if: linux + then: + - test -f ${PREFIX}/etc/profile.d/mamba.sh + - mamba create -n test_py2 python=2.7 --dry-run + - mamba install xtensor xsimd -c conda-forge --dry-run + - if: unix + then: + - test -f ${PREFIX}/condabin/mamba + recipe: + name: mamba-split + source: + sha256: 6ddaf4b0758eb7ca1250f427bc40c2c3ede43257a60bac54e4320a4de66759a6 + url: https://github.com/mamba-org/mamba/archive/refs/tags/2024.03.25.tar.gz ''' # --- diff --git a/tests/data/context.yaml b/tests/data/context.yaml deleted file mode 100644 index af611ce..0000000 --- a/tests/data/context.yaml +++ /dev/null @@ -1,11 +0,0 @@ -context: - name: "foo" - version: "bla" - name_version: ${{ name }}-${{ version }} - -package: - name: ${{ name }} - version: ${{ version }} - -build: - string: ${{ blas_variant }}${{ hash }}_${{ name_version }} diff --git a/tests/data/mamba_recipe.yaml b/tests/data/mamba_recipe.yaml new file mode 100644 index 0000000..0478704 --- /dev/null +++ b/tests/data/mamba_recipe.yaml @@ -0,0 +1,150 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json + +context: + name: mamba + libmamba_version: "1.5.8" + libmambapy_version: "1.5.8" + mamba_version: "1.5.8" + release: "2024.03.25" + build_number: 2 + +recipe: + name: mamba-split + +source: + url: https://github.com/mamba-org/mamba/archive/refs/tags/${{ release }}.tar.gz + sha256: 6ddaf4b0758eb7ca1250f427bc40c2c3ede43257a60bac54e4320a4de66759a6 + +build: + number: ${{ build_number }} + +outputs: + - package: + name: libmamba + version: ${{ libmamba_version }} + build: + script: + - ${{ "build_mamba.sh" if unix }} + - ${{ "build_mamba.bat" if win }} + requirements: + build: + - ${{ compiler('cxx') }} + - cmake + - ninja + - ${{ "python" if win }} + host: + - libsolv >=0.7.23 + - libcurl >=8.4.0 + - fmt + - ${{ "winreg" if win }} + run: + - libsolv >=0.7.23 + run_exports: + - ${{ pin_subpackage('libmamba', max_pin='x.x') }} + ignore_run_exports: + by_name: + - spdlog + - python + tests: + - script: + - if: unix + then: + - test -d ${PREFIX}/include/mamba # [unix] + else: + - if not exist %LIBRARY_PREFIX%\include\mamba\version.hpp (exit 1) # [win] + + - package: + name: libmambapy + version: ${{ libmambapy_version }} + build: + script: + - ${{ "build_mamba.sh" if unix }} + - ${{ "build_mamba.bat" if win }} + # string: py_sup${{ python | version_to_buildstring }}h${{ hash }}_${{ build_number }} + requirements: + build: + - ${{ compiler('cxx') }} + - cmake + - ninja + - if: build_platform != target_platform + then: + - python + - cross-python_${{ target_platform }} + - pybind11 + - pybind11-abi + host: + - python + - nlohmann_json + - ${{ pin_subpackage('libmamba', exact=True) }} + run: + - python + - ${{ pin_subpackage('libmamba', exact=True) }} + run_exports: + - ${{ pin_subpackage('libmambapy', max_pin='x.x') }} + ignore_run_exports: + by_name: + - spdlog + tests: + - python: + imports: + - libmambapy + - libmambapy.bindings + - script: + - python -c "import libmambapy._version; assert libmambapy._version.__version__ == '${{ libmambapy_version }}'" + + - package: + name: mamba + version: ${{ mamba_version }} + build: + script: + - ${{ "build_mamba.sh" if unix }} + - ${{ "build_mamba.bat" if win }} + - ${{ "test_mamba.bat" if target_platform == win }} + string: py${{ python | version_to_buildstring }}h${{ hash }}_${{ build_number }} + python: + entry_points: + - mamba = mamba.mamba:main + requirements: + build: + - if: build_platform != target_platform + then: + - python + - cross-python_${{ target_platform }} + run: + - python + - conda >=23.9,<24 + - ${{ pin_subpackage('libmambapy', exact=True) }} + + tests: + - python: + imports: + - mamba + - script: + - mamba --help + # for some reason tqdm doesn't have a proper colorama dependency so pip check fails + # but that's completely unrelated to mamba + - python -c "import mamba._version; assert mamba._version.__version__ == '${{ mamba_version }}'" + + - if: linux + then: + - test -f ${PREFIX}/etc/profile.d/mamba.sh + # these tests work when run on win, but for some reason not during conda build + - mamba create -n test_py2 python=2.7 --dry-run + - mamba install xtensor xsimd -c conda-forge --dry-run + - if: unix + then: + - test -f ${PREFIX}/condabin/mamba + +about: + homepage: https://github.com/mamba-org/mamba + license: BSD-3-Clause + license_file: LICENSE + license_family: BSD + summary: A fast drop-in alternative to conda, using libsolv for dependency resolution + description: | + # Mamba, the Fast Cross-Platform Package Manager + ${{ env.get("MY_ENV_VAR") }} + ${{ env.get("MY_ENV_VAR", default="default_value") }} + ${{ env.exists("MY_ENV_VAR")}} + + repository: https://github.com/mamba-org/mamba diff --git a/tests/test_jinja.py b/tests/test_jinja.py index 3feb3a0..62e58a0 100644 --- a/tests/test_jinja.py +++ b/tests/test_jinja.py @@ -1,11 +1,14 @@ from pathlib import Path import yaml -from rattler_build_conda_compat.jinja import load_yaml, render_recipe_with_context +from rattler_build_conda_compat.jinja.filters import _version_to_build_string +from rattler_build_conda_compat.jinja.jinja import render_recipe_with_context +from rattler_build_conda_compat.jinja.utils import _MissingUndefined +from rattler_build_conda_compat.loader import load_yaml def test_render_recipe_with_context(snapshot) -> None: - recipe = Path("tests/data/context.yaml") + recipe = Path("tests/data/mamba_recipe.yaml") with recipe.open() as f: recipe_yaml = load_yaml(f) @@ -13,3 +16,11 @@ def test_render_recipe_with_context(snapshot) -> None: into_yaml = yaml.dump(rendered) assert into_yaml == snapshot + + +def test_version_to_build_string() -> None: + assert _version_to_build_string("1.2.3") == "12" + assert _version_to_build_string("1.2") == "12" + assert _version_to_build_string("nothing") == "nothing" + some_undefined = _MissingUndefined(name="python") + assert _version_to_build_string(some_undefined) == "python_version_to_build_string"