Merge pull request #16 from tdejager/feat/get-all-url-sources

feat: get all url sources

src/rattler_build_conda_compat/conditional_list.py

@@ -14,7 +14,7 @@ class IfStatement(Generic[T]):
 ConditionalList = Union[T, "IfStatement[T]", list[Union[T, "IfStatement[T]"]]]
 
 
-def visit_conditional_list(
+def visit_conditional_list(  # noqa: C901
     value: ConditionalList[T], evaluator: Callable[[Any], bool] | None = None
 ) -> Generator[T, None, None]:
     """
@@ -36,23 +36,29 @@ def yield_from_list(value: list[T] | T) -> Generator[T, None, None]:
         else:
             yield value
 
-    value = value if isinstance(value, list) else [value]
+    if not isinstance(value, list):
+        value = [value]
 
     for element in value:
         if isinstance(element, dict):
             if (expr := element.get("if", None)) is not None:
                 then = element.get("then")
                 otherwise = element.get("else")
+                # Evaluate the if expression if the evaluator is provided
                 if evaluator:
                     if evaluator(expr):
                         yield from yield_from_list(then)
                     elif otherwise:
                         yield from yield_from_list(otherwise)
+                # Otherwise, just yield the branches
                 else:
                     yield from yield_from_list(then)
                     if otherwise:
                         yield from yield_from_list(otherwise)
             else:
+                # In this case its not an if statement
                 yield element
+        # If the element is not a dictionary, just yield it
         else:
+            # (tim) I get a pyright error here, but I don't know how to fix it
             yield element

src/rattler_build_conda_compat/recipe_sources.py

@@ -0,0 +1,50 @@
+import typing
+from typing import Any, Iterator, Mapping, NotRequired, TypedDict
+
+from .conditional_list import ConditionalList, visit_conditional_list
+
+OptionalUrlList = str | list[str] | None
+
+
+class Source(TypedDict):
+    url: NotRequired[str]
+
+
+def get_all_url_sources(recipe: Mapping[Any, Any]) -> Iterator[str]:
+    """
+    Get all url sources from the recipe. This can be from a list of sources,
+    a single source, or conditional and its branches.
+
+    Arguments
+    ---------
+    * `recipe` - The recipe to inspect. This should be a yaml object.
+
+    Returns
+    -------
+    A list of sources.
+    """
+
+    sources = recipe.get("source", None)
+    sources = typing.cast(ConditionalList[Source], sources)
+
+    # Try getting all url top-level sources
+    if sources is not None:
+        sources = visit_conditional_list(sources, None)
+        for source in sources:
+            if "url" in source:
+                yield source["url"]
+
+    outputs = recipe.get("outputs", None)
+    if outputs is None:
+        return
+
+    outputs = visit_conditional_list(outputs, None)
+    for output in outputs:
+        sources = output.get("source", None)
+        sources = typing.cast(ConditionalList[Source], sources)
+        if sources is None:
+            return
+        sources = visit_conditional_list(sources, None)
+        for source in sources:
+            if "url" in source:
+                yield source["url"]

tests/data/if_then_source.yaml

@@ -0,0 +1,6 @@
+source:
+  - if: windows
+    then:
+      - url: https://foo.com
+    else:
+      - url: https://bar.com

tests/data/list_of_url_sources.yaml

@@ -0,0 +1,4 @@
+source:
+  - url:
+      - "https://foo.com"
+      - "https://bar.com"

tests/data/multiple_sources.yaml

@@ -0,0 +1,5 @@
+source:
+  - url: https://foo.com
+    sha256: 1234567890abcdef
+  - url: https://bar.com
+    sha256: 1234567890abcdef

tests/data/outputs_source.yaml

@@ -0,0 +1,11 @@
+outputs:
+  source:
+    - if: foo
+      then:
+        - url: https://foo.com
+    - if: bla
+      then:
+        - url: https://bar.com
+      else:
+        - url: https://baz.com
+    - url: https://qux.com

tests/data/single_source.yaml

@@ -0,0 +1,2 @@
+source:
+  - url: https://foo.com

tests/test_recipe_sources.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+from rattler_build_conda_compat.loader import load_yaml
+from rattler_build_conda_compat.recipe_sources import get_all_url_sources
+
+
+@pytest.mark.parametrize(
+    ("partial_recipe", "expected_output"),
+    [
+        ("single_source.yaml", ["https://foo.com"]),
+        ("multiple_sources.yaml", ["https://foo.com", "https://bar.com"]),
+        ("if_then_source.yaml", ["https://foo.com", "https://bar.com"]),
+        (
+            "outputs_source.yaml",
+            ["https://foo.com", "https://bar.com", "https://baz.com", "https://qux.com"],
+        ),
+    ],
+)
+def test_recipe_sources(partial_recipe: str, expected_output: list[str]) -> None:
+    """Test that the recipe sources are correctly extracted from the recipe"""
+    path = Path(f"{Path(__file__).parent}/data/{partial_recipe}")
+    recipe = load_yaml(path.read_text())
+    assert list(get_all_url_sources(recipe)) == expected_output