Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test: add backend option for test configuration and filter constructors accordingly #1705

Closed
wants to merge 4 commits into from
Closed

test: add backend option for test configuration and filter constructors accordingly #1705

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5f4114b
test: add backend option for test configuration and filter constructo…
luke396 Jan 2, 2025
f73e27c
revert else
luke396 Jan 2, 2025
93dd600
unknown backend handling
luke396 Jan 2, 2025
dbf7308
Merge branch 'main' into add-test-backend
luke396 Jan 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 40 additions & 5 deletions tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,12 @@ def pytest_addoption(parser: Any) -> None:
type=str,
help="libraries to test",
)
parser.addoption(
"--backend",
action="store",
default="all",
help="specify test backend (pandas, modin, dask, all)",
)


def pytest_configure(config: pytest.Config) -> None:
Expand Down Expand Up @@ -171,8 +177,34 @@ def spark_session() -> Generator[SparkSession, None, None]: # pragma: no cover
}
GPU_CONSTRUCTORS: dict[str, Callable[[Any], IntoFrame]] = {"cudf": cudf_constructor}

BACKEND_MAPPING = {
"pandas": ["pandas", "pandas[nullable]", "pandas[pyarrow]"],
"modin": ["modin", "modin[pyarrow]"],
"dask": ["dask"],
"polars": ["polars[eager]", "polars[lazy]"],
"pyarrow": ["pyarrow"],
"cudf": ["cudf"],
}


def get_filtered_constructors(
backend: str, all_constructors: dict[str, Callable[..., Any]]
) -> dict[str, Callable[..., Any]]:
if backend == "all":
return all_constructors
if backend not in BACKEND_MAPPING: # pragma: no cover
msg = f"Unknown backend {backend}"
raise ValueError(msg)
return { # pragma: no cover
name: func
for name, func in all_constructors.items()
if any(name.startswith(prefix) for prefix in BACKEND_MAPPING.get(backend, []))
}


def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
backend = metafunc.config.getoption("--backend")

if metafunc.config.getoption("all_cpu_constructors"):
selected_constructors: list[str] = [
*iter(EAGER_CONSTRUCTORS.keys()),
Expand All @@ -184,19 +216,22 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
else: # pragma: no cover
selected_constructors = metafunc.config.getoption("constructors").split(",")

filtered_eager = get_filtered_constructors(backend, EAGER_CONSTRUCTORS)
filtered_lazy = get_filtered_constructors(backend, LAZY_CONSTRUCTORS)

eager_constructors: list[Callable[[Any], IntoDataFrame]] = []
eager_constructors_ids: list[str] = []
constructors: list[Callable[[Any], IntoFrame]] = []
constructors_ids: list[str] = []

for constructor in selected_constructors:
if constructor in EAGER_CONSTRUCTORS:
eager_constructors.append(EAGER_CONSTRUCTORS[constructor])
if constructor in filtered_eager:
eager_constructors.append(filtered_eager[constructor])
eager_constructors_ids.append(constructor)
constructors.append(EAGER_CONSTRUCTORS[constructor])
constructors.append(filtered_eager[constructor])
constructors_ids.append(constructor)
elif constructor in LAZY_CONSTRUCTORS:
constructors.append(LAZY_CONSTRUCTORS[constructor])
elif constructor in filtered_lazy:
constructors.append(filtered_lazy[constructor])
constructors_ids.append(constructor)
else: # pragma: no cover
msg = f"Expected one of {EAGER_CONSTRUCTORS.keys()} or {LAZY_CONSTRUCTORS.keys()}, got {constructor}"
Expand Down
Loading