Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test: codspeed CI workflow #819

Closed
wants to merge 22 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions .github/workflows/codspeed.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
name: codspeed benchmarks

on:
pull_request:
workflow_dispatch:
push:
branches:
- main

jobs:
codspeed-benchmarks:
name: codspeed benchmarks
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install uv
run: curl -LsSf https://astral.sh/uv/install.sh | sh
- name: Install dependencies
run: |
uv pip install -r requirements-dev.txt "numpy<2.0.0" --system
- name: show-deps
run: uv pip freeze
- name: Run benchmarks
uses: CodSpeedHQ/action@v3
with:
run: pytest tpch/benchmarks --codspeed
1 change: 1 addition & 0 deletions requirements-dev.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ polars
pre-commit
pyarrow
pytest
pytest-codspeed
pytest-cov
pytest-env
hypothesis
Expand Down
Empty file added tpch/benchmarks/__init__.py
Empty file.
71 changes: 71 additions & 0 deletions tpch/benchmarks/q10_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
from __future__ import annotations

from datetime import datetime
from pathlib import Path
from typing import TYPE_CHECKING

import pytest

import narwhals.stable.v1 as nw
from tpch.benchmarks.utils import lib_to_reader

if TYPE_CHECKING:
from pytest_codspeed.plugin import BenchmarkFixture

DATA_FOLDER = Path("tests/data")


@pytest.mark.parametrize("library", ["pandas", "polars", "pyarrow", "dask"])
def test_q10(benchmark: BenchmarkFixture, library: str) -> None:
read_fn = lib_to_reader[library]

customer = nw.from_native(read_fn(DATA_FOLDER / "customer.parquet")).lazy()
lineitem = nw.from_native(read_fn(DATA_FOLDER / "lineitem.parquet")).lazy()
nation = nw.from_native(read_fn(DATA_FOLDER / "nation.parquet")).lazy()
orders = nw.from_native(read_fn(DATA_FOLDER / "orders.parquet")).lazy()

_ = benchmark(q10, customer, nation, lineitem, orders)


def q10(
customer: nw.LazyFrame,
nation: nw.LazyFrame,
lineitem: nw.LazyFrame,
orders: nw.LazyFrame,
) -> nw.DataFrame:
var1 = datetime(1993, 10, 1)
var2 = datetime(1994, 1, 1)

return (
customer.join(orders, left_on="c_custkey", right_on="o_custkey")
.join(lineitem, left_on="o_orderkey", right_on="l_orderkey")
.join(nation, left_on="c_nationkey", right_on="n_nationkey")
.filter(nw.col("o_orderdate").is_between(var1, var2, closed="left"))
.filter(nw.col("l_returnflag") == "R")
.with_columns(
(nw.col("l_extendedprice") * (1 - nw.col("l_discount"))).alias("revenue")
)
.group_by(
"c_custkey",
"c_name",
"c_acctbal",
"c_phone",
"n_name",
"c_address",
"c_comment",
)
.agg(nw.sum("revenue"))
.select(
"c_custkey",
"c_name",
"revenue",
"c_acctbal",
"n_name",
"c_address",
"c_phone",
"c_comment",
)
.sort(by="revenue", descending=True)
.head(20)
.collect()
)
53 changes: 53 additions & 0 deletions tpch/benchmarks/q11_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
from __future__ import annotations

from pathlib import Path
from typing import TYPE_CHECKING

import pytest

import narwhals.stable.v1 as nw
from tpch.benchmarks.utils import lib_to_reader

if TYPE_CHECKING:
from pytest_codspeed.plugin import BenchmarkFixture

DATA_FOLDER = Path("tests/data")


@pytest.mark.parametrize("library", ["pandas", "polars", "pyarrow", "dask"])
def test_q11(benchmark: BenchmarkFixture, library: str) -> None:
read_fn = lib_to_reader[library]

nation = nw.from_native(read_fn(DATA_FOLDER / "nation.parquet")).lazy()
partsupp = nw.from_native(read_fn(DATA_FOLDER / "partsupp.parquet")).lazy()
supplier = nw.from_native(read_fn(DATA_FOLDER / "supplier.parquet")).lazy()

_ = benchmark(q11, partsupp, nation, supplier)


def q11(
partsupp: nw.LazyFrame, nation: nw.LazyFrame, supplier: nw.LazyFrame
) -> nw.DataFrame:
var1 = "GERMANY"
var2 = 0.0001

q1 = (
partsupp.join(supplier, left_on="ps_suppkey", right_on="s_suppkey")
.join(nation, left_on="s_nationkey", right_on="n_nationkey")
.filter(nw.col("n_name") == var1)
)
q2 = q1.select(
(nw.col("ps_supplycost") * nw.col("ps_availqty")).sum().round(2).alias("tmp")
* var2
)

return (
q1.with_columns((nw.col("ps_supplycost") * nw.col("ps_availqty")).alias("value"))
.group_by("ps_partkey")
.agg(nw.sum("value"))
.join(q2, how="cross")
.filter(nw.col("value") > nw.col("tmp"))
.select("ps_partkey", "value")
.sort("value", descending=True)
.collect()
)
54 changes: 54 additions & 0 deletions tpch/benchmarks/q15_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
from __future__ import annotations

from datetime import date
from pathlib import Path
from typing import TYPE_CHECKING

import pytest

import narwhals.stable.v1 as nw
from tpch.benchmarks.utils import lib_to_reader

if TYPE_CHECKING:
from pytest_codspeed.plugin import BenchmarkFixture

DATA_FOLDER = Path("tests/data")


@pytest.mark.parametrize("library", ["pandas", "polars", "pyarrow", "dask"])
def test_q15(benchmark: BenchmarkFixture, library: str) -> None:
read_fn = lib_to_reader[library]

lineitem = nw.from_native(read_fn(DATA_FOLDER / "lineitem.parquet")).lazy()
supplier = nw.from_native(read_fn(DATA_FOLDER / "supplier.parquet")).lazy()

_ = benchmark(q15, lineitem, supplier)


def q15(
lineitem: nw.LazyFrame,
supplier: nw.LazyFrame,
) -> nw.DataFrame:
var1 = date(1996, 1, 1)
var2 = date(1996, 4, 1)

revenue = (
lineitem.filter(nw.col("l_shipdate").is_between(var1, var2, closed="left"))
.with_columns(
(nw.col("l_extendedprice") * (1 - nw.col("l_discount"))).alias(
"total_revenue"
)
)
.group_by("l_suppkey")
.agg(nw.sum("total_revenue"))
.select(nw.col("l_suppkey").alias("supplier_no"), nw.col("total_revenue"))
)

return (
supplier.join(revenue, left_on="s_suppkey", right_on="supplier_no")
.filter(nw.col("total_revenue") == nw.col("total_revenue").max())
.with_columns(nw.col("total_revenue").round(2))
.select("s_suppkey", "s_name", "s_address", "s_phone", "total_revenue")
.sort("s_suppkey")
.collect()
)
46 changes: 46 additions & 0 deletions tpch/benchmarks/q17_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
from __future__ import annotations

from pathlib import Path
from typing import TYPE_CHECKING

import pytest

import narwhals.stable.v1 as nw
from tpch.benchmarks.utils import lib_to_reader

if TYPE_CHECKING:
from pytest_codspeed.plugin import BenchmarkFixture

DATA_FOLDER = Path("tests/data")


@pytest.mark.parametrize("library", ["pandas", "polars", "pyarrow", "dask"])
def test_q17(benchmark: BenchmarkFixture, library: str) -> None:
read_fn = lib_to_reader[library]

lineitem = nw.from_native(read_fn(DATA_FOLDER / "lineitem.parquet")).lazy()
part = nw.from_native(read_fn(DATA_FOLDER / "part.parquet")).lazy()

_ = benchmark(q17, lineitem, part)


def q17(lineitem: nw.LazyFrame, part: nw.LazyFrame) -> nw.DataFrame:
var1 = "Brand#23"
var2 = "MED BOX"

query1 = (
part.filter(nw.col("p_brand") == var1)
.filter(nw.col("p_container") == var2)
.join(lineitem, how="left", left_on="p_partkey", right_on="l_partkey")
)

return (
query1.with_columns(avg_quantity=0.2 * nw.col("l_quantity"))
.group_by("p_partkey")
.agg(nw.col("avg_quantity").mean())
.select(nw.col("p_partkey").alias("key"), nw.col("avg_quantity"))
.join(query1, left_on="key", right_on="p_partkey")
.filter(nw.col("l_quantity") < nw.col("avg_quantity"))
.select((nw.col("l_extendedprice").sum() / 7.0).round(2).alias("avg_yearly"))
.collect()
)
56 changes: 56 additions & 0 deletions tpch/benchmarks/q18_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
from __future__ import annotations

from pathlib import Path
from typing import TYPE_CHECKING

import pytest

import narwhals.stable.v1 as nw
from tpch.benchmarks.utils import lib_to_reader

if TYPE_CHECKING:
from pytest_codspeed.plugin import BenchmarkFixture

DATA_FOLDER = Path("tests/data")


@pytest.mark.parametrize("library", ["pandas", "polars", "pyarrow", "dask"])
def test_q18(benchmark: BenchmarkFixture, library: str) -> None:
read_fn = lib_to_reader[library]

customer = nw.from_native(read_fn(DATA_FOLDER / "customer.parquet")).lazy()
lineitem = nw.from_native(read_fn(DATA_FOLDER / "lineitem.parquet")).lazy()
orders = nw.from_native(read_fn(DATA_FOLDER / "orders.parquet")).lazy()

_ = benchmark(q18, customer, lineitem, orders)


def q18(
customer: nw.LazyFrame, lineitem: nw.LazyFrame, orders: nw.LazyFrame
) -> nw.DataFrame:
var1 = 300

query1 = (
lineitem.group_by("l_orderkey")
.agg(nw.col("l_quantity").sum().alias("sum_quantity"))
.filter(nw.col("sum_quantity") > var1)
)

return (
orders.join(query1, left_on="o_orderkey", right_on="l_orderkey", how="semi")
.join(lineitem, left_on="o_orderkey", right_on="l_orderkey")
.join(customer, left_on="o_custkey", right_on="c_custkey")
.group_by("c_name", "o_custkey", "o_orderkey", "o_orderdate", "o_totalprice")
.agg(nw.col("l_quantity").sum().alias("col6"))
.select(
nw.col("c_name"),
nw.col("o_custkey").alias("c_custkey"),
nw.col("o_orderkey"),
nw.col("o_orderdate").alias("o_orderdat"),
nw.col("o_totalprice"),
nw.col("col6"),
)
.sort(by=["o_totalprice", "o_orderdat"], descending=[True, False])
.head(100)
.collect()
)
61 changes: 61 additions & 0 deletions tpch/benchmarks/q19_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
from __future__ import annotations

from pathlib import Path
from typing import TYPE_CHECKING

import pytest

import narwhals.stable.v1 as nw
from tpch.benchmarks.utils import lib_to_reader

if TYPE_CHECKING:
from pytest_codspeed.plugin import BenchmarkFixture

DATA_FOLDER = Path("tests/data")


@pytest.mark.parametrize("library", ["pandas", "polars", "pyarrow", "dask"])
def test_q19(benchmark: BenchmarkFixture, library: str) -> None:
read_fn = lib_to_reader[library]

lineitem = nw.from_native(read_fn(DATA_FOLDER / "lineitem.parquet")).lazy()
part = nw.from_native(read_fn(DATA_FOLDER / "part.parquet")).lazy()

_ = benchmark(q19, lineitem, part)


def q19(lineitem: nw.LazyFrame, part: nw.LazyFrame) -> nw.DataFrame:
return (
part.join(lineitem, left_on="p_partkey", right_on="l_partkey")
.filter(nw.col("l_shipmode").is_in(["AIR", "AIR REG"]))
.filter(nw.col("l_shipinstruct") == "DELIVER IN PERSON")
.filter(
(
(nw.col("p_brand") == "Brand#12")
& nw.col("p_container").is_in(["SM CASE", "SM BOX", "SM PACK", "SM PKG"])
& (nw.col("l_quantity").is_between(1, 11))
& (nw.col("p_size").is_between(1, 5))
)
| (
(nw.col("p_brand") == "Brand#23")
& nw.col("p_container").is_in(
["MED BAG", "MED BOX", "MED PKG", "MED PACK"]
)
& (nw.col("l_quantity").is_between(10, 20))
& (nw.col("p_size").is_between(1, 10))
)
| (
(nw.col("p_brand") == "Brand#34")
& nw.col("p_container").is_in(["LG CASE", "LG BOX", "LG PACK", "LG PKG"])
& (nw.col("l_quantity").is_between(20, 30))
& (nw.col("p_size").is_between(1, 15))
)
)
.select(
(nw.col("l_extendedprice") * (1 - nw.col("l_discount")))
.sum()
.round(2)
.alias("revenue")
)
.collect()
)
Loading
Loading