Run Unit Tests for Different Parameters

thicket/tests/conftest.py

@@ -12,19 +12,35 @@
 from thicket import Thicket
 
 
+@pytest.fixture(params=[True, False], ids=["FillPerfdata", "NoFillPerfdata"])
+def fill_perfdata(request):
+    return request.param
+
+
+@pytest.fixture(params=[True, False], ids=["Intersection", "Union"])
+def intersection(request):
+    return request.param
+
+
 @pytest.fixture
-def thicket_axis_columns(rajaperf_cali_1trial):
+def thicket_axis_columns(rajaperf_cali_1trial, intersection, fill_perfdata):
     """Generator for 'concat_thickets(axis="columns")' thicket.
 
     Arguments:
-        mpi_scaling_cali (list): List of Caliper files for MPI scaling study.
-        rajaperf_cuda_block128_1M_cali (list): List of Caliper files for base cuda variant.
+        rajaperf_cali_1trial (list): All tunings and variants for the first trial.
+        intersection (bool): Whether to use intersection or union for calltree.
+        fill_perfdata (bool): Whether to fill perfdata or not.
 
     Returns:
         list: List of original thickets, list of deepcopies of original thickets, and
             column-joined thicket.
     """
-    tk = Thicket.from_caliperreader(rajaperf_cali_1trial, disable_tqdm=True)
+    tk = Thicket.from_caliperreader(
+        rajaperf_cali_1trial,
+        intersection=intersection,
+        fill_perfdata=fill_perfdata,
+        disable_tqdm=True,
+    )
 
     gb = tk.groupby("tuning")
 
@@ -36,6 +52,7 @@ def thicket_axis_columns(rajaperf_cali_1trial):
     combined_th = Thicket.concat_thickets(
         thickets=thickets,
         axis="columns",
+        calltree="intersection" if intersection else "union",
         headers=headers,
         metadata_key="ProblemSizeRunParam",
         disable_tqdm=True,
@@ -45,21 +62,31 @@ def thicket_axis_columns(rajaperf_cali_1trial):
 
 
 @pytest.fixture
-def stats_thicket_axis_columns(rajaperf_cuda_block128_1M_cali):
+def stats_thicket_axis_columns(
+    rajaperf_cuda_block128_1M_cali, intersection, fill_perfdata
+):
     """Generator for 'concat_thickets(axis="columns")' thicket for test_stats.py.
 
     Arguments:
         rajaperf_cuda_block128_1M_cali (list): List of Caliper files for base cuda variant.
+        intersection (bool): Whether to use intersection or union for calltree.
+        fill_perfdata (bool): Whether to fill perfdata or not.
 
     Returns:
         list: List of original thickets, list of deepcopies of original thickets, and
             column-joined thicket.
     """
     th_cuda128_1 = Thicket.from_caliperreader(
-        rajaperf_cuda_block128_1M_cali[0:4], disable_tqdm=True
+        rajaperf_cuda_block128_1M_cali[0:4],
+        intersection=intersection,
+        fill_perfdata=fill_perfdata,
+        disable_tqdm=True,
     )
     th_cuda128_2 = Thicket.from_caliperreader(
-        rajaperf_cuda_block128_1M_cali[5:9], disable_tqdm=True
+        rajaperf_cuda_block128_1M_cali[5:9],
+        intersection=intersection,
+        fill_perfdata=fill_perfdata,
+        disable_tqdm=True,
     )
 
     # To check later if modifications were unexpectedly made
@@ -71,6 +98,7 @@ def stats_thicket_axis_columns(rajaperf_cuda_block128_1M_cali):
     combined_th = Thicket.concat_thickets(
         thickets=thickets,
         axis="columns",
+        calltree="intersection" if intersection else "union",
         headers=["Cuda 1", "Cuda 2"],
         disable_tqdm=True,
     )

thicket/tests/test_caliperreader.py

@@ -6,9 +6,14 @@
 from thicket import Thicket
 
 
-def test_from_caliperreader(rajaperf_seq_O3_1M_cali):
+def test_from_caliperreader(rajaperf_seq_O3_1M_cali, intersection, fill_perfdata):
     """Sanity test a thicket object with known data."""
-    tk = Thicket.from_caliperreader(rajaperf_seq_O3_1M_cali[0], disable_tqdm=True)
+    tk = Thicket.from_caliperreader(
+        rajaperf_seq_O3_1M_cali[0],
+        intersection=intersection,
+        fill_perfdata=fill_perfdata,
+        disable_tqdm=True,
+    )
 
     # Check the object type
     assert isinstance(tk, Thicket)
@@ -25,10 +30,15 @@ def test_from_caliperreader(rajaperf_seq_O3_1M_cali):
     )
 
 
-def test_node_ordering_from_caliper(caliper_ordered):
+def test_node_ordering_from_caliper(caliper_ordered, intersection, fill_perfdata):
     """Check the order of output from the native Caliper reader by examining a known input with node order column."""
 
-    tk = Thicket.from_caliperreader(caliper_ordered)
+    tk = Thicket.from_caliperreader(
+        caliper_ordered,
+        intersection=intersection,
+        fill_perfdata=fill_perfdata,
+        disable_tqdm=True,
+    )
 
     expected_order = [
         "main",

thicket/tests/test_concat_thickets.py

@@ -17,14 +17,30 @@
 from thicket.utils import DuplicateIndexError
 
 
-def test_concat_thickets_index(mpi_scaling_cali):
-    th_27 = Thicket.from_caliperreader(mpi_scaling_cali[0], disable_tqdm=True)
-    th_64 = Thicket.from_caliperreader(mpi_scaling_cali[1], disable_tqdm=True)
-
-    tk = Thicket.concat_thickets([th_27, th_64], disable_tqdm=True)
+def test_concat_thickets_index(mpi_scaling_cali, intersection, fill_perfdata):
+    th_27 = Thicket.from_caliperreader(
+        mpi_scaling_cali[0],
+        intersection=intersection,
+        fill_perfdata=fill_perfdata,
+        disable_tqdm=True,
+    )
+    th_64 = Thicket.from_caliperreader(
+        mpi_scaling_cali[1],
+        intersection=intersection,
+        fill_perfdata=fill_perfdata,
+        disable_tqdm=True,
+    )
 
-    # Check dataframe shape
-    assert tk.dataframe.shape == (90, 7)
+    if intersection:
+        calltree = "intersection"
+    else:
+        calltree = "union"
+    tk = Thicket.concat_thickets(
+        [th_27, th_64],
+        calltree=calltree,
+        fill_perfdata=fill_perfdata,
+        disable_tqdm=True,
+    )
 
     # Check specific values. Row order can vary so use "sum" to check
     node = tk.dataframe.index.get_level_values("node")[8]
@@ -34,7 +50,12 @@ def test_concat_thickets_index(mpi_scaling_cali):
     with pytest.raises(
         DuplicateIndexError,
     ):
-        Thicket.from_caliperreader([mpi_scaling_cali[0], mpi_scaling_cali[0]])
+        Thicket.from_caliperreader(
+            [mpi_scaling_cali[0], mpi_scaling_cali[0]],
+            intersection=intersection,
+            fill_perfdata=fill_perfdata,
+            disable_tqdm=True,
+        )
 
 
 def test_concat_thickets_columns(thicket_axis_columns):
@@ -90,38 +111,56 @@ def test_filter_concat_thickets_columns(thicket_axis_columns):
     filter_multiple_and(combined_th, columns_values)
 
 
-def test_filter_stats_concat_thickets_columns(thicket_axis_columns):
+def test_filter_stats_concat_thickets_columns(thicket_axis_columns, intersection):
     thickets, thickets_cp, combined_th = thicket_axis_columns
-    # columns and corresponding values to filter by
+
     columns_values = {
         ("test", "test_string_column"): ["less than 20"],
         ("test", "test_numeric_column"): [4, 15],
     }
     # set string column values
     less_than_20 = ["less than 20"] * 21
     less_than_45 = ["less than 45"] * 25
-    less_than_178 = ["less than 75"] * 28
-    new_col = less_than_20 + less_than_45 + less_than_178
-    combined_th.statsframe.dataframe[("test", "test_string_column")] = new_col
-    # set numeric column values
-    combined_th.statsframe.dataframe[("test", "test_numeric_column")] = range(0, 74)
+
+    if intersection:
+        less_than_65 = ["less than 65"] * 18
+        new_col = less_than_20 + less_than_45 + less_than_65
+        combined_th.statsframe.dataframe[("test", "test_string_column")] = new_col
+        # set numeric column values
+        combined_th.statsframe.dataframe[("test", "test_numeric_column")] = range(0, 64)
+    else:
+        less_than_75 = ["less than 75"] * 28
+        new_col = less_than_20 + less_than_45 + less_than_75
+        combined_th.statsframe.dataframe[("test", "test_string_column")] = new_col
+        # set numeric column values
+        combined_th.statsframe.dataframe[("test", "test_numeric_column")] = range(0, 74)
 
     check_filter_stats(combined_th, columns_values)
 
 
-def test_query_concat_thickets_columns(thicket_axis_columns):
+def test_query_concat_thickets_columns(thicket_axis_columns, intersection):
     thickets, thickets_cp, combined_th = thicket_axis_columns
     # test arguments
-    hnids = [
-        0,
-        1,
-        2,
-        3,
-        4,
-        5,
-        6,
-        7,
-    ]  # "0" because top-level node "RAJAPerf" will be included in query result.
+    if intersection:
+        # Shorter graph for intersection
+        hnids = [
+            0,
+            1,
+            2,
+            3,
+            4,
+        ]
+    else:
+        hnids = [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+        ]  # "0" because top-level node "RAJAPerf" will be included in query result.
     query = (
         ht.QueryMatcher()
         .match("*")

thicket/tests/test_copy.py

@@ -6,8 +6,13 @@
 from thicket import Thicket
 
 
-def test_copy(rajaperf_seq_O3_1M_cali):
-    self = Thicket.from_caliperreader(rajaperf_seq_O3_1M_cali[0], disable_tqdm=True)
+def test_copy(rajaperf_seq_O3_1M_cali, intersection, fill_perfdata):
+    self = Thicket.from_caliperreader(
+        rajaperf_seq_O3_1M_cali[0],
+        intersection=intersection,
+        fill_perfdata=fill_perfdata,
+        disable_tqdm=True,
+    )
     self.exc_metrics.append("value")
     other = self.copy()
 
@@ -67,8 +72,13 @@ def test_copy(rajaperf_seq_O3_1M_cali):
     )
 
 
-def test_deepcopy(rajaperf_seq_O3_1M_cali):
-    self = Thicket.from_caliperreader(rajaperf_seq_O3_1M_cali[0], disable_tqdm=True)
+def test_deepcopy(rajaperf_seq_O3_1M_cali, intersection, fill_perfdata):
+    self = Thicket.from_caliperreader(
+        rajaperf_seq_O3_1M_cali[0],
+        intersection=intersection,
+        fill_perfdata=fill_perfdata,
+        disable_tqdm=True,
+    )
     self.exc_metrics.append("value")
     other = self.deepcopy()
 

thicket/tests/test_display.py

@@ -10,8 +10,13 @@
 import thicket as th
 
 
-def test_display_histogram(rajaperf_seq_O3_1M_cali):
-    tk = th.Thicket.from_caliperreader(rajaperf_seq_O3_1M_cali, disable_tqdm=True)
+def test_display_histogram(rajaperf_seq_O3_1M_cali, intersection, fill_perfdata):
+    tk = th.Thicket.from_caliperreader(
+        rajaperf_seq_O3_1M_cali,
+        intersection=intersection,
+        fill_perfdata=fill_perfdata,
+        disable_tqdm=True,
+    )
 
     node = pd.unique(tk.dataframe.reset_index()["node"])[4]
 
@@ -78,8 +83,13 @@ def test_display_histogram_columnar_join(thicket_axis_columns):
     plt.close()
 
 
-def test_display_heatmap(rajaperf_seq_O3_1M_cali):
-    tk = th.Thicket.from_caliperreader(rajaperf_seq_O3_1M_cali, disable_tqdm=True)
+def test_display_heatmap(rajaperf_seq_O3_1M_cali, intersection, fill_perfdata):
+    tk = th.Thicket.from_caliperreader(
+        rajaperf_seq_O3_1M_cali,
+        intersection=intersection,
+        fill_perfdata=fill_perfdata,
+        disable_tqdm=True,
+    )
 
     th.stats.variance(tk, columns=["Min time/rank"])
 
@@ -151,8 +161,13 @@ def test_display_heatmap_columnar_join(thicket_axis_columns):
     plt.close()
 
 
-def test_display_boxplot(rajaperf_seq_O3_1M_cali):
-    tk = th.Thicket.from_caliperreader(rajaperf_seq_O3_1M_cali, disable_tqdm=True)
+def test_display_boxplot(rajaperf_seq_O3_1M_cali, intersection, fill_perfdata):
+    tk = th.Thicket.from_caliperreader(
+        rajaperf_seq_O3_1M_cali,
+        intersection=intersection,
+        fill_perfdata=fill_perfdata,
+        disable_tqdm=True,
+    )
 
     nodes = list(pd.unique(tk.dataframe.reset_index()["node"])[0:2])
 

thicket/tests/test_filter_metadata.py

@@ -163,8 +163,13 @@ def filter_multiple_or(th, columns_values):
     assert "name" in new_th.statsframe.dataframe.columns
 
 
-def test_check_errors(rajaperf_seq_O3_1M_cali):
-    th = Thicket.from_caliperreader(rajaperf_seq_O3_1M_cali, disable_tqdm=True)
+def test_check_errors(rajaperf_seq_O3_1M_cali, intersection, fill_perfdata):
+    th = Thicket.from_caliperreader(
+        rajaperf_seq_O3_1M_cali,
+        intersection=intersection,
+        fill_perfdata=fill_perfdata,
+        disable_tqdm=True,
+    )
 
     # check for invalid filter exception
     with pytest.raises(InvalidFilter):
@@ -192,9 +197,14 @@ def test_check_errors(rajaperf_seq_O3_1M_cali):
         th.filter_metadata(lambda x: x["cluster"] == "quartz")
 
 
-def test_filter_metadata(rajaperf_seq_O3_1M_cali):
+def test_filter_metadata(rajaperf_seq_O3_1M_cali, intersection, fill_perfdata):
     # example thicket
-    th = Thicket.from_caliperreader(rajaperf_seq_O3_1M_cali, disable_tqdm=True)
+    th = Thicket.from_caliperreader(
+        rajaperf_seq_O3_1M_cali,
+        intersection=intersection,
+        fill_perfdata=fill_perfdata,
+        disable_tqdm=True,
+    )
     # columns and corresponding values to filter by
     columns_values = {"ProblemSizeRunParam": [1048576.0], "cluster": ["quartz"]}
     filter_one_column(th, columns_values)

thicket/tests/test_filter_stats.py

@@ -54,9 +54,14 @@ def check_filter_stats(th, columns_values):
             assert ensemble_nodes == exp_nodes
 
 
-def test_filter_stats(rajaperf_seq_O3_1M_cali):
+def test_filter_stats(rajaperf_seq_O3_1M_cali, intersection, fill_perfdata):
     # example thicket
-    th = Thicket.from_caliperreader(rajaperf_seq_O3_1M_cali, disable_tqdm=True)
+    th = Thicket.from_caliperreader(
+        rajaperf_seq_O3_1M_cali,
+        intersection=intersection,
+        fill_perfdata=fill_perfdata,
+        disable_tqdm=True,
+    )
     # columns and corresponding values to filter by
     columns_values = {
         "test_string_column": ["less than 20"],