Switch to CTest (pyMBE-dev#87)

* Modernize test cases

* Manage testsuite with CTest

* use a test driver to run test cases in parallel
* configure test cases with labels and arguments
* only show test case output on test failure
* allow skipped test cases (error code 5)

* Use Python multiprocessing

.github/workflows/samples.yml

@@ -31,5 +31,5 @@ jobs:
         run: |
           module restore pymbe
           source venv/bin/activate
-          make functional_tests
+          make functional_tests -j $(nproc)
         shell: bash

.github/workflows/testsuite.yml

@@ -37,7 +37,7 @@ jobs:
           module restore pymbe
           source venv/bin/activate
           make pylint
-          make unit_tests COVERAGE=1
+          make unit_tests -j $(nproc) COVERAGE=1
           make docs
           make coverage_xml
         shell: bash

.gitignore

@@ -6,3 +6,4 @@ frames
 __pycache__
 traj*.vtf
 *_system.png
+testsuite/Testing/

Makefile

@@ -29,52 +29,30 @@ COVERAGE_HTML = coverage
 
 # Python executable or launcher, possibly with command line arguments
 PYTHON = python3
-ifeq ($(COVERAGE),1)
-PYTHON := ${PYTHON} -m coverage run --parallel-mode --source=$(CURDIR)
-endif
+
+# number of threads
+THREADS = $(shell echo $(MAKEFLAGS) | grep -oP "\\-j *\\d+")
 
 docs:
 	mkdir -p ./documentation
 	PDOC_ALLOW_EXEC=0 ${PYTHON} -m pdoc ./pyMBE.py -o ./documentation --docformat google
 
 unit_tests:
-	${PYTHON} testsuite/serialization_test.py
-	${PYTHON} testsuite/lj_tests.py
-	${PYTHON} testsuite/set_particle_acidity_test.py
-	${PYTHON} testsuite/bond_tests.py
-	${PYTHON} testsuite/generate_perpendicular_vectors_test.py
-	${PYTHON} testsuite/define_and_create_molecules_unit_tests.py
-	${PYTHON} testsuite/create_molecule_position_test.py
-	${PYTHON} testsuite/seed_test.py
-	${PYTHON} testsuite/read-write-df_test.py
-	${PYTHON} testsuite/parameter_test.py
-	${PYTHON} testsuite/henderson_hasselbalch_tests.py
-	${PYTHON} testsuite/calculate_net_charge_unit_test.py
-	${PYTHON} testsuite/setup_salt_ions_unit_tests.py
-	${PYTHON} testsuite/globular_protein_unit_tests.py
-	${PYTHON} testsuite/analysis_tests.py
-	${PYTHON} testsuite/charge_number_map_tests.py
-	${PYTHON} testsuite/generate_coordinates_tests.py
-	${PYTHON} testsuite/reaction_methods_unit_tests.py
-	${PYTHON} testsuite/determine_reservoir_concentrations_unit_test.py
+	COVERAGE=$(COVERAGE) ctest --output-on-failure $(THREADS) --test-dir testsuite -LE long --timeout 300
 
 functional_tests:
-	${PYTHON} testsuite/cph_ideal_tests.py
-	${PYTHON} testsuite/grxmc_ideal_tests.py
-	${PYTHON} testsuite/peptide_tests.py
-	${PYTHON} testsuite/gcmc_tests.py
-	${PYTHON} testsuite/weak_polyelectrolyte_dialysis_test.py
-	${PYTHON} testsuite/globular_protein_tests.py
+	COVERAGE=$(COVERAGE) ctest --output-on-failure $(THREADS) --test-dir testsuite -L long
 
-tests: unit_tests functional_tests
+tests:
+	COVERAGE=$(COVERAGE) ctest --output-on-failure $(THREADS) --test-dir testsuite
 
 coverage_xml:
-	${PYTHON} -m coverage combine .
+	${PYTHON} -m coverage combine testsuite
 	${PYTHON} -m coverage report
 	${PYTHON} -m coverage xml
 
 coverage_html:
-	${PYTHON} -m coverage combine .
+	${PYTHON} -m coverage combine testsuite
 	${PYTHON} -m coverage html --directory="${COVERAGE_HTML}"
 
 sample:

README.md

@@ -20,6 +20,7 @@ pyMBE provides tools to facilitate building up molecules with complex architectu
 - [Numpy](https://numpy.org/) >=1.23
 - [SciPy](https://scipy.org/) 
 - [pdoc](https://pdoc.dev/) (for building the docs)
+- [CMake](https://cmake.org/) (for running the testsuite)
 
 ## Contents
 
@@ -153,12 +154,18 @@ To make sure your code is valid, please run the testsuite before submitting your
 
 ```sh
 source pymbe/bin/activate
-make tests
+make tests -j4
 deactivate
 ```
 
+Here, `-j4` instructs CTest to run the test cases in parallel using 4 CPU cores.
+This number can be adjusted depending on your hardware specifications.
+You can use `make unit_tests -j4` to run the subset of fast tests, but keep in mind those
+won't be able to detect more serious bugs that only manifest themselves in long simulations.
+You can also run individual test cases directly, for example with `python3 testsuite/parameter_test.py`.
+
 When contributing new features, consider adding a unit test in the `testsuite/`
-folder and a corresponding line in the `testsuite` target of the Makefile.
+folder and a corresponding line in the `testsuite/CTestTestfile.cmake` file.
 
 Every contribution is automatically tested in CI using EESSI (https://www.eessi.io)
 and the [EESSI GitHub Action](https://github.com/marketplace/actions/eessi).

requirements.txt

@@ -5,4 +5,7 @@ pint-pandas>=0.3
 biopandas==0.5.1.dev0
 scipy>=1.8.0
 matplotlib>=3.5.1
+# soft dependencies to run the samples
 tqdm>=4.57.0
+# soft dependencies to run the testsuite
+cmake>=3.22.1 # for CTest

testsuite/CTestTestfile.cmake

@@ -0,0 +1,68 @@
+#
+# Copyright (C) 2024 pyMBE-dev team
+#
+# This file is part of pyMBE.
+#
+# pyMBE is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyMBE is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+cmake_minimum_required(VERSION 3.22.1)
+find_package(Python 3.8 REQUIRED COMPONENTS Interpreter NumPy)
+
+file(REAL_PATH "CTestTestfile.cmake" CMAKE_CURRENT_SOURCE_FILE)
+cmake_path(GET CMAKE_CURRENT_SOURCE_FILE PARENT_PATH CMAKE_CURRENT_SOURCE_DIR)
+cmake_path(GET CMAKE_CURRENT_SOURCE_DIR PARENT_PATH CMAKE_SOURCE_DIR)
+
+function(pymbe_add_test)
+  cmake_parse_arguments(TEST "" "PATH;THREADS" "LABELS" ${ARGN})
+  cmake_path(GET TEST_PATH STEM TEST_NAME)
+  if(DEFINED ENV{COVERAGE} AND "$ENV{COVERAGE}" STREQUAL "1")
+    list(APPEND PYTHON_ARGUMENTS "-m" "coverage" "run" "--parallel-mode" "--source=${CMAKE_SOURCE_DIR}")
+  endif()
+  add_test(${TEST_NAME} "${Python_EXECUTABLE}" ${PYTHON_ARGUMENTS} "${TEST_PATH}")
+  set_tests_properties(${TEST_NAME} PROPERTIES SKIP_RETURN_CODE 5)
+  set_tests_properties(${TEST_NAME} PROPERTIES LABELS ${TEST_LABELS})
+  if(DEFINED TEST_THREADS)
+    set_tests_properties(${TEST_NAME} PROPERTIES PROCESSORS ${TEST_THREADS})
+  endif()
+endfunction()
+
+# functional tests, e.g. long simulations and ensemble averages
+pymbe_add_test(PATH globular_protein_tests.py LABELS long beyer2024 THREADS 2)
+pymbe_add_test(PATH peptide_tests.py LABELS long beyer2024 THREADS 2)
+pymbe_add_test(PATH weak_polyelectrolyte_dialysis_test.py LABELS long beyer2024)
+pymbe_add_test(PATH cph_ideal_tests.py LABELS long)
+pymbe_add_test(PATH grxmc_ideal_tests.py LABELS long)
+pymbe_add_test(PATH gcmc_tests.py LABELS long)
+
+# unit tests
+pymbe_add_test(PATH serialization_test.py)
+pymbe_add_test(PATH lj_tests.py)
+pymbe_add_test(PATH set_particle_acidity_test.py)
+pymbe_add_test(PATH bond_tests.py)
+pymbe_add_test(PATH generate_perpendicular_vectors_test.py)
+pymbe_add_test(PATH define_and_create_molecules_unit_tests.py)
+pymbe_add_test(PATH create_molecule_position_test.py)
+pymbe_add_test(PATH seed_test.py)
+pymbe_add_test(PATH read-write-df_test.py)
+pymbe_add_test(PATH parameter_test.py)
+pymbe_add_test(PATH henderson_hasselbalch_tests.py)
+pymbe_add_test(PATH calculate_net_charge_unit_test.py)
+pymbe_add_test(PATH setup_salt_ions_unit_tests.py)
+pymbe_add_test(PATH globular_protein_unit_tests.py)
+pymbe_add_test(PATH analysis_tests.py)
+pymbe_add_test(PATH charge_number_map_tests.py)
+pymbe_add_test(PATH generate_coordinates_tests.py)
+pymbe_add_test(PATH reaction_methods_unit_tests.py)
+pymbe_add_test(PATH determine_reservoir_concentrations_unit_test.py)

testsuite/analysis_tests.py

@@ -19,24 +19,23 @@
 import unittest as ut
 import pandas as pd
 import lib.analysis as ana
+import pathlib
 
 
 class Serialization(ut.TestCase):
+    data_root = pathlib.Path(__file__).parent.resolve() / "tests_data"
 
     def test_analyze_time_series(self):
         print("*** Unit test: test that analysis.analyze_time_series analyzes all data in a folder correctly ***")
-        analyzed_data = ana.analyze_time_series(path_to_datafolder="testsuite/tests_data",
+        analyzed_data = ana.analyze_time_series(path_to_datafolder=self.data_root,
                                        filename_extension="_time_series.csv",
                                        minus_separator=True)
         analyzed_data[["Dens","eps"]] = analyzed_data[["Dens","eps"]].apply(pd.to_numeric)
-        reference_data = pd.read_csv("testsuite/tests_data/average_data.csv", header=[0,1])
+        reference_data = pd.read_csv(self.data_root / "average_data.csv", header=[0,1])
         analyzed_data.columns = analyzed_data.sort_index(axis=1,level=[0,1],ascending=[True,True]).columns
         reference_data.columns = reference_data.sort_index(axis=1,level=[0,1],ascending=[True,True]).columns
         pd.testing.assert_frame_equal(analyzed_data.dropna(),reference_data.dropna(), check_column_type=False, check_dtype=False)
         print("*** Unit passed ***")
-
-        return
-
 
     def test_get_dt(self):
         print("*** Unit test: test that analysis.get_dt returns the right time step ***")
@@ -123,12 +122,12 @@ def test_get_params_from_file_name(self):
 
     def test_block_analyze(self):
         print("*** Unit test: test that block_analyze yields the expected outputs and reports the number of blocks and the block size. It should print that it encountered 1 repeated time value. ***")
-        data = pd.read_csv("testsuite/tests_data/N-064_Solvent-good_Init-coil_time_series.csv")
+        data = pd.read_csv(self.data_root / "N-064_Solvent-good_Init-coil_time_series.csv")
         analyzed_data = ana.block_analyze(full_data=data, verbose=True)
         analyzed_data = ana.add_data_to_df(df=pd.DataFrame(),
                             data_dict=analyzed_data.to_dict(),
                             index=[0])
-        reference_data = pd.read_csv("testsuite/tests_data/N-064_Solvent-good_Init-coil_time_series_analyzed.csv", header=[0,1])
+        reference_data = pd.read_csv(self.data_root / "N-064_Solvent-good_Init-coil_time_series_analyzed.csv", header=[0,1])
         pd.testing.assert_frame_equal(analyzed_data.dropna(),reference_data.dropna(), check_column_type=False)
         print("*** Unit passed ***")
 
@@ -137,7 +136,7 @@ def test_block_analyze(self):
         analyzed_data = ana.add_data_to_df(df=pd.DataFrame(),
                             data_dict=analyzed_data.to_dict(),
                             index=[0])
-        reference_data = pd.read_csv("testsuite/tests_data/N-064_Solvent-good_Init-coil_time_series_analyzed.csv", header=[0,1])
+        reference_data = pd.read_csv(self.data_root / "N-064_Solvent-good_Init-coil_time_series_analyzed.csv", header=[0,1])
         reference_data = reference_data[[("mean","Rg"),("err_mean","Rg"),("n_eff","Rg"),("tau_int","Rg")]]
         pd.testing.assert_frame_equal(analyzed_data.dropna(),reference_data.dropna(), check_column_type=False)
         print("*** Unit passed ***")

testsuite/globular_protein_tests.py

@@ -16,30 +16,32 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# Import pyMBE and other libraries
-import pyMBE
 from lib import analysis
 import sys
+import pathlib
 import tempfile
 import subprocess
+import multiprocessing
 import numpy as np
 import pandas as pd
+import unittest as ut
 
-# Template of the test
 
-def run_protein_test(script_path, test_pH_values, protein_pdb, rtol, atol,mode="test"):
+root = pathlib.Path(__file__).parent.parent.resolve()
+data_root = root / "testsuite" / "globular_protein_tests_data"
+script_path = root / "samples" / "Beyer2024" / "globular_protein.py"
+test_pH_values = [2, 5, 7]
+tasks = ["1beb", "1f6s"]
+mode = "test"
+
+
+def kernel(protein_pdb):
     """
     Runs a set of tests for a given protein pdb.
 
     Args:
-        script_path(`str`): Path to the script to run the test.
-        test_pH_values(`lst`): List of pH values to be tested.
         protein_pdb(`str`): PDB code of the protein.
     """
-    valid_modes=["test","save"]
-    assert mode in valid_modes, f"Mode {mode} not supported, valid modes: {valid_modes}"
-
-    print(f"Running tests for {protein_pdb}")
     with tempfile.TemporaryDirectory() as time_series_path:
         for pH in test_pH_values:
             print(f"pH = {pH}")
@@ -51,47 +53,31 @@ def run_protein_test(script_path, test_pH_values, protein_pdb, rtol, atol,mode="
         # Analyze all time series
         data=analysis.analyze_time_series(path_to_datafolder=time_series_path,
                                           filename_extension="_time_series.csv")
+    return (protein_pdb, data)
 
-        data_path=pmb.get_resource(path="testsuite/globular_protein_tests_data")
-
-    if mode == "test":
-        # Get reference test data
-        ref_data=pd.read_csv(f"{data_path}/{protein_pdb}.csv", header=[0, 1])
-        # Check charge
-        test_charge=np.sort(data["mean","charge"].to_numpy())
-        ref_charge=np.sort(ref_data["mean","charge"].to_numpy())
-        np.testing.assert_allclose(test_charge, ref_charge, rtol=rtol, atol=atol)
-        print(f"Test for {protein_pdb} was successful")
-
-    elif mode == "save":
-        # Save data for future testing
-        data.to_csv(f"{data_path}/{protein_pdb}.csv", index=False)
-    else:
-        raise RuntimeError
-
-# Create an instance of pyMBE library
-pmb = pyMBE.pymbe_library(seed=42)
-
-script_path=pmb.get_resource("samples/Beyer2024/globular_protein.py")
-test_pH_values=[2,5,7]
-rtol=0.1 # relative tolerance
-atol=0.5 # absolute tolerance
-
-# Run test for 1BEB case
-protein_pdb = "1beb"
-run_protein_test(script_path=script_path,
-                    test_pH_values=test_pH_values,
-                    protein_pdb=protein_pdb,
-                    rtol=rtol,
-                    atol=atol)
 
-# Run test for 1F6S case
-protein_pdb = "1f6s"
-run_protein_test(script_path=script_path,
-                    test_pH_values=test_pH_values,
-                    protein_pdb=protein_pdb,
-                    rtol=rtol,
-                    atol=atol)   
+class Test(ut.TestCase):
 
+    def test_globular_protein(self):
+        with multiprocessing.Pool(processes=2) as pool:
+            results = dict(pool.map(kernel, tasks, chunksize=1))
 
+        rtol=0.1 # relative tolerance
+        atol=0.5 # absolute tolerance
+        for protein_pdb, data in results.items():
+            # Save data for future testing
+            if mode == "save":
+                data.to_csv(data_root / f"{protein_pdb}.csv", index=False)
+                continue
+            assert mode == "test", f"Mode {mode} not supported, valid modes: ['save', 'test']"
+            with self.subTest(msg=f"Protein {protein_pdb}"):
+                # Get reference test data
+                ref_data=pd.read_csv(data_root / f"{protein_pdb}.csv", header=[0, 1])
+                # Check charge
+                test_charge=np.sort(data["mean","charge"].to_numpy())
+                ref_charge=np.sort(ref_data["mean","charge"].to_numpy())
+                np.testing.assert_allclose(
+                    test_charge, ref_charge, rtol=rtol, atol=atol)
 
+if __name__ == "__main__":
+    ut.main()