benchmark for 5 subjects

braindatalab · Jul 25, 2023 · 5a44072 · 5a44072
1 parent 25a5672
commit 5a44072
Showing 1 changed file with 139 additions and 140 deletions.
diff --git a/bsi_zoo/run_benchmark.py b/bsi_zoo/run_benchmark.py
@@ -18,12 +18,10 @@
 from bsi_zoo.metrics import euclidean_distance, mse, emd, f1, reconstructed_noise
 from bsi_zoo.config import get_leadfield_path
 
-n_jobs = 1
-nruns = 1
-do_spatial_cv = True
-subjects = ["CC120166"]
-# , "CC120313"]
-# , "CC120264", "CC120313", "CC120309"]
+n_jobs = 40
+nruns = 10
+spatial_cv = [True, False]
+subjects = ["CC120166", "CC120313", "CC120264", "CC120313", "CC120309"]
 metrics = [
     euclidean_distance,
     mse,
@@ -46,137 +44,138 @@
 estimator_alphas = np.logspace(0, -2, 20)[1:]
 memory = Memory(".")
 
-for subject in subjects:
-    """Fixed orientation parameters for the benchmark"""
-
-    orientation_type = "fixed"
-    data_args_I = {
-        # "n_sensors": [50],
-        "n_times": [10],
-        # "n_sources": [200],
-        "nnz": nnzs,
-        "cov_type": ["diag"],
-        "path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)],
-        "orientation_type": [orientation_type],
-        "alpha": alpha_SNR,  # this is actually SNR
-    }
-
-    data_args_II = {
-        # "n_sensors": [50],
-        "n_times": [10],
-        # "n_sources": [200],
-        "nnz": nnzs,
-        "cov_type": ["full"],
-        "path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)],
-        "orientation_type": [orientation_type],
-        "alpha": alpha_SNR,  # this is actually SNR
-    }
-
-    estimators = [
-        (fake_solver, data_args_I, {"alpha": estimator_alphas}, {}),
-        (eloreta, data_args_I, {"alpha": estimator_alphas}, {}),
-        (iterative_L1, data_args_I, {"alpha": estimator_alphas}, {}),
-        (iterative_L2, data_args_I, {"alpha": estimator_alphas}, {}),
-        (iterative_sqrt, data_args_I, {"alpha": estimator_alphas}, {}),
-        (iterative_L1_typeII, data_args_II, {"alpha": estimator_alphas}, {}),
-        (iterative_L2_typeII, data_args_II, {"alpha": estimator_alphas}, {}),
-        # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 1}),
-        (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 2}),
-        # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 3}),
-    ]
-
-    df_results = []
-    for estimator, data_args, estimator_args, estimator_extra_params in estimators:
-        benchmark = Benchmark(
-            estimator,
-            subject,
-            metrics,
-            data_args,
-            estimator_args,
-            random_state=42,
-            memory=memory,
-            n_jobs=n_jobs,
-            do_spatial_cv=do_spatial_cv,
-            estimator_extra_params=estimator_extra_params,
-        )
-        results = benchmark.run(nruns=nruns)
-        df_results.append(results)
-
-    df_results = pd.concat(df_results, axis=0)
-
-    data_path = Path("bsi_zoo/data")
-    data_path.mkdir(exist_ok=True)
-    if do_spatial_cv:
-        FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_spatialCV_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
-    else:
-        FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
-    df_results.to_pickle(data_path / FILE_NAME)
-
-    print(df_results)
-
-    """ Free orientation parameters for the benchmark """
-
-    orientation_type = "free"
-    data_args_I = {
-        "n_sensors": [50],
-        "n_times": [10],
-        "n_sources": [200],
-        "nnz": nnzs,
-        "cov_type": ["diag"],
-        "path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)],
-        "orientation_type": [orientation_type],
-        "alpha": alpha_SNR,  # this is actually SNR
-    }
-
-    data_args_II = {
-        "n_sensors": [50],
-        "n_times": [10],
-        "n_sources": [200],
-        "nnz": nnzs,
-        "cov_type": ["full"],
-        "path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)],
-        "orientation_type": [orientation_type],
-        "alpha": alpha_SNR,  # this is actually SNR
-    }
-
-    estimators = [
-        (fake_solver, data_args_I, {"alpha": estimator_alphas}, {}),
-        (eloreta, data_args_I, {"alpha": estimator_alphas}, {}),
-        (iterative_L1, data_args_I, {"alpha": estimator_alphas}, {}),
-        (iterative_L2, data_args_I, {"alpha": estimator_alphas}, {}),
-        (iterative_sqrt, data_args_I, {"alpha": estimator_alphas}, {}),
-        (iterative_L1_typeII, data_args_II, {"alpha": estimator_alphas}, {}),
-        (iterative_L2_typeII, data_args_II, {"alpha": estimator_alphas}, {}),
-        # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 1}),
-        (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 2}),
-        # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 3}),
-    ]
-
-    df_results = []
-    for estimator, data_args, estimator_args, estimator_extra_params in estimators:
-        benchmark = Benchmark(
-            estimator,
-            subject,
-            metrics,
-            data_args,
-            estimator_args,
-            random_state=42,
-            memory=memory,
-            n_jobs=n_jobs,
-            do_spatial_cv=do_spatial_cv,
-            estimator_extra_params=estimator_extra_params,
-        )
-        results = benchmark.run(nruns=nruns)
-        df_results.append(results)
-
-    df_results = pd.concat(df_results, axis=0)
-
-    data_path = Path("bsi_zoo/data")
-    data_path.mkdir(exist_ok=True)
-    if do_spatial_cv:
-        FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_spatialCV_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
-    else:
-        FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
-    df_results.to_pickle(data_path / FILE_NAME)
-
-    print(df_results)
+for do_spatial_cv in spatial_cv:
+    for subject in subjects:
+        """Fixed orientation parameters for the benchmark"""
+
+        orientation_type = "fixed"
+        data_args_I = {
+            # "n_sensors": [50],
+            "n_times": [10],
+            # "n_sources": [200],
+            "nnz": nnzs,
+            "cov_type": ["diag"],
+            "path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)],
+            "orientation_type": [orientation_type],
+            "alpha": alpha_SNR,  # this is actually SNR
+        }
+
+        data_args_II = {
+            # "n_sensors": [50],
+            "n_times": [10],
+            # "n_sources": [200],
+            "nnz": nnzs,
+            "cov_type": ["full"],
+            "path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)],
+            "orientation_type": [orientation_type],
+            "alpha": alpha_SNR,  # this is actually SNR
+        }
+
+        estimators = [
+            (fake_solver, data_args_I, {"alpha": estimator_alphas}, {}),
+            (eloreta, data_args_I, {"alpha": estimator_alphas}, {}),
+            (iterative_L1, data_args_I, {"alpha": estimator_alphas}, {}),
+            (iterative_L2, data_args_I, {"alpha": estimator_alphas}, {}),
+            (iterative_sqrt, data_args_I, {"alpha": estimator_alphas}, {}),
+            (iterative_L1_typeII, data_args_II, {"alpha": estimator_alphas}, {}),
+            (iterative_L2_typeII, data_args_II, {"alpha": estimator_alphas}, {}),
+            # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 1}),
+            (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 2}),
+            # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 3}),
+        ]
+
+        df_results = []
+        for estimator, data_args, estimator_args, estimator_extra_params in estimators:
+            benchmark = Benchmark(
+                estimator,
+                subject,
+                metrics,
+                data_args,
+                estimator_args,
+                random_state=42,
+                memory=memory,
+                n_jobs=n_jobs,
+                do_spatial_cv=do_spatial_cv,
+                estimator_extra_params=estimator_extra_params,
+            )
+            results = benchmark.run(nruns=nruns)
+            df_results.append(results)
+
+        df_results = pd.concat(df_results, axis=0)
+
+        data_path = Path("bsi_zoo/data")
+        data_path.mkdir(exist_ok=True)
+        if do_spatial_cv:
+            FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_spatialCV_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
+        else:
+            FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
+        df_results.to_pickle(data_path / FILE_NAME)
+
+        print(df_results)
+
+        """ Free orientation parameters for the benchmark """
+
+        orientation_type = "free"
+        data_args_I = {
+            "n_sensors": [50],
+            "n_times": [10],
+            "n_sources": [200],
+            "nnz": nnzs,
+            "cov_type": ["diag"],
+            "path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)],
+            "orientation_type": [orientation_type],
+            "alpha": alpha_SNR,  # this is actually SNR
+        }
+
+        data_args_II = {
+            "n_sensors": [50],
+            "n_times": [10],
+            "n_sources": [200],
+            "nnz": nnzs,
+            "cov_type": ["full"],
+            "path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)],
+            "orientation_type": [orientation_type],
+            "alpha": alpha_SNR,  # this is actually SNR
+        }
+
+        estimators = [
+            (fake_solver, data_args_I, {"alpha": estimator_alphas}, {}),
+            (eloreta, data_args_I, {"alpha": estimator_alphas}, {}),
+            (iterative_L1, data_args_I, {"alpha": estimator_alphas}, {}),
+            (iterative_L2, data_args_I, {"alpha": estimator_alphas}, {}),
+            (iterative_sqrt, data_args_I, {"alpha": estimator_alphas}, {}),
+            (iterative_L1_typeII, data_args_II, {"alpha": estimator_alphas}, {}),
+            (iterative_L2_typeII, data_args_II, {"alpha": estimator_alphas}, {}),
+            # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 1}),
+            (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 2}),
+            # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 3}),
+        ]
+
+        df_results = []
+        for estimator, data_args, estimator_args, estimator_extra_params in estimators:
+            benchmark = Benchmark(
+                estimator,
+                subject,
+                metrics,
+                data_args,
+                estimator_args,
+                random_state=42,
+                memory=memory,
+                n_jobs=n_jobs,
+                do_spatial_cv=do_spatial_cv,
+                estimator_extra_params=estimator_extra_params,
+            )
+            results = benchmark.run(nruns=nruns)
+            df_results.append(results)
+
+        df_results = pd.concat(df_results, axis=0)
+
+        data_path = Path("bsi_zoo/data")
+        data_path.mkdir(exist_ok=True)
+        if do_spatial_cv:
+            FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_spatialCV_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
+        else:
+            FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
+        df_results.to_pickle(data_path / FILE_NAME)
+
+        print(df_results)