diff --git a/bsi_zoo/run_benchmark.py b/bsi_zoo/run_benchmark.py index 20c25c4..a28a753 100644 --- a/bsi_zoo/run_benchmark.py +++ b/bsi_zoo/run_benchmark.py @@ -73,14 +73,14 @@ estimators = [ (fake_solver, data_args_I, {"alpha": estimator_alphas}, {}), - (eloreta, data_args_I, {"alpha": estimator_alphas}, {}), - (iterative_L1, data_args_I, {"alpha": estimator_alphas}, {}), - (iterative_L2, data_args_I, {"alpha": estimator_alphas}, {}), - (iterative_sqrt, data_args_I, {"alpha": estimator_alphas}, {}), - (iterative_L1_typeII, data_args_II, {"alpha": estimator_alphas}, {}), - (iterative_L2_typeII, data_args_II, {"alpha": estimator_alphas}, {}), + # (eloreta, data_args_I, {"alpha": estimator_alphas}, {}), + # (iterative_L1, data_args_I, {"alpha": estimator_alphas}, {}), + # (iterative_L2, data_args_I, {"alpha": estimator_alphas}, {}), + # (iterative_sqrt, data_args_I, {"alpha": estimator_alphas}, {}), + # (iterative_L1_typeII, data_args_II, {"alpha": estimator_alphas}, {}), + # (iterative_L2_typeII, data_args_II, {"alpha": estimator_alphas}, {}), # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 1}), - (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 2}), + # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 2}), # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 3}), ] @@ -100,73 +100,12 @@ ) results = benchmark.run(nruns=nruns) df_results.append(results) + # save results + data_path = Path("bsi_zoo/data") + data_path.mkdir(exist_ok=True) + FILE_NAME = f"{estimator}_{subject}_{data_args['orientation_type'][0]}_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl" + results.to_pickle(data_path / FILE_NAME) - df_results = pd.concat(df_results, axis=0) - - data_path = Path("bsi_zoo/data") - data_path.mkdir(exist_ok=True) - if do_spatial_cv: - FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_spatialCV_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl" - else: - FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl" - df_results.to_pickle(data_path / FILE_NAME) - - print(df_results) - - """ Free orientation parameters for the benchmark """ - - orientation_type = "free" - data_args_I = { - "n_sensors": [50], - "n_times": [10], - "n_sources": [200], - "nnz": nnzs, - "cov_type": ["diag"], - "path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)], - "orientation_type": [orientation_type], - "alpha": alpha_SNR, # this is actually SNR - } - - data_args_II = { - "n_sensors": [50], - "n_times": [10], - "n_sources": [200], - "nnz": nnzs, - "cov_type": ["full"], - "path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)], - "orientation_type": [orientation_type], - "alpha": alpha_SNR, # this is actually SNR - } - - estimators = [ - (fake_solver, data_args_I, {"alpha": estimator_alphas}, {}), - (eloreta, data_args_I, {"alpha": estimator_alphas}, {}), - (iterative_L1, data_args_I, {"alpha": estimator_alphas}, {}), - (iterative_L2, data_args_I, {"alpha": estimator_alphas}, {}), - (iterative_sqrt, data_args_I, {"alpha": estimator_alphas}, {}), - (iterative_L1_typeII, data_args_II, {"alpha": estimator_alphas}, {}), - (iterative_L2_typeII, data_args_II, {"alpha": estimator_alphas}, {}), - # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 1}), - (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 2}), - # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 3}), - ] - - df_results = [] - for estimator, data_args, estimator_args, estimator_extra_params in estimators: - benchmark = Benchmark( - estimator, - subject, - metrics, - data_args, - estimator_args, - random_state=42, - memory=memory, - n_jobs=n_jobs, - do_spatial_cv=do_spatial_cv, - estimator_extra_params=estimator_extra_params, - ) - results = benchmark.run(nruns=nruns) - df_results.append(results) df_results = pd.concat(df_results, axis=0) @@ -179,3 +118,75 @@ df_results.to_pickle(data_path / FILE_NAME) print(df_results) + + # """ Free orientation parameters for the benchmark """ + + # orientation_type = "free" + # data_args_I = { + # "n_sensors": [50], + # "n_times": [10], + # "n_sources": [200], + # "nnz": nnzs, + # "cov_type": ["diag"], + # "path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)], + # "orientation_type": [orientation_type], + # "alpha": alpha_SNR, # this is actually SNR + # } + + # data_args_II = { + # "n_sensors": [50], + # "n_times": [10], + # "n_sources": [200], + # "nnz": nnzs, + # "cov_type": ["full"], + # "path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)], + # "orientation_type": [orientation_type], + # "alpha": alpha_SNR, # this is actually SNR + # } + + # estimators = [ + # (fake_solver, data_args_I, {"alpha": estimator_alphas}, {}), + # (eloreta, data_args_I, {"alpha": estimator_alphas}, {}), + # (iterative_L1, data_args_I, {"alpha": estimator_alphas}, {}), + # (iterative_L2, data_args_I, {"alpha": estimator_alphas}, {}), + # (iterative_sqrt, data_args_I, {"alpha": estimator_alphas}, {}), + # (iterative_L1_typeII, data_args_II, {"alpha": estimator_alphas}, {}), + # (iterative_L2_typeII, data_args_II, {"alpha": estimator_alphas}, {}), + # # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 1}), + # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 2}), + # # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 3}), + # ] + + # df_results = [] + # for estimator, data_args, estimator_args, estimator_extra_params in estimators: + # benchmark = Benchmark( + # estimator, + # subject, + # metrics, + # data_args, + # estimator_args, + # random_state=42, + # memory=memory, + # n_jobs=n_jobs, + # do_spatial_cv=do_spatial_cv, + # estimator_extra_params=estimator_extra_params, + # ) + # results = benchmark.run(nruns=nruns) + # df_results.append(results) + # # save results + # data_path = Path("bsi_zoo/data") + # data_path.mkdir(exist_ok=True) + # FILE_NAME = f"{estimator}_{subject}_{data_args['orientation_type'][0]}_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl" + # results.to_pickle(data_path / FILE_NAME) + + # df_results = pd.concat(df_results, axis=0) + + # data_path = Path("bsi_zoo/data") + # data_path.mkdir(exist_ok=True) + # if do_spatial_cv: + # FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_spatialCV_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl" + # else: + # FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl" + # df_results.to_pickle(data_path / FILE_NAME) + + # print(df_results)