Merge pull request #369 from IMSY-DKFZ/T365_refine_benchmarking

refine benchmarking scripts

docs/source/benchmarking.md

@@ -7,15 +7,20 @@ It allows customization of initial spacing, final spacing, step size, output fil
 To check and amend which simulations are run, see the [performance check script](../../simpa_examples/benchmarking/performance_check.py).
 
 ## Usage
-To use this script, run it from the command line with the desired options. Please ensure you check two things before
-running the script: First, ensure that the device will not be in use for the duration - ideally restart before 
-benchmarking - of the benchmarking process, 
+In order to be able to use this script, please first ensure that you have the dependencies required for the benchmarking
+scripts. To do this, please navigate to the simpa directory and execute `pip install .[profile]`.
+
+Now, you can run [performance_check.py](../../simpa_examples/benchmarking/performance_check.py) and [run_benchmarking.sh](../../simpa_examples/benchmarking/run_benchmarking.sh) from the command line with the desired
+options. Please ensure you check two things before running the script: First, ensure that the device will not be in use
+for the duration - ideally restart before benchmarking - of the benchmarking process, 
 as this will create large uncertainties within the outcomes. Second, ensure that you don't accidentally write over any 
 existing file by saving the files created by this script after runtime to different location.
 
-The script will create multiple text files (eg. benchmarking_data_TIME_0.2.txt), showing the line by line profiling of
-the most recent runs, as well as two csv's with the data from all the runs (benchmarking_data_frame.csv) and the means
-and standard deviations of all the runs (benchmarking_data_frame_mean.csv).
+The both scripts create text files (eg. benchmarking_data_TIME_0.2.txt), showing the line by line profiling of
+the most recent runs. The [run_benchmarking.sh](../../simpa_examples/benchmarking/run_benchmarking.sh) also creates two csv's: one with the data from all the runs
+(benchmarking_data_frame.csv) and one with the means and standard deviations of all the runs
+(benchmarking_data_frame_mean.csv). With both scripts, unless the user intentionally changes the save folder name,
+the output files will be overwritten.
 
 Below is a description of the available options and how to use them.
 
@@ -28,8 +33,8 @@ Please put this in the conversation of the pull request, and not add it to the f
 
 ## Options
 - **`-i, --init`**: First spacing to benchmark (default = 0.2mm).
-- **`-c, --cease`**: Final spacing to benchmark (default = 0.4mm).
-- **`-s, --step`**: Step between spacings (default = 0.1mm).
+- **`-c, --cease`**: Final spacing to benchmark (default = 0.25mm).
+- **`-s, --step`**: Step between spacings (default = 0.05mm).
 - **`-f, --file`**: Where to store the output files (default = save in current directory; 'print' prints it in console).
 - **`-t, --time`**: Profile times taken (if no profile is specified, all are set).
 - **`-g, --gpu`**: Profile GPU usage (if no profile is specified, all are set).
@@ -40,9 +45,9 @@ Please put this in the conversation of the pull request, and not add it to the f
 ## Default Values
 If no options are provided for initial spacing, final spacing, or step size, the script uses the following default
 values:
-- **Initial Spacing**: 0.2mm
-- **Final Spacing**: 0.4mm
-- **Step Size**: 0.1mm
+- **Initial Spacing**: 0.15mm
+- **Final Spacing**: 0.25mm
+- **Step Size**: 0.05mm
 
 If no profiling options are specified, all three profilers (time, GPU memory, and memory) are used by default.
 

simpa/core/simulation_modules/reconstruction_module/reconstruction_module_time_reversal_adapter.py

@@ -55,7 +55,8 @@ def get_acoustic_properties(self, input_data: dict, detection_geometry):
             raise AttributeError("Please specify a value for SPACING_MM")
 
         detector_positions = detection_geometry.get_detector_element_positions_accounting_for_device_position_mm()
-        detector_positions_voxels = np.round(detector_positions / spacing_in_mm).astype(int)
+        # we add eps of 1e-10 because numpy rounds 0.5 to the next even number
+        detector_positions_voxels = np.round(detector_positions / spacing_in_mm + 1e-10).astype(int)
 
         # plus 2 because of off-
         volume_x_dim = int(np.ceil(self.global_settings[Tags.DIM_VOLUME_X_MM] / spacing_in_mm) + 1)

simpa_examples/__init__.py

@@ -9,3 +9,4 @@
 from simpa_examples.optical_and_acoustic_simulation import run_optical_and_acoustic_simulation
 from simpa_examples.perform_image_reconstruction import run_perform_image_reconstruction
 from simpa_examples.perform_iterative_qPAI_reconstruction import run_perform_iterative_qPAI_reconstruction
+from simpa_examples.segmentation_loader import run_segmentation_loader

simpa_examples/benchmarking/performance_check.py

@@ -31,13 +31,10 @@ def run_benchmarking_tests(spacing=0.4, profile: str = "TIME", savefolder: str =
     examples = [simpa_examples.run_linear_unmixing, simpa_examples.run_minimal_optical_simulation,
                 simpa_examples.run_minimal_optical_simulation_uniform_cube, simpa_examples.run_msot_invision_simulation,
                 simpa_examples.run_optical_and_acoustic_simulation,
-                simpa_examples.run_perform_iterative_qPAI_reconstruction]
+                simpa_examples.run_perform_iterative_qPAI_reconstruction, simpa_examples.run_segmentation_loader]
 
     for example in examples:
-        try:
-            example(spacing=spacing, path_manager=None, visualise=False)
-        except AttributeError:
-            print("simulation cannot be run on {} with spacing {}".format(example, spacing))
+        example(spacing=spacing, path_manager=None, visualise=False)
 
 
 if __name__ == "__main__":

simpa_examples/benchmarking/run_benchmarking.sh

@@ -1,14 +1,16 @@
 #!/bin/bash
 
+set -e
+
 help() {
 echo "Usage: calculate benchmarking for [options]"
 echo "For further details see readme"
 echo "Number of examples can be selected in performance_check.py"
 echo "For comparable benchmarks, please use default"
 echo "Options:"
-echo "  -i, --init           First spacing to benchmark: default = 0.2mm"
-echo "  -c, --cease          Final spacing to benchmark: default = 0.4mm"
-echo "  -s, --step           Step between spacings: default = 0.1mm"
+echo "  -i, --init           First spacing to benchmark: default = 0.15mm"
+echo "  -c, --cease          Final spacing to benchmark: default = 0.25mm"
+echo "  -s, --step           Step between spacings: default = 0.05mm"
 echo "  -f, --file           Where to store the output files: default save in current directory; 'print' prints it in console"
 echo "  -t, --time           Profile times taken: if no profile, all are set"
 echo "  -g, --gpu            Profile GPU usage: if no profile, all are set"
@@ -57,15 +59,15 @@ shift 1
 done
 
 if [ "$start" == 0 ]; then
-  start=0.2
+  start=0.15
 fi
 
 if [ "$stop" == 0 ]; then
-  stop=0.4
+  stop=0.25
 fi
 
 if [ "$step" == 0 ]; then
-  step=0.1
+  step=0.05
 fi
 
 if [ ${#profiles[@]} -eq 0 ]; then

simpa_examples/minimal_optical_simulation.py

@@ -152,7 +152,7 @@ def __init__(self):
 
     device = ExampleDeviceSlitIlluminationLinearDetector()
 
-    sp.simulate(pipeline, settings, device, logging_level=Tags.LOGGER_ERROR)
+    sp.simulate(pipeline, settings, device)
 
     if Tags.WAVELENGTH in settings:
         WAVELENGTH = settings[Tags.WAVELENGTH]