Allow different metrics as objective function and add a stopping criteria for the calibration

.gitignore

@@ -0,0 +1,11 @@
+slurm-*.out
+output*.out
+
+# notebook stuff
+.ipynb_checkpoints
+
+# Ignore macOS system files
+.DS_Store
+
+# Ignore environment configuration files
+.env

demo1/control_active.txt

@@ -5,30 +5,35 @@
 # Note on path specification. If deviating from default paths, a full path must be specified. E.g. '/home/user/non-default/path'
 
 ## ---- PART 1. Paramerter estimation settings ---- 
-calib_path             | <demo_directory_path>    # (01) Path where parameter estimation is stored. e.g., /____/demo1
+calib_path             | <demo_directory_path> # (01) Path where parameter estimation is stored. e.g., /____/demo1
 object_parameters      | k_macropore, k_soil, theta_sat, aquiferBaseflowExp, aquiferBaseflowRate, qSurfScale, summerLAI, frozenPrecipMultip, heightCanopyBottom, heightCanopyTop, routingGammaScale, routingGammaShape, Fcapil  # (02) Parameter names to be optimized or evaluated. The parameter names should be the same as they are in the summa source code.
 ostIn_tpl              | tpl/ostIn.DDS.tpl     # (03) Name of ostIn template file. Input file in '[calib_path]/[ostIn_tpl]'. 
-max_iterations         | 10                    # (04) Maximum Number of iterations for optimization. Optional input, depending on the optimization method. Eg, DDS needs this configuration.
-WarmStart              | no                    # (05) If set to "yes" OSTRICH will read the contents of any previously created "OstModel" output files and use the entries therein to restart an optimization or calibration exercise.
+WarmStart              | no                    # (04) If set to "yes" OSTRICH will read the contents of any previously created "OstModel" output files and use the entries therein to restart an optimization or calibration exercise.
+objective_function     | KGE                   # (05) Name of the objective function. Must be: 'KGE', 'KGEp', 'NSE', 'MAE', 'RMSE'.
+max_iterations         | 10                    # (06) Maximum Number of iterations for optimization. Optional input, depending on the optimization method. Eg, DDS or SCE needs this configuration (MAXN).
+stopping_criteria      | FALSE                 # (07) If TRUE, add a stopping criteria to stop calibration if there is no further improvement. Optional input, depending on the optimization method. Eg, can be used with DDS but already included in SCE.
+loop_stagnation        | 5                     # (08) Number of previous value of the objective function to check. Optional input, depending on the optimization method. Eg, SCE needs this configuration (KSTOP), and it's optional for DDS (only if stopping criteria = TRUE).
+per_change             | 0.01                  # (09) Percentage by which the optimization criterion value must change. Optional input, depending on the optimization method. Eg, SCE needs this configuration (PCENTO), and it's optional for DDS (only if stopping criteria = TRUE).
+
 
 ## ---- PART 2. Hydrologic model settings  ---- 
-model_path             | default               # (06) Path of destination hydrologic model. If 'default', use '[calib_path]/model'.
+model_path             | default               # (10) Path of destination hydrologic model. If 'default', use '[calib_path]/model'.
 
-summa_settings_relpath | settings/SUMMA        # (07) Relative path of summa model settings folder, relative to [model_path]. 
-summa_filemanager      | fileManager.txt       # (08) Name of the SUMMA master configuration file.
-summa_exe_path         | <summa_exe_path>      # (09) summa executable path. e.g., /____/summa.exe
+summa_settings_relpath | settings/SUMMA        # (11) Relative path of summa model settings folder, relative to [model_path]. 
+summa_filemanager      | fileManager.txt       # (12) Name of the SUMMA master configuration file.
+summa_exe_path         | <summa_exe_path>      # (13) summa executable path. e.g., /____/summa.exe
 
-route_settings_relpath | settings/mizuRoute    # (10) Relative path of summa model settings folder, relative to [model_path]. 
-route_control          | mizuroute.control     # (11) Name of the mizuRoute configuration file.
-route_exe_path         | <mizuroute_exe_path>  # (12) muziroute executable path, e.g., /____/mizuroute.exe
+route_settings_relpath | settings/mizuRoute    # (14) Relative path of summa model settings folder, relative to [model_path]. 
+route_control          | mizuroute.control     # (15) Name of the mizuRoute configuration file.
+route_exe_path         | <mizuroute_exe_path>  # (16) muziroute executable path, e.g., /____/mizuroute.exe
 
-simStartTime           | 2008-07-15 00:00      # (13) Start time for hydrologic simualtion, in format yyyy-mm-dd hh:mm. 
-simEndTime             | 2008-07-31 23:00      # (14) End time for hydrologic simualtion, in format yyyy-mm-dd hh:mm.  
+simStartTime           | 2008-07-15 00:00      # (17) Start time for hydrologic simualtion, in format yyyy-mm-dd hh:mm. 
+simEndTime             | 2008-07-31 23:00      # (18) End time for hydrologic simualtion, in format yyyy-mm-dd hh:mm.  
 
 ## ---- PART 3. Calculate statistics settings ----
-q_seg_index            | 49                    # (15) segment index in routing output file that matches obs location (start from 1). For the demo domain, its outlet is located on reachID 71028585 which corresponds to the 49th segment.
-obs_file_path          | ./obs_flow.BowRiveratBanff.cfs.csv  # (16) Path of observed streamflow data.
-obs_unit               | cfs                   # (17) Observation streamflow data unit (cfs or cms).
-stat_output            | trial_stats.txt       # (18) Name of file with statistical metric results. Output file in [calib_path].
-statStartDate          | 2008-07-15            # (19) Start date for statistics calculation, in format yyyy-mm-dd. 
-statEndDate            | 2008-07-31            # (20) End date for statistics calculation, in format yyyy-mm-dd.  
+q_seg_index            | 49                    # (19) segment index in routing output file that matches obs location (start from 1). For the demo domain, its outlet is located on reachID 71028585 which corresponds to the 49th segment.
+obs_file_path          | ./obs_flow.BowRiveratBanff.cfs.csv  # (20) Path of observed streamflow data.
+obs_unit               | cfs                   # (21) Observation streamflow data unit (cfs or cms).
+stat_output            | trial_stats.txt       # (22) Name of file with statistical metric results. Output file in [calib_path].
+statStartDate          | 2008-07-15            # (23) Start date for statistics calculation, in format yyyy-mm-dd. 
+statEndDate            | 2008-07-31            # (24) End date for statistics calculation, in format yyyy-mm-dd.

demo1/save_best.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Save use-specifed files every time a best paraemter set is discovered.
+# Save use-specifed files every time a best parameter set is discovered.
 
 # -----------------------------------------------------------------------------------------
 # ----------------------------- User specified input --------------------------------------
@@ -104,5 +104,16 @@ cp $stat_output $outDir/
 cp $calib_path/Ost*.txt $outDir/
 cp $calib_path/timetrack.log $outDir/
 
+# Read stopping_criteria condition from control_file.
+stopping_criteria="$(read_from_control $control_file "stopping_criteria")"
+
+# Convert to lowercase and remove spaces
+stopping_criteria=$(echo "$stopping_criteria" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')
+
+# Run stopping_criteria.py script if specified in the control file
+if [ "$stopping_criteria" = "true" ]; then
+    python ../scripts/stopping_criteria.py $control_file
+fi
+
 exit 0
 

demo2/control_active.txt

@@ -5,30 +5,35 @@
 # Note on path specification. If deviating from default paths, a full path must be specified. E.g. '/home/user/non-default/path'
 
 ## ---- PART 1. Paramerter estimation settings ---- 
-calib_path             | <demo_directory_path>    # (01) Path where parameter estimation is stored. e.g., /____/demo2
-object_parameters      | k_macropore, k_soil, theta_sat, aquiferBaseflowExp, aquiferBaseflowRate, qSurfScale, summerLAI, frozenPrecipMultip, heightCanopyBottom, heightCanopyTop, routingGammaScale, routingGammaShape, Fcapil         # (02) Parameter names to be optimized or evaluated. The parameter names should be the same as they are in summa.
-initial_option         | UseInitialParamValues # (03) Initial value option: UseInitialParamValues or UseRandomParamValues.
-max_iterations         | 10                    # (04) Maximum Number of iterations for optimization. Optional input, depending on the optimization method. Eg, DDS needs this configuration.
-WarmStart              | no                    # (05) If set to "yes" OSTRICH will read the contents of any previously created "OstModel" output files and use the entries therein to restart an optimization or calibration exercise.
+calib_path             | <demo_directory_path> # (01) Path where parameter estimation is stored. e.g., /____/demo1
+object_parameters      | k_macropore, k_soil, theta_sat, aquiferBaseflowExp, aquiferBaseflowRate, qSurfScale, summerLAI, frozenPrecipMultip, heightCanopyBottom, heightCanopyTop, routingGammaScale, routingGammaShape, Fcapil  # (02) Parameter names to be optimized or evaluated. The parameter names should be the same as they are in the summa source code.
+ostIn_tpl              | tpl/ostIn.DDS.tpl     # (03) Name of ostIn template file. Input file in '[calib_path]/[ostIn_tpl]'. 
+WarmStart              | no                    # (04) If set to "yes" OSTRICH will read the contents of any previously created "OstModel" output files and use the entries therein to restart an optimization or calibration exercise.
+objective_function     | KGE                   # (05) Name of the objective function. Must be: 'KGE', 'KGEp', 'NSE', 'MAE', 'RMSE'.
+max_iterations         | 10                    # (06) Maximum Number of iterations for optimization. Optional input, depending on the optimization method. Eg, DDS or SCE needs this configuration (MAXN).
+stopping_criteria      | FALSE                 # (07) If TRUE, add a stopping criteria to stop calibration if there is no further improvement. Optional input, depending on the optimization method. Eg, can be used with DDS but already included in SCE.
+loop_stagnation        | 5                     # (08) Number of previous value of the objective function to check. Optional input, depending on the optimization method. Eg, SCE needs this configuration (KSTOP), and it's optional for DDS (only if stopping criteria = TRUE).
+per_change             | 0.01                  # (09) Percentage by which the optimization criterion value must change. Optional input, depending on the optimization method. Eg, SCE needs this configuration (PCENTO), and it's optional for DDS (only if stopping criteria = TRUE).
+
 
 ## ---- PART 2. Hydrologic model settings  ---- 
-model_path             | default               # (06) Path of destination hydrologic model. If 'default', use '[calib_path]/model'.
+model_path             | default               # (10) Path of destination hydrologic model. If 'default', use '[calib_path]/model'.
 
-summa_settings_relpath | settings/SUMMA        # (07) Relative path of summa model settings folder, relative to [model_path]. 
-summa_filemanager      | fileManager.txt       # (08) Name of the SUMMA master configuration file.
-summa_exe_path         | <summa_exe_path>      # (09) summa executable path. e.g., /____/summa.exe
+summa_settings_relpath | settings/SUMMA        # (11) Relative path of summa model settings folder, relative to [model_path]. 
+summa_filemanager      | fileManager.txt       # (12) Name of the SUMMA master configuration file.
+summa_exe_path         | <summa_exe_path>      # (13) summa executable path. e.g., /____/summa.exe
 
-route_settings_relpath | settings/mizuRoute    # (10) Relative path of summa model settings folder, relative to [model_path]. 
-route_control          | mizuroute.control     # (11) Name of the mizuRoute configuration file.
-route_exe_path         | <mizuroute_exe_path>  # (12) muziroute executable path, e.g., /____/mizuroute.exe
+route_settings_relpath | settings/mizuRoute    # (14) Relative path of summa model settings folder, relative to [model_path]. 
+route_control          | mizuroute.control     # (15) Name of the mizuRoute configuration file.
+route_exe_path         | <mizuroute_exe_path>  # (16) muziroute executable path, e.g., /____/mizuroute.exe
 
-simStartTime           | 2008-07-15 00:00       # (13) Start time for hydrologic simualtion, in format yyyy-mm-dd hh:mm. 
-simEndTime             | 2008-07-31 23:00       # (14) End time for hydrologic simualtion, in format yyyy-mm-dd hh:mm.  
+simStartTime           | 2008-07-15 00:00      # (17) Start time for hydrologic simualtion, in format yyyy-mm-dd hh:mm. 
+simEndTime             | 2008-07-31 23:00      # (18) End time for hydrologic simualtion, in format yyyy-mm-dd hh:mm.  
 
 ## ---- PART 3. Calculate statistics settings ----
-q_seg_index            | 49                      # (15) segment index in routing output file that matches obs location (start from 1). For the demo domain, its outlet is located on reachID 71028585 which corresponds to the 49th segment.
-obs_file               | ./obs_flow.BowRiveratBanff.cfs.csv  # (16) Path of observed streamflow data.
-obs_unit               | cfs                    # (17) Observation streamflow data unit (cfs or cms).
-stat_output            | trial_stats.txt        # (18) Name of file with statistical metric results. Output file in [calib_path].
-statStartDate          | 2008-07-15             # (19) Start date for statistics calculation, in format yyyy-mm-dd. 
-statEndDate            | 2008-07-31             # (20) End date for statistics calculation, in format yyyy-mm-dd.  
+q_seg_index            | 49                    # (19) segment index in routing output file that matches obs location (start from 1). For the demo domain, its outlet is located on reachID 71028585 which corresponds to the 49th segment.
+obs_file_path          | ./obs_flow.BowRiveratBanff.cfs.csv  # (20) Path of observed streamflow data.
+obs_unit               | cfs                   # (21) Observation streamflow data unit (cfs or cms).
+stat_output            | trial_stats.txt       # (22) Name of file with statistical metric results. Output file in [calib_path].
+statStartDate          | 2008-07-15            # (23) Start date for statistics calculation, in format yyyy-mm-dd. 
+statEndDate            | 2008-07-31            # (24) End date for statistics calculation, in format yyyy-mm-dd.