Add post--model-selection analysis methods (#125)

* track iterations; fix `ModelHash.get_model`; add `Model.__repr__`

* add analysis module

* fixme: use pypesto PR branch

* unfix pypesto in tox.ini to use pyproject.toml

* sort iterations in `group_by_iteration`

* add `VIRTUAL_INITIAL_MODEL_HASH`

* method to convert Models to dataframe; update plotting code

* update vis notebook

* doc analysis methods briefly

* move models criterion getter to `Models`; add `Models.values()` for backwards compatibility; update plot code

* test case 0009: add caveat

* update expected test case results

---------

Co-authored-by: Daniel Weindl <dweindl@users.noreply.github.com>

Author: dilpath

doc/analysis.rst

@@ -0,0 +1,11 @@
+Analysis
+========
+
+After using PEtab Select to perform model selection, you may want to operate on all "good" calibrated models.
+The PEtab Select Python library provides some methods to help with this. Please request any missing methods.
+
+See the Python API docs for the :class:`petab_select.Models` class, which provides some methods. In particular, :attr:`petab_select.Models.df` can be used
+to get a quick overview over all models, as a pandas dataframe.
+
+Additionally, see the Python API docs for the ``petab_select.analysis`` module, which contains some methods to subset and group models,
+or compute "weights" (e.g. Akaike weights).

doc/api.rst

@@ -7,6 +7,7 @@ petab-select Python API
    :toctree: generated
 
    petab_select
+   petab_select.analyze
    petab_select.candidate_space
    petab_select.constants
    petab_select.criteria

doc/examples/calibrated_models/calibrated_models.yaml

@@ -2,7 +2,8 @@
     AICc: 37.97523003111246
     NLLH: 17.48761501555623
   estimated_parameters:
-    sigma_x2: 4.462298385653177
+    sigma_x2: 4.462298422134608
+  iteration: 1
   model_hash: M_0-000
   model_id: M_0-000
   model_subspace_id: M_0
@@ -17,11 +18,12 @@
   petab_yaml: petab_problem.yaml
   predecessor_model_hash: virtual_initial_model
 - criteria:
-    AICc: -0.1754060811089051
-    NLLH: -4.0877030405544525
+    AICc: -0.17540608110890332
+    NLLH: -4.087703040554452
   estimated_parameters:
     k3: 0.0
-    sigma_x2: 0.12242920113658744
+    sigma_x2: 0.12242920113658338
+  iteration: 2
   model_hash: M_1-000
   model_id: M_1-000
   model_subspace_id: M_1
@@ -36,11 +38,12 @@
   petab_yaml: petab_problem.yaml
   predecessor_model_hash: M_0-000
 - criteria:
-    AICc: -0.27451405630430337
-    NLLH: -4.137257028152152
+    AICc: -0.27451438069575573
+    NLLH: -4.137257190347878
   estimated_parameters:
-    k2: 0.10147827639089564
-    sigma_x2: 0.12142256779953603
+    k2: 0.10147824307890803
+    sigma_x2: 0.12142219599557078
+  iteration: 2
   model_hash: M_2-000
   model_id: M_2-000
   model_subspace_id: M_2
@@ -55,11 +58,12 @@
   petab_yaml: petab_problem.yaml
   predecessor_model_hash: M_0-000
 - criteria:
-    AICc: -0.7053270517931587
-    NLLH: -4.352663525896579
+    AICc: -0.7053270766271886
+    NLLH: -4.352663538313594
   estimated_parameters:
-    k1: 0.20160888007873565
-    sigma_x2: 0.11713858557052499
+    k1: 0.20160925279667963
+    sigma_x2: 0.11714017664827497
+  iteration: 2
   model_hash: M_3-000
   model_id: M_3-000
   model_subspace_id: M_3
@@ -74,12 +78,13 @@
   petab_yaml: petab_problem.yaml
   predecessor_model_hash: M_0-000
 - criteria:
-    AICc: 9.294672948206841
-    NLLH: -4.352663525896579
+    AICc: 9.294672923372811
+    NLLH: -4.352663538313594
   estimated_parameters:
-    k1: 0.20160888007873565
+    k1: 0.20160925279667963
     k3: 0.0
-    sigma_x2: 0.11713858557052499
+    sigma_x2: 0.11714017664827497
+  iteration: 3
   model_hash: M_5-000
   model_id: M_5-000
   model_subspace_id: M_5
@@ -94,12 +99,13 @@
   petab_yaml: petab_problem.yaml
   predecessor_model_hash: M_3-000
 - criteria:
-    AICc: 7.852170288089528
-    NLLH: -5.073914855955236
+    AICc: 7.8521704398854
+    NLLH: -5.0739147800573
   estimated_parameters:
-    k1: 0.20924739987621038
-    k2: 0.0859065470362628
-    sigma_x2: 0.1038731029818225
+    k1: 0.20924804320838675
+    k2: 0.0859052351446815
+    sigma_x2: 0.10386846319370771
+  iteration: 3
   model_hash: M_6-000
   model_id: M_6-000
   model_subspace_id: M_6
@@ -113,3 +119,25 @@
     k3: 0
   petab_yaml: petab_problem.yaml
   predecessor_model_hash: M_3-000
+- criteria:
+    AICc: 35.94352968170024
+    NLLH: -6.028235159149878
+  estimated_parameters:
+    k1: 0.6228488917665873
+    k2: 0.020189424009226256
+    k3: 0.0010850434974038557
+    sigma_x2: 0.08859278245811462
+  iteration: 4
+  model_hash: M_7-000
+  model_id: M_7-000
+  model_subspace_id: M_7
+  model_subspace_indices:
+  - 0
+  - 0
+  - 0
+  parameters:
+    k1: estimate
+    k2: estimate
+    k3: estimate
+  petab_yaml: petab_problem.yaml
+  predecessor_model_hash: M_3-000

doc/examples/example_cli_famos.ipynb

@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "1f04dce0",
    "metadata": {},
    "outputs": [],
@@ -44,7 +44,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "a81560e6",
    "metadata": {},
    "outputs": [],
@@ -109,69 +109,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "id": "bb1a5144",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Executing iteration 1\n",
-      "Executing iteration 2\n",
-      "Executing iteration 3\n",
-      "Executing iteration 4\n",
-      "Executing iteration 5\n",
-      "Executing iteration 6\n",
-      "Executing iteration 7\n",
-      "Executing iteration 8\n",
-      "Executing iteration 9\n",
-      "Executing iteration 10\n",
-      "Executing iteration 11\n",
-      "Executing iteration 12\n",
-      "Executing iteration 13\n",
-      "Executing iteration 14\n",
-      "Executing iteration 15\n",
-      "Executing iteration 16\n",
-      "Executing iteration 17\n",
-      "Executing iteration 18\n",
-      "Executing iteration 19\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "petab_select/petab_select/candidate_space.py:1142: RuntimeWarning: Model `model_subspace_1-0001011010010010` has been previously excluded from the candidate space so is skipped here.\n",
-      "  return_value = self.inner_candidate_space.consider(model)\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Executing iteration 20\n",
-      "Executing iteration 21\n",
-      "Executing iteration 22\n",
-      "Executing iteration 23\n",
-      "Executing iteration 24\n",
-      "Executing iteration 25\n",
-      "Executing iteration 26\n",
-      "Executing iteration 27\n",
-      "Executing iteration 28\n",
-      "Executing iteration 29\n",
-      "Executing iteration 30\n",
-      "Executing iteration 31\n",
-      "Executing iteration 32\n",
-      "Executing iteration 33\n",
-      "Executing iteration 34\n",
-      "Executing iteration 35\n",
-      "Executing iteration 36\n",
-      "Executing iteration 37\n",
-      "Model selection has terminated.\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "%%bash -s \"$petab_select_problem_yaml\" \"$output_path_str\"\n",
     "petab_select_problem_yaml=$1\n",
@@ -217,7 +158,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "93caf071",
    "metadata": {},
    "outputs": [],
@@ -227,7 +168,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "cb61d0f7",
    "metadata": {},
    "outputs": [],

doc/examples/model_selection/calibrated_M1_4.yaml

@@ -3,6 +3,7 @@ criteria:
 estimated_parameters:
   k2: 0.15
   k3: 0.0
+iteration: 1
 model_hash: M1_4-000
 model_id: M1_4-000
 model_subspace_id: M1_4

doc/examples/model_selection/calibrated_M1_7.yaml

@@ -4,6 +4,7 @@ estimated_parameters:
   k1: 0.25
   k2: 0.1
   k3: 0.0
+iteration: 2
 model_hash: M1_7-000
 model_id: M1_7-000
 model_subspace_id: M1_7

doc/examples/model_selection/calibrated_models_1.yaml

@@ -1,6 +1,7 @@
 - criteria:
     AIC: 180
   estimated_parameters: {}
+  iteration: 1
   model_hash: M1_0-000
   model_id: M1_0-000
   model_subspace_id: M1_0
@@ -17,6 +18,7 @@
 - criteria:
     AIC: 100
   estimated_parameters: {}
+  iteration: 1
   model_hash: M1_1-000
   model_id: M1_1-000
   model_subspace_id: M1_1
@@ -33,6 +35,7 @@
 - criteria:
     AIC: 50
   estimated_parameters: {}
+  iteration: 1
   model_hash: M1_2-000
   model_id: M1_2-000
   model_subspace_id: M1_2