Use returned snapshot instead of log file on disk

equinor · Nov 22, 2023 · 27e8a9a · 27e8a9a
1 parent 246ac4f
commit 27e8a9a
Show file tree

Hide file tree

Showing 4 changed files with 266 additions and 69 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -27,7 +27,7 @@ classifiers=[
 dynamic = ["version"]
 dependencies=[
     "resdata",
-    "ert>=6.0.0-rc0",
+    "ert>=8.0.2",
     "configsuite>=0.6",
     "numpy",
     "pandas>1.3.0",

diff --git a/semeio/workflows/ahm_analysis/ahmanalysis.py b/semeio/workflows/ahm_analysis/ahmanalysis.py
@@ -1,14 +1,13 @@
 import collections
 import itertools
-import os
 import tempfile
 import warnings
 from pathlib import Path
 
 import numpy as np
 import pandas as pd
 import xtgeo
-from ert.analysis import ErtAnalysisError
+from ert.analysis import ErtAnalysisError, SmootherSnapshot
 from ert.shared.plugins.plugin_manager import hook_implementation
 from scipy.stats import ks_2samp
 from sklearn.decomposition import PCA
@@ -170,7 +169,7 @@ def run(
             print("Processing:", group_name)
 
             #  Use localization to evaluate change of parameters for each observation
-            with tempfile.TemporaryDirectory() as update_log_path:
+            with tempfile.TemporaryDirectory():
                 try:
                     prior_ensemble = self.storage.get_ensemble_by_name(prior_name)
                     target_ensemble = self.storage.create_ensemble(
@@ -179,16 +178,15 @@ def run(
                         ensemble_size=prior_ensemble.ensemble_size,
                         prior_ensemble=prior_ensemble,
                     )
-                    _run_ministep(
+                    update_log = _run_ministep(
                         self.facade,
                         prior_ensemble,
                         target_ensemble,
                         obs_group,
                         field_parameters + scalar_parameters,
-                        update_log_path,
                     )
                     # Get the active vs total observation info
-                    df_update_log = make_update_log_df(update_log_path)
+                    df_update_log = make_update_log_df(update_log)
                 except ErtAnalysisError:
                     df_update_log = pd.DataFrame(
                         columns=[
@@ -293,20 +291,15 @@ def run(
         self.reporter.publish_csv("prior", prior_data)
 
 
-def _run_ministep(
-    facade, prior_storage, target_storage, obs_group, data_parameters, output_path
-):
+def _run_ministep(facade, prior_storage, target_storage, obs_group, data_parameters):
     # pylint: disable=too-many-arguments
     update_step = {
         "name": "MINISTEP",
         "observations": obs_group,
         "parameters": data_parameters,
     }
     facade.update_configuration = [update_step]
-    facade.set_log_path(output_path)
-    # Perform update analysis
-
-    facade.smoother_update(prior_storage, target_storage, target_storage.name)
+    return facade.smoother_update(prior_storage, target_storage, target_storage.name)
 
 
 def _get_field_params(facade, field_parameters, target_ensemble):
@@ -326,39 +319,28 @@ def _get_field_params(facade, field_parameters, target_ensemble):
     return field_data
 
 
-def make_update_log_df(update_log_dir):
+def make_update_log_df(update_log: SmootherSnapshot):
     """Read update_log file to get active and inactive observations"""
-    list_of_files = [
-        os.path.join(update_log_dir, f) for f in os.listdir(update_log_dir)
+    update_step = update_log.update_step_snapshots["MINISTEP"]
+    obs_key = [step.obs_name for step in update_step]
+    obs_mean = [step.obs_val for step in update_step]
+    obs_std = [step.obs_std for step in update_step]
+    sim_mean = [step.response_mean for step in update_step]
+    sim_std = [step.response_std for step in update_step]
+    status = [
+        "Active" if step.response_mean_mask and step.response_std_mask else "Inactive"
+        for step in update_step
     ]
-    if len(list_of_files) > 1:
-        raise OSError("ERROR more than one update_log_file.")
-    if len(list_of_files) == 0:
-        raise OSError("ERROR empty update_log directory.")
-    # read file
-    updatelog = pd.read_csv(
-        list_of_files[0],
-        delim_whitespace=True,
-        skiprows=12,
-        usecols=[2, 3, 5, 6, 8, 10],
-        header=None,
-        engine="python",
-        skipfooter=1,
+    updatelog = pd.DataFrame(
+        {
+            "obs_key": obs_key,
+            "obs_mean": obs_mean,
+            "obs_std": obs_std,
+            "status": status,
+            "sim_mean": sim_mean,
+            "sim_std": sim_std,
+        }
     )
-    # define header
-    updatelog.columns = [
-        "obs_key",
-        "obs_mean",
-        "obs_std",
-        "status",
-        "sim_mean",
-        "sim_std",
-    ]
-
-    # ---------------------------------
-    # add proper name for obs_key in rows where value is '...'
-    updatelog.replace("...", np.nan, inplace=True)
-    updatelog.ffill(inplace=True)
 
     return updatelog
 

diff --git a/tests/jobs/ahm_analysis/snapshots/test_ahm_analysis/test_make_update_log_df/update_log.csv b/tests/jobs/ahm_analysis/snapshots/test_ahm_analysis/test_make_update_log_df/update_log.csv
@@ -0,0 +1,211 @@
+,obs_key,obs_mean,obs_std,status,sim_mean,sim_std
+0,FOPR,0.0017,0.1,Active,0.0649,0.0912
+1,FOPR,0.0075,0.1,Active,0.0675,0.0927
+2,FOPR,0.0175,0.1,Active,0.0727,0.0949
+3,FOPR,0.0316,0.1,Active,0.0801,0.098
+4,FOPR,0.0496,0.1,Active,0.0904,0.1014
+5,FOPR,0.0715,0.1,Active,0.1028,0.1058
+6,FOPR,0.097,0.1,Active,0.1183,0.1102
+7,FOPR,0.1261,0.1,Active,0.1379,0.1135
+8,FOPR,0.1586,0.1,Active,0.1608,0.1185
+9,FOPR,0.1943,0.1,Active,0.187,0.1247
+10,FOPR,0.2328,0.1,Active,0.2187,0.1297
+11,FOPR,0.274,0.1,Active,0.2531,0.1354
+12,FOPR,0.3177,0.1,Active,0.2943,0.139
+13,FOPR,0.3634,0.1,Active,0.3413,0.1393
+14,FOPR,0.411,0.1,Active,0.3903,0.1395
+15,FOPR,0.46,0.1,Active,0.4411,0.1396
+16,FOPR,0.5103,0.1,Active,0.4931,0.1399
+17,FOPR,0.5614,0.1,Active,0.5459,0.1404
+18,FOPR,0.6134,0.1,Active,0.5995,0.1404
+19,FOPR,0.6655,0.1,Active,0.6536,0.1406
+20,FOPR,0.7179,0.1,Active,0.7079,0.1408
+21,FOPR,0.7704,0.1,Active,0.7617,0.1409
+22,FOPR,0.8227,0.1,Active,0.8146,0.1411
+23,FOPR,0.8746,0.1,Active,0.8665,0.1412
+24,FOPR,0.9262,0.1,Active,0.9172,0.1415
+25,FOPR,0.977,0.1,Active,0.9669,0.1413
+26,FOPR,1.0267,0.1027,Active,1.0139,0.1416
+27,FOPR,1.075,0.1075,Active,1.0579,0.1422
+28,FOPR,1.1216,0.1122,Active,1.0993,0.1429
+29,FOPR,1.1658,0.1166,Active,1.1383,0.1432
+30,FOPR,1.2075,0.1208,Active,1.1741,0.1438
+31,FOPR,1.247,0.1247,Active,1.2068,0.1443
+32,FOPR,1.2835,0.1284,Active,1.2362,0.1446
+33,FOPR,1.3167,0.1317,Active,1.2624,0.1451
+34,FOPR,1.3461,0.1346,Active,1.284,0.1459
+35,FOPR,1.3711,0.1371,Active,1.3023,0.1475
+36,FOPR,1.3916,0.1392,Active,1.3179,0.1509
+37,FOPR,1.4074,0.1407,Active,1.3277,0.1546
+38,FOPR,1.4176,0.1418,Active,1.3327,0.1582
+39,FOPR,1.4222,0.1422,Active,1.3328,0.1604
+40,FOPR,1.4236,0.1424,Active,1.3292,0.1621
+41,FOPR,1.4252,0.1425,Active,1.3241,0.1653
+42,FOPR,1.4273,0.1427,Active,1.3176,0.1668
+43,FOPR,1.43,0.143,Active,1.3099,0.1696
+44,FOPR,1.4334,0.1433,Active,1.3012,0.1698
+45,FOPR,1.4377,0.1438,Active,1.289,0.169
+46,FOPR,1.4429,0.1443,Active,1.2754,0.1689
+47,FOPR,1.4489,0.1449,Active,1.2602,0.1672
+48,FOPR,1.4548,0.1455,Active,1.2456,0.167
+49,FOPR,1.4603,0.146,Active,1.2309,0.1665
+50,FOPR,1.4656,0.1466,Active,1.2145,0.1675
+51,FOPR,1.4704,0.147,Active,1.1953,0.1656
+52,FOPR,1.4742,0.1474,Active,1.1737,0.1634
+53,FOPR,1.4753,0.1475,Active,1.1515,0.163
+54,FOPR,1.4738,0.1474,Active,1.1294,0.1662
+55,FOPR,1.4694,0.1469,Active,1.1103,0.1709
+56,FOPR,1.4612,0.1461,Active,1.0907,0.1758
+57,FOPR,1.4495,0.1449,Active,1.0707,0.1826
+58,FOPR,1.4364,0.1436,Active,1.049,0.185
+59,FOPR,1.4214,0.1421,Active,1.0258,0.1881
+60,FOPR,1.4027,0.1403,Active,0.9986,0.1898
+61,FOPR,1.3793,0.1379,Active,0.976,0.1888
+62,FOPR,1.3527,0.1353,Active,0.9499,0.1903
+63,FOPR,1.3241,0.1324,Active,0.9261,0.1937
+64,FOPR,1.2966,0.1297,Active,0.9011,0.197
+65,FOPR,1.2705,0.127,Active,0.8773,0.1988
+66,FOPR,1.2434,0.1243,Active,0.8552,0.1988
+67,FOPR,1.216,0.1216,Active,0.835,0.1979
+68,FOPR,1.1891,0.1189,Active,0.8165,0.1984
+69,FOPR,1.1615,0.1161,Active,0.7949,0.1966
+70,FOPR,1.1344,0.1134,Active,0.7724,0.1952
+71,FOPR,1.1116,0.1112,Active,0.7495,0.1942
+72,FOPR,1.0911,0.1091,Active,0.7256,0.1942
+73,FOPR,1.0719,0.1072,Active,0.7031,0.195
+74,FOPR,1.0527,0.1053,Active,0.6777,0.1951
+75,FOPR,1.0326,0.1033,Active,0.6513,0.1968
+76,FOPR,1.0129,0.1013,Active,0.6226,0.2007
+77,FOPR,0.9946,0.1,Active,0.5976,0.2039
+78,FOPR,0.9755,0.1,Active,0.5714,0.2081
+79,FOPR,0.9557,0.1,Active,0.5499,0.2095
+80,FOPR,0.9363,0.1,Active,0.5285,0.2088
+81,FOPR,0.9158,0.1,Active,0.5092,0.2081
+82,FOPR,0.8928,0.1,Active,0.4917,0.2084
+83,FOPR,0.8687,0.1,Active,0.4751,0.2088
+84,FOPR,0.8423,0.1,Active,0.4587,0.2104
+85,FOPR,0.8125,0.1,Active,0.4425,0.2128
+86,FOPR,0.7789,0.1,Active,0.4275,0.215
+87,FOPR,0.7419,0.1,Active,0.4149,0.2178
+88,FOPR,0.7025,0.1,Active,0.4036,0.2204
+89,FOPR,0.6613,0.1,Active,0.3949,0.2225
+90,FOPR,0.6194,0.1,Active,0.3881,0.2268
+91,FOPR,0.5782,0.1,Active,0.3825,0.2332
+92,FOPR,0.5397,0.1,Active,0.3771,0.239
+93,FOPR,0.5051,0.1,Active,0.3739,0.2464
+94,FOPR,0.4749,0.1,Active,0.3705,0.2508
+95,FOPR,0.4502,0.1,Active,0.3674,0.2523
+96,FOPR,0.4314,0.1,Active,0.3684,0.2547
+97,FOPR,0.4186,0.1,Active,0.3707,0.2556
+98,FOPR,0.41,0.1,Active,0.3694,0.2551
+99,FOPR,0.4057,0.1,Active,0.3709,0.255
+100,FOPR,0.4042,0.1,Active,0.3695,0.2525
+101,FOPR,0.3992,0.1,Active,0.3685,0.2462
+102,FOPR,0.3887,0.1,Active,0.3684,0.2384
+103,FOPR,0.3742,0.1,Active,0.3686,0.2303
+104,FOPR,0.3552,0.1,Active,0.3689,0.2232
+105,FOPR,0.3317,0.1,Active,0.3679,0.2162
+106,FOPR,0.3062,0.1,Active,0.3653,0.2076
+107,FOPR,0.2815,0.1,Active,0.3616,0.1987
+108,FOPR,0.2637,0.1,Active,0.3568,0.1903
+109,FOPR,0.2481,0.1,Active,0.3521,0.1841
+110,FOPR,0.2333,0.1,Active,0.3505,0.1777
+111,FOPR,0.2185,0.1,Active,0.3512,0.1714
+112,FOPR,0.2047,0.1,Active,0.3482,0.1705
+113,FOPR,0.192,0.1,Active,0.3461,0.1692
+114,FOPR,0.1798,0.1,Active,0.3457,0.1671
+115,FOPR,0.1688,0.1,Active,0.3447,0.1661
+116,FOPR,0.1595,0.1,Active,0.3416,0.1665
+117,FOPR,0.1517,0.1,Active,0.3371,0.1672
+118,FOPR,0.1457,0.1,Active,0.3316,0.1685
+119,FOPR,0.1411,0.1,Active,0.3244,0.168
+120,FOPR,0.1373,0.1,Active,0.317,0.1654
+121,FOPR,0.1337,0.1,Active,0.3094,0.1633
+122,FOPR,0.1302,0.1,Active,0.3038,0.1602
+123,FOPR,0.1265,0.1,Active,0.2981,0.1577
+124,FOPR,0.1228,0.1,Active,0.2941,0.1562
+125,FOPR,0.1193,0.1,Active,0.2918,0.1546
+126,FOPR,0.1199,0.1,Active,0.2889,0.1527
+127,FOPR,0.1281,0.1,Active,0.2856,0.1495
+128,FOPR,0.136,0.1,Active,0.2824,0.1437
+129,FOPR,0.1434,0.1,Active,0.2768,0.1379
+130,FOPR,0.1499,0.1,Active,0.2735,0.1342
+131,FOPR,0.1549,0.1,Active,0.2722,0.1324
+132,FOPR,0.1591,0.1,Active,0.2695,0.1322
+133,FOPR,0.1627,0.1,Active,0.2671,0.1337
+134,FOPR,0.1656,0.1,Active,0.2642,0.1338
+135,FOPR,0.167,0.1,Active,0.2632,0.1316
+136,FOPR,0.1672,0.1,Active,0.2625,0.1305
+137,FOPR,0.1661,0.1,Active,0.2614,0.1296
+138,FOPR,0.165,0.1,Active,0.2609,0.1285
+139,FOPR,0.1643,0.1,Active,0.2602,0.1279
+140,FOPR,0.1651,0.1,Active,0.2595,0.1276
+141,FOPR,0.169,0.1,Active,0.2566,0.1262
+142,FOPR,0.1759,0.1,Active,0.2532,0.1238
+143,FOPR,0.1857,0.1,Active,0.2489,0.1216
+144,FOPR,0.1974,0.1,Active,0.2426,0.1188
+145,FOPR,0.2106,0.1,Active,0.2359,0.1156
+146,FOPR,0.2248,0.1,Active,0.2287,0.112
+147,FOPR,0.2392,0.1,Active,0.2242,0.1097
+148,FOPR,0.2524,0.1,Active,0.2191,0.1063
+149,FOPR,0.2641,0.1,Active,0.2132,0.1034
+150,FOPR,0.2747,0.1,Active,0.2094,0.1011
+151,FOPR,0.2849,0.1,Active,0.2059,0.0992
+152,FOPR,0.2945,0.1,Active,0.2032,0.0978
+153,FOPR,0.3028,0.1,Active,0.1993,0.0988
+154,FOPR,0.3088,0.1,Active,0.1971,0.0989
+155,FOPR,0.312,0.1,Active,0.1936,0.0999
+156,FOPR,0.3125,0.1,Active,0.1905,0.0999
+157,FOPR,0.3097,0.1,Active,0.1869,0.0983
+158,FOPR,0.3039,0.1,Active,0.1835,0.0971
+159,FOPR,0.2956,0.1,Active,0.1795,0.096
+160,FOPR,0.2859,0.1,Active,0.1752,0.0948
+161,FOPR,0.2754,0.1,Active,0.171,0.0938
+162,FOPR,0.2642,0.1,Active,0.1663,0.0925
+163,FOPR,0.2527,0.1,Active,0.1612,0.0913
+164,FOPR,0.2412,0.1,Active,0.156,0.0902
+165,FOPR,0.2295,0.1,Active,0.1509,0.0897
+166,FOPR,0.2178,0.1,Active,0.146,0.0894
+167,FOPR,0.2069,0.1,Active,0.141,0.0893
+168,FOPR,0.1965,0.1,Active,0.1359,0.0892
+169,FOPR,0.1869,0.1,Active,0.1308,0.0892
+170,FOPR,0.1776,0.1,Active,0.1258,0.0892
+171,FOPR,0.1683,0.1,Active,0.1212,0.089
+172,FOPR,0.1588,0.1,Active,0.117,0.0887
+173,FOPR,0.1496,0.1,Active,0.1129,0.0881
+174,FOPR,0.1412,0.1,Active,0.1093,0.0878
+175,FOPR,0.1335,0.1,Active,0.1057,0.0875
+176,FOPR,0.1266,0.1,Active,0.1024,0.0874
+177,FOPR,0.1205,0.1,Active,0.0994,0.0874
+178,FOPR,0.1153,0.1,Active,0.0966,0.0873
+179,FOPR,0.1109,0.1,Active,0.0938,0.0869
+180,FOPR,0.1065,0.1,Active,0.091,0.0865
+181,FOPR,0.1015,0.1,Active,0.088,0.0865
+182,FOPR,0.0956,0.1,Active,0.0849,0.0868
+183,FOPR,0.0888,0.1,Active,0.082,0.0874
+184,FOPR,0.0812,0.1,Active,0.0792,0.0881
+185,FOPR,0.0734,0.1,Active,0.0768,0.0888
+186,FOPR,0.0655,0.1,Active,0.0751,0.0895
+187,FOPR,0.0577,0.1,Active,0.0736,0.0902
+188,FOPR,0.0503,0.1,Active,0.0723,0.0909
+189,FOPR,0.0438,0.1,Active,0.0714,0.0916
+190,FOPR,0.0379,0.1,Active,0.0709,0.0922
+191,FOPR,0.0329,0.1,Active,0.0707,0.0929
+192,FOPR,0.029,0.1,Active,0.0706,0.0933
+193,FOPR,0.026,0.1,Active,0.0709,0.0937
+194,FOPR,0.0238,0.1,Active,0.071,0.094
+195,FOPR,0.0222,0.1,Active,0.0713,0.0942
+196,FOPR,0.021,0.1,Active,0.0715,0.0943
+197,FOPR,0.0203,0.1,Active,0.0716,0.0944
+198,FOPR,0.0198,0.1,Active,0.0717,0.0945
+199,FOPR,0.0196,0.1,Active,0.0718,0.0945
+200,WOPR_OP1_108,0.3,0.075,Active,0.1868,0.1205
+201,WOPR_OP1_144,0.2,0.035,Active,0.1262,0.0977
+202,WOPR_OP1_190,0.015,0.01,Active,0.0243,0.0346
+203,WOPR_OP1_36,0.7,0.07,Active,0.6561,0.0681
+204,WOPR_OP1_72,0.5,0.05,Active,0.3852,0.1431
+205,WOPR_OP1_9,0.1,0.05,Active,0.0777,0.0508
+206,WPR_DIFF_1,0.0,0.1,Active,-0.0318,0.0713
+207,WPR_DIFF_1,0.1,0.2,Active,0.0172,0.143
+208,WPR_DIFF_1,0.2,0.15,Active,0.049,0.1755
+209,WPR_DIFF_1,0.0,0.05,Active,0.0451,0.1667