New waterwebservices

HISTORY.rst

@@ -4,6 +4,7 @@ History
 
 UNRELEASED
 ----------
+* updated to new waterwebservices (ddapi20-waterwebservices.rijkswaterstaat.nl) in https://github.com/Deltares/ddlpy/pull/124
 
 0.7.0 (2025-06-12)
 ------------------

ddlpy/cli.py

@@ -37,6 +37,11 @@ def cli(verbose,  args=None):
     help='Station codes, e.g. HOEKVHLD',
     multiple=True
 )
+@click.option(
+    '--procestype',
+    help='Procestype, e.g. meting, astronomisch, verwachting',
+    multiple=True
+)
 @click.option(
     '--grootheid-code',
     help='Grootheid code, e.g. WATHTE',
@@ -69,6 +74,7 @@ def cli(verbose,  args=None):
 )
 def locations(output,
               station,
+              procestype,
               grootheid_code,
               groepering_code,
               hoedanigheid_code,
@@ -82,7 +88,8 @@ def locations(output,
     locations_df = ddlpy.locations()
 
     stations = station
-    quantities = {'Grootheid.Code': list(grootheid_code),
+    quantities = {'ProcesType':list(procestype),
+                  'Grootheid.Code': list(grootheid_code),
                   'Groepering.Code': list(groepering_code),
                   'Hoedanigheid.Code': list(hoedanigheid_code),
                   'Eenheid.Code': list(eenheid_code),
@@ -128,24 +135,23 @@ def measurements(locations, start_date, end_date):
     except:
         raise ValueError('locations.json file not found. First run "ddlpy locations"')
 
-    for obs in range(locations_df.shape[0]): #goes through rows in table
-        selected = locations_df.loc[obs]
-
+    for irow, selected in locations_df.iterrows(): #goes through rows in table
         measurements = ddlpy.measurements(
             selected, start_date=start_date, end_date=end_date)
 
-        if (len(measurements) > 0):
+        if len(measurements) > 0:
             print('Measurements of %s were obtained' % selected['Code'])
             station = selected['Code']
+            pt = selected['ProcesType']
             cc = selected['Compartiment.Code']
             ec = selected['Eenheid.Code']
             gc = selected['Grootheid.Code']
             grc = selected['Groepering.Code']
             hc = selected['Hoedanigheid.Code']
             pc = selected['Parameter.Code']
 
-            measurements.to_csv('%s_%s_%s_%s_%s_%s_%s.csv' %
-                                (station, cc, ec, gc, grc, hc, pc))
+            measurements.to_csv('%s_%s_%s_%s_%s_%s_%s_%s.csv' %
+                                (station, pt ,cc, ec, gc, grc, hc, pc))
         else:
             print('No Data of station %s were retrieved from Water Info' %
                   selected['Code'])

ddlpy/ddlpy.py

@@ -16,6 +16,7 @@
 
 BASE_URL = "https://waterwebservices.rijkswaterstaat.nl/"
 ENDPOINTS_PATH = pathlib.Path(__file__).with_name("endpoints.json")
+logger = logging.getLogger(__name__)
 
 with ENDPOINTS_PATH.open() as f:
     ENDPOINTS = json.load(f)
@@ -25,36 +26,21 @@ class NoDataError(ValueError):
     pass
 
 
-class UnsuccessfulRequestError(ValueError):
-    pass
-
-
-# Web Feature Service
-# Web Mapping Service
-logger = logging.getLogger(__name__)
-
-
 def _send_post_request(url, request, timeout=None):
     logger.debug("Requesting at {} with request: {}".format(url, json.dumps(request)))
     resp = requests.post(url, json=request, timeout=timeout)
+
     if not resp.ok:
-        raise IOError("Request failed: {}".format(resp.text))
+        # in case of for instance
+        # resp.status_code: 400, resp.reason: Bad Request, resp.text: {"Succesvol":false,"Foutmelding":"Het maximaal aantal waarnemingen (160000) is overschreden. Beperk uw request.","WaarnemingenLijst":[]}
+        # resp.status_code: 500, resp.reason: Internal Server Error
+        raise IOError(f"{resp.status_code} {resp.reason}: {resp.text}")
 
-    result = resp.json()
-    if not result['Succesvol']:
-        logger.debug('Response result is unsuccessful: {}'.format(result))
-        error_message = result.get('Foutmelding', 'No error returned')
-        if error_message == "Geen gegevens gevonden!":
-            # Foutmelding: "Geen gegevens gevonden!"
-            # this is a valid response for periods where there is no data
-            # this error is raised here, but catched in ddlpy.ddlpy.measurements() so the process can continue.
-            raise NoDataError(error_message)
-        else:
-            # Foutmelding: "Het max aantal waarnemingen (157681) is overschreven, beperk uw request."
-            # or any other possible error message are raised here
-            raise UnsuccessfulRequestError(error_message)
+    if resp.status_code==204:
+        # "204 No Content" is raised here, but catched in ddlpy.ddlpy.measurements() so the process can continue.
+        raise NoDataError(f"{resp.status_code} {resp.reason}: {resp.text}")
 
-    # continue if request was successful
+    result = resp.json()
     return result
 
 
@@ -139,13 +125,14 @@ def _get_request_dicts(location):
     # generate aquometadata dict from location "*.Code" values
     key_list = [x.replace(".Code","") for x in location.index if x.endswith(".Code")]
     aquometadata_dict = {key:{"Code":location[f"{key}.Code"]} for key in key_list}
+    # additional code required for ProcesType since this does not adhere to the
+    # Code/Omschrijving convention.
+    if "ProcesType" in location.index:
+        aquometadata_dict["ProcesType"] = location["ProcesType"]
 
     # generate location dict from relevant values
     locatie_dict = {
-        "X": location["X"],
-        "Y": location["Y"],
         # assert code is used as index
-        # TODO: use  a numpy  compatible json encoder in requests
         "Code": location.get("Code", location.name),
     }
 
@@ -261,7 +248,10 @@ def measurements_amount(location:pd.Series, start_date:(str,pd.Timestamp), end_d
         df = df.set_index("Groeperingsperiode")
         df = df[["AantalMetingen"]]
         df_list.append(df)
-
+
+    if len(df_list) == 0:
+        raise NoDataError("no measurements available returned")
+
     # concatenate and sum duplicated index
     df_amount = pd.concat(df_list).sort_index()
     df_amount = df_amount.groupby(df_amount.index).sum()
@@ -279,8 +269,7 @@ def _combine_waarnemingenlijst(result, location):
             new_row = {}
             for key, value in row["WaarnemingMetadata"].items():
                 new_key = "WaarnemingMetadata." + key
-                new_val = value[0] if len(value) == 1 else value
-                new_row[new_key] = new_val
+                new_row[new_key] = value
 
             # add remaining data
             for key, val in row.items():
@@ -289,7 +278,7 @@ def _combine_waarnemingenlijst(result, location):
                 new_row[key] = val
 
             # add metadata
-            for key, val in list(waarneming["AquoMetadata"].items()):
+            for key, val in waarneming["AquoMetadata"].items():
                 if isinstance(val, dict) and "Code" in val and "Omschrijving" in val:
                     # some values have a code/omschrijving pair, flatten them
                     new_key = key + ".Code"
@@ -311,8 +300,8 @@ def _combine_waarnemingenlijst(result, location):
     for name in [
         "Coordinatenstelsel",
         "Naam",
-        "X",
-        "Y",
+        "Lon",
+        "Lat",
     ]:
         df[name] = location[name]
 

ddlpy/endpoints.json

@@ -1,7 +1,7 @@
 {
   "collect_catalogue": {
     "name": "OphalenCatalogus",
-    "url": "https://waterwebservices.rijkswaterstaat.nl/METADATASERVICES_DBO/OphalenCatalogus",
+    "url": "https://ddapi20-waterwebservices.rijkswaterstaat.nl/METADATASERVICES/OphalenCatalogus",
     "type": "POST",
     "request": {
       "CatalogusFilter": {
@@ -10,20 +10,19 @@
         "Hoedanigheden": true,
         "Groeperingen": true,
         "Parameters": true,
+        "ProcesTypes": true,
         "Compartimenten": true
       }
     }
   },
   "collect_observations": {
     "name": "OphalenWaarnemingen",
-    "url": "https://waterwebservices.rijkswaterstaat.nl/ONLINEWAARNEMINGENSERVICES_DBO/OphalenWaarnemingen",
+    "url": "https://ddapi20-waterwebservices.rijkswaterstaat.nl/ONLINEWAARNEMINGENSERVICES/OphalenWaarnemingen",
     "type": "POST",
     "_comment": "Make sure you specify the request location to the exact number of decimals and the time in the exact format (samen number of zeros).",
     "request": {
       "Locatie": {
-        "X": 761899.770959577,
-        "Y": 5915790.48491405,
-        "Code": "DELFZL"
+        "Code": "delfzijl"
       },
       "AquoPlusWaarnemingMetadata": {
         "AquoMetadata": {
@@ -46,7 +45,7 @@
   },
   "collect_latest_observations": {
     "name": "OphalenLaatsteWaarnemingen",
-    "url": "https://waterwebservices.rijkswaterstaat.nl/ONLINEWAARNEMINGENSERVICES_DBO/OphalenLaatsteWaarnemingen",
+    "url": "https://ddapi20-waterwebservices.rijkswaterstaat.nl/ONLINEWAARNEMINGENSERVICES/OphalenLaatsteWaarnemingen",
     "type": "POST",
     "_comment": "Make sure you specify the request location to the exact number of decimals.",
     "request": {
@@ -67,16 +66,14 @@
       ],
       "LocatieLijst": [
         {
-          "X": 518882.333320247,
-          "Y": 5760829.11729589,
-          "Code": "EURPFM"
+          "Code": "europlatform"
         }
       ]
     }
   },
   "check_observations_available": {
     "name": "CheckWaarnemingenAanwezig",
-    "url": "https://waterwebservices.rijkswaterstaat.nl/ONLINEWAARNEMINGENSERVICES_DBO/CheckWaarnemingenAanwezig",
+    "url": "https://ddapi20-waterwebservices.rijkswaterstaat.nl/ONLINEWAARNEMINGENSERVICES/CheckWaarnemingenAanwezig",
     "type": "POST",
     "_comment": "Make sure you specify the request location to the exact number of decimals and the time in the exact format (samen number of zeros).",
     "request": {
@@ -92,9 +89,7 @@
       ],
       "LocatieLijst": [
         {
-          "X": 518882.333320247,
-          "Y": 5760829.11729589,
-          "Code": "EURPFM"
+          "Code": "europlatform"
         }
       ],
       "Periode": {
@@ -105,7 +100,7 @@
   },
   "collect_number_of_observations": {
     "name": "OphalenAantalWaarnemingen",
-    "url": "https://waterwebservices.rijkswaterstaat.nl/ONLINEWAARNEMINGENSERVICES_DBO/OphalenAantalWaarnemingen",
+    "url": "https://ddapi20-waterwebservices.rijkswaterstaat.nl/ONLINEWAARNEMINGENSERVICES/OphalenAantalWaarnemingen",
     "type": "POST",
     "request": {
       "AquoMetadataLijst": [
@@ -117,9 +112,7 @@
       "Groeperingsperiode": "Week",
       "LocatieLijst": [
         {
-          "X": 518882.333320247,
-          "Y": 5760829.11729589,
-          "Code": "EURPFM"
+          "Code": "europlatform"
         }
       ],
       "Periode": {
@@ -130,7 +123,7 @@
   },
   "request_bulk_observations": {
     "name": "AanvragenBulkWaarnemingen",
-    "url": "https://waterwebservices.rijkswaterstaat.nl/BULKWAARNEMINGSERVICES_DBO/AanvragenBulkWaarnemingen",
+    "url": "https://ddapi20-waterwebservices.rijkswaterstaat.nl/BULKWAARNEMINGSERVICES/AanvragenBulkWaarnemingen",
     "type": "POST",
     "_comment": "Make sure you specify the request location to the exact number of decimals and the time in the exact format (samen number of zeros).",
     "request": {
@@ -172,19 +165,13 @@
         ],
         "LocatieLijst": [
           {
-            "X": 742469.913149676,
-            "Y": 5940708.14824459,
-            "Code": "HUIBGOT"
+            "Code": "huibertgat.oost"
           },
           {
-            "X": 595875.376191307,
-            "Y": 5790952.82210343,
-            "Code": "NOORDWK2"
+            "Code": "noordwijk.2kmuitdekust.flachsee"
           },
           {
-            "X": 571670.054611366,
-            "Y": 5822651.05560318,
-            "Code": "IJMDMNTSPS"
+            "Code": "ijmuiden.munitiestort.1"
           }
         ],
         "Periode": {

ddlpy/utils.py

@@ -98,7 +98,7 @@ def dataframe_to_xarray(df: pd.DataFrame, drop_if_constant=[]):
     cols_onlynvt_code = [x for x in cols_onlynvt_code if x.endswith(".Code")]
 
     # create list of location columns, will be dropped (added as ds.attrs)
-    cols_location = ['Code', 'Naam', 'Coordinatenstelsel', 'X', 'Y']
+    cols_location = ['Code', 'Naam', 'Coordinatenstelsel', 'Lon', 'Lat']
 
     # add drop_if_constant colums to list if values are indeed constant, will be dropped (added as ds.attrs)
     cols_constant = []

docs/examples/minimal_example.py

@@ -10,14 +10,17 @@
 
 #select a set of parameters 
 # Filter the locations dataframe with the desired parameters and stations.
-bool_stations = locations.index.isin(['IJMDBTHVN', 'DANTZGZD','HOEKVHLD'])
-# measured (WATHTE) versus computed/astro
+bool_stations = locations.index.isin(['ijmuiden.buitenhaven', 'dantziggat.zuid', 'hoekvanholland', 'ameland.nes'])
+# meting/astronomisch/verwachting
+bool_procestype = locations['ProcesType'].isin(['meting'])
+# waterlevel/waterhoogte (WATHTE)
 bool_grootheid = locations['Grootheid.Code'].isin(['WATHTE'])
 # timeseries (NVT) versus extremes
-bool_groepering = locations['Groepering.Code'].isin(['NVT'])
+bool_groepering = locations['Groepering.Code'].isin([''])
 # vertical reference (NAP/MSL)
 bool_hoedanigheid = locations['Hoedanigheid.Code'].isin(['NAP'])
-selected = locations.loc[bool_stations & bool_grootheid & 
+selected = locations.loc[bool_procestype &
+                         bool_stations & bool_grootheid & 
                          bool_groepering & bool_hoedanigheid]
 
 start_date = dt.datetime(2023, 1, 1)

docs/examples/retrieve_parallel_to_netcdf.py

@@ -13,7 +13,6 @@
 
 
 def get_data(location, start_date, end_date, dir_output, overwrite=True):
-
     station_id = location.name
     station_messageid = location["Locatie_MessageID"]
     filename = os.path.join(dir_output, f"{station_id}-{station_messageid}.nc")
@@ -44,19 +43,18 @@ def get_data(location, start_date, end_date, dir_output, overwrite=True):
     ds.to_netcdf(filename, format="NETCDF4_CLASSIC")
 
 
-if ( __name__ == "__main__" ):
-
+if __name__ == "__main__":
     dir_output = './ddl_retrieved_data'
     os.makedirs(dir_output, exist_ok=True)
 
     # get locations
     locations = ddlpy.locations()
-    bool_stations = locations.index.isin(['IJMDBTHVN', 'DANTZGZD', 'HOEKVHLD', 'VLISSGN', 'HOEK', 'VLIS', "OLST"])
-    bool_grootheid = locations['Grootheid.Code'].isin(['WATHTE']) # measured (WATHTE) versus computed/astro
-    bool_groepering = locations['Groepering.Code'].isin(['NVT']) # timeseries (NVT) versus extremes
+    bool_stations = locations.index.isin(['ijmuiden.buitenhaven', 'dantziggat.zuid', 'hoekvanholland', 'ameland.nes', 'vlissingen', 'olst'])
+    bool_procestype = locations['ProcesType'].isin(['meting']) # meting/astronomisch/verwachting
+    bool_grootheid = locations['Grootheid.Code'].isin(['WATHTE']) # waterlevel (WATHTE)
+    bool_groepering = locations['Groepering.Code'].isin(['']) # timeseries (NVT) versus extremes
     bool_hoedanigheid = locations['Hoedanigheid.Code'].isin(['NAP']) # vertical reference (NAP/MSL)
-    selected = locations.loc[bool_stations & bool_grootheid & bool_groepering & bool_hoedanigheid]
-
+    selected = locations.loc[bool_stations & bool_procestype & bool_grootheid & bool_groepering & bool_hoedanigheid]
 
     start_date = dt.datetime(2022, 1, 1)
     end_date = dt.datetime(2022, 3, 1)
@@ -70,7 +68,6 @@ def get_data(location, start_date, end_date, dir_output, overwrite=True):
         for station_code, location in selected.iterrows():
             executor.submit(get_data, location, start_date, end_date, dir_output)
 
-
     file_list = glob.glob(os.path.join(dir_output, "*.nc"))
     fig, ax = plt.subplots()
     for file_nc in file_list: