Release 0.8.7

Dockerfile

@@ -108,7 +108,7 @@ ARG ecbuild_version=3.8.2
 ARG eccodes_version=2.33.1
 ARG eckit_version=1.28.0
 ARG fdb_version=5.13.2
-ARG pyfdb_version=0.0.3
+ARG pyfdb_version=0.1.0
 RUN apt update
 # COPY polytope-deployment/common/default_fdb_schema /polytope/config/fdb/default
 
@@ -173,6 +173,7 @@ RUN set -eux && \
 # Install pyfdb \
 RUN set -eux \
     && git clone --single-branch --branch ${pyfdb_version} https://github.com/ecmwf/pyfdb.git \
+    && python -m pip install "numpy<2.0" --user\
     && python -m pip install ./pyfdb --user
 
 #######################################################
@@ -200,7 +201,7 @@ RUN set -eux \
     ls -R /opt
 
 RUN set -eux \
-    && git clone --single-branch --branch develop https://github.com/ecmwf/gribjump.git
+    && git clone --single-branch --branch ${gribjump_version} https://github.com/ecmwf/gribjump.git
 # Install pygribjump
 RUN set -eux \
     && cd /gribjump \
@@ -229,9 +230,13 @@ FROM mars-base AS mars-base-c
 RUN apt update && apt install -y liblapack3 mars-client=${mars_client_c_version} mars-client-cloud
 
 FROM mars-base AS mars-base-cpp
+ARG pyfdb_version=0.1.0
 RUN apt update && apt install -y mars-client-cpp=${mars_client_cpp_version}
 RUN set -eux \
-    && python3 -m pip install git+https://github.com/ecmwf/pyfdb.git@master --user
+    && git clone --single-branch --branch ${pyfdb_version} https://github.com/ecmwf/pyfdb.git \
+    && python -m pip install "numpy<2.0" --user\
+    && python -m pip install ./pyfdb --user
+
 
 FROM blank-base AS blank-base-c
 FROM blank-base AS blank-base-cpp
@@ -342,7 +347,6 @@ COPY --chown=polytope --from=gribjump-base-final /root/.local /home/polytope/.lo
 # Copy python requirements
 COPY --chown=polytope --from=worker-base /root/.venv /home/polytope/.local
 
-
 # Install the server source
 COPY --chown=polytope . /polytope/
 

polytope_server/common/auth.py

@@ -132,16 +132,16 @@ def authenticate(self, auth_header) -> User:
     def is_authorized(self, user, roles):
         """Checks if the user has any of the provided roles"""
 
-        # roles can be a single value; convert to a list
-        if not isinstance(roles, (tuple, list, set)):
-            roles = [roles]
-
         # roles can be a dict of realm:[roles] mapping; find the relevant realm.
         if isinstance(roles, dict):
             if user.realm not in roles:
                 raise ForbiddenRequest("Not authorized to access this resource.")
             roles = roles[user.realm]
 
+        # roles can be a single value; convert to a list
+        if not isinstance(roles, (tuple, list, set)):
+            roles = [roles]
+
         for required_role in roles:
             if required_role in user.roles:
                 return True

polytope_server/common/authorization/ldap_authorization.py

@@ -23,6 +23,7 @@
 from ldap3 import SUBTREE, Connection, Server
 
 from ..auth import User
+from ..caching import cache
 from . import authorization
 
 
@@ -40,6 +41,7 @@ def __init__(self, name, realm, config):
         self.username_attribute = config.get("username-attribute", None)
         super().__init__(name, realm, config)
 
+    @cache(lifetime=120)
     def get_roles(self, user: User) -> list:
         if user.realm != self.realm():
             raise ValueError(

polytope_server/common/datasource/coercion.py

@@ -0,0 +1,251 @@
+import copy
+from datetime import datetime, timedelta
+from typing import Any, Dict
+
+
+class CoercionError(Exception):
+    pass
+
+
+class Coercion:
+
+    allow_ranges = [
+        "number",
+        "step",
+        "date",
+    ]
+    allow_lists = ["class", "stream", "type", "expver", "param", "number", "date", "step"]
+
+    @staticmethod
+    def coerce(request: Dict[str, Any]) -> Dict[str, Any]:
+        request = copy.deepcopy(request)
+        for key, value in request.items():
+            request[key] = Coercion.coerce_value(key, value)
+        return request
+
+    @staticmethod
+    def coerce_value(key: str, value: Any):
+        if key in Coercion.coercer:
+            coercer_func = Coercion.coercer[key]
+
+            if isinstance(value, list):
+                # Coerce each item in the list
+                coerced_values = [Coercion.coerce_value(key, v) for v in value]
+                return "/".join(coerced_values)
+            elif isinstance(value, str):
+
+                if "/to/" in value and key in Coercion.allow_ranges:
+                    # Handle ranges with possible "/by/" suffix
+                    start_value, rest = value.split("/to/", 1)
+                    if not rest:
+                        raise CoercionError(f"Invalid range format for key {key}.")
+
+                    if "/by/" in rest:
+                        end_value, suffix = rest.split("/by/", 1)
+                        suffix = "/by/" + suffix  # Add back the '/by/'
+                    else:
+                        end_value = rest
+                        suffix = ""
+
+                    # Coerce start_value and end_value
+                    start_coerced = coercer_func(start_value)
+                    end_coerced = coercer_func(end_value)
+
+                    return f"{start_coerced}/to/{end_coerced}{suffix}"
+                elif "/" in value and key in Coercion.allow_lists:
+                    # Handle lists
+                    coerced_values = [coercer_func(v) for v in value.split("/")]
+                    return "/".join(coerced_values)
+                else:
+                    # Single value
+                    return coercer_func(value)
+            else:  # not list or string
+                return coercer_func(value)
+        else:
+            if isinstance(value, list):
+                # Join list into '/' separated string
+                coerced_values = [str(v) for v in value]
+                return "/".join(coerced_values)
+            else:
+                return value
+
+    @staticmethod
+    def coerce_date(value: Any) -> str:
+        try:
+            # Attempt to convert the value to an integer
+            int_value = int(value)
+            if int_value > 0:
+                # Positive integers are assumed to be dates in YYYYMMDD format
+                date_str = str(int_value)
+                try:
+                    datetime.strptime(date_str, "%Y%m%d")
+                    return date_str
+                except ValueError:
+                    raise CoercionError("Invalid date format, expected YYYYMMDD or YYYY-MM-DD.")
+            else:
+                # Zero or negative integers represent relative days from today
+                target_date = datetime.today() + timedelta(days=int_value)
+                return target_date.strftime("%Y%m%d")
+        except (ValueError, TypeError):
+            # The value is not an integer or cannot be converted to an integer
+            pass
+
+        if isinstance(value, str):
+            value_stripped = value.strip()
+            # Try parsing as YYYYMMDD
+            try:
+                datetime.strptime(value_stripped, "%Y%m%d")
+                return value_stripped
+            except ValueError:
+                # Try parsing as YYYY-MM-DD
+                try:
+                    date_obj = datetime.strptime(value_stripped, "%Y-%m-%d")
+                    return date_obj.strftime("%Y%m%d")
+                except ValueError:
+                    raise CoercionError("Invalid date format, expected YYYYMMDD or YYYY-MM-DD.")
+        else:
+            raise CoercionError("Invalid date format, expected YYYYMMDD or YYYY-MM-DD.")
+
+    @staticmethod
+    def coerce_step(value: Any) -> str:
+
+        if isinstance(value, int):
+            if value < 0:
+                raise CoercionError("Step must be greater than or equal to 0.")
+            else:
+                return str(value)
+        elif isinstance(value, str):
+            if not value.isdigit() or int(value) < 0:
+                raise CoercionError("Step must be greater than or equal to 0.")
+            return value
+        else:
+            raise CoercionError("Invalid type, expected integer or string.")
+
+    @staticmethod
+    def coerce_number(value: Any) -> str:
+
+        if isinstance(value, int):
+            if value <= 0:
+                raise CoercionError("Number must be a positive value.")
+            else:
+                return str(value)
+        elif isinstance(value, str):
+            if not value.isdigit() or int(value) <= 0:
+                raise CoercionError("Number must be a positive integer.")
+            return value
+        else:
+            raise CoercionError("Invalid type, expected integer or string.")
+
+    @staticmethod
+    def coerce_param(value: Any) -> str:
+        if isinstance(value, int):
+            return str(value)
+        elif isinstance(value, str):
+            return value
+        else:
+            raise CoercionError("Invalid param type, expected integer or string.")
+
+    @staticmethod
+    def coerce_time(value: Any) -> str:
+        if isinstance(value, int):
+            if value < 0:
+                raise CoercionError("Invalid time format, expected HHMM or HH greater than zero.")
+            elif value < 24:
+                # Treat as hour with minute=0
+                hour = value
+                minute = 0
+            elif 100 <= value <= 2359:
+                # Possible HHMM format
+                hour = value // 100
+                minute = value % 100
+            else:
+                raise CoercionError("Invalid time format, expected HHMM or HH.")
+        elif isinstance(value, str):
+            value_stripped = value.strip()
+            # Check for colon-separated time (e.g., "12:00")
+            if ":" in value_stripped:
+                parts = value_stripped.split(":")
+                if len(parts) != 2:
+                    raise CoercionError("Invalid time format, expected HHMM or HH.")
+                hour_str, minute_str = parts
+                if not (hour_str.isdigit() and minute_str.isdigit()):
+                    raise CoercionError("Invalid time format, expected HHMM or HH.")
+                hour = int(hour_str)
+                minute = int(minute_str)
+            else:
+                if value_stripped.isdigit():
+                    num_digits = len(value_stripped)
+                    if num_digits == 4:
+                        # Format is "HHMM"
+                        hour = int(value_stripped[:2])
+                        minute = int(value_stripped[2:])
+                    elif num_digits <= 2:
+                        # Format is "H" or "HH"
+                        hour = int(value_stripped)
+                        minute = 0
+                    else:
+                        raise CoercionError("Invalid time format, expected HHMM or HH.")
+                else:
+                    raise CoercionError("Invalid time format, expected HHMM or HH.")
+        else:
+            raise CoercionError("Invalid type for time, expected string or integer.")
+
+        # Validate hour and minute
+        if not (0 <= hour <= 23):
+            raise CoercionError("Invalid time format, expected HHMM or HH.")
+        if not (0 <= minute <= 59):
+            raise CoercionError("Invalid time format, expected HHMM or HH.")
+        if minute != 0:
+            raise CoercionError("Invalid time format, expected HHMM or HH.")
+
+        # Format time as HHMM
+        time_str = f"{hour:02d}{minute:02d}"
+        return time_str
+
+        # Validate hour and minute
+        if not (0 <= hour <= 23):
+            raise CoercionError("Hour must be between 0 and 23.")
+        if not (0 <= minute <= 59):
+            raise CoercionError("Minute must be between 0 and 59.")
+        if minute != 0:
+            # In your test cases, minute must be zero
+            raise CoercionError("Minute must be zero.")
+
+        # Format time as HHMM
+        time_str = f"{hour:02d}{minute:02d}"
+        return time_str
+
+    @staticmethod
+    def coerce_expver(value: Any) -> str:
+
+        # Integers accepted, converted to 4-length strings
+        if isinstance(value, int):
+            if 0 <= value <= 9999:
+                return f"{value:0>4d}"
+            else:
+                raise CoercionError("expver integer must be between 0 and 9999 inclusive.")
+
+        # Strings accepted if they are convertible to integer or exactly 4 characters long
+        elif isinstance(value, str):
+            if value.isdigit():
+                int_value = int(value.lstrip("0") or "0")
+                if 0 <= int_value <= 9999:
+                    return f"{int_value:0>4d}"
+                else:
+                    raise CoercionError("expver integer string must represent a number between 0 and 9999 inclusive.")
+            elif len(value) == 4:
+                return value
+            else:
+                raise CoercionError("expver string length must be 4 characters exactly.")
+
+        else:
+            raise CoercionError("expver must be an integer or a string.")
+
+    coercer = {
+        "date": coerce_date,
+        "step": coerce_step,
+        "number": coerce_number,
+        "param": coerce_param,
+        "time": coerce_time,
+        "expver": coerce_expver,
+    }

polytope_server/common/datasource/datasource.py

@@ -50,6 +50,10 @@ def match(self, request: str) -> None:
         """Checks if the request matches the datasource, raises on failure"""
         raise NotImplementedError()
 
+    def repr(self) -> str:
+        """Returns a string name of the datasource, presented to the user on error"""
+        raise NotImplementedError
+
     def get_type(self) -> str:
         """Returns a string stating the type of this object (e.g. fdb, mars, echo)"""
         raise NotImplementedError()
@@ -84,9 +88,7 @@ def dispatch(self, request, input_data) -> bool:
             if hasattr(self, "silent_match") and self.silent_match:
                 pass
             else:
-                request.user_message += "Skipping datasource {} due to match error: {}\n".format(
-                    self.get_type(), repr(e)
-                )
+                request.user_message += "Skipping datasource {}: {}\n".format(self.repr(), str(e))
             tb = traceback.format_exception(None, e, e.__traceback__)
             logging.info(tb)
 
@@ -97,7 +99,7 @@ def dispatch(self, request, input_data) -> bool:
             datasource_role_rules = self.config.get("roles", None)
             if datasource_role_rules is not None:
                 if not any(role in request.user.roles for role in datasource_role_rules.get(request.user.realm, [])):
-                    request.user_message += "Skipping datasource {}. User is forbidden.\n".format(self.get_type())
+                    request.user_message += "Skipping datasource {}: user is not authorised.\n".format(self.repr())
                     return False
 
         # Retrieve/Archive/etc.
@@ -111,8 +113,8 @@ def dispatch(self, request, input_data) -> bool:
                 raise NotImplementedError()
 
         except NotImplementedError as e:
-            request.user_message += "Skipping datasource {}. Verb {} not available: {}\n".format(
-                self.get_type(), request.verb, repr(e)
+            request.user_message += "Skipping datasource {}: method '{}' not available: {}\n".format(
+                self.repr(), request.verb, repr(e)
             )
             return False
 

polytope_server/common/datasource/dummy.py

@@ -43,6 +43,9 @@ def retrieve(self, request):
 
         return True
 
+    def repr(self):
+        return self.config.get("repr", "dummy")
+
     def result(self, request):
         chunk_size = 2 * 1024 * 1024
         data_generated = 0

polytope_server/common/datasource/echo.py

@@ -40,6 +40,9 @@ def retrieve(self, request):
             self.data = request.user_request
         return True
 
+    def repr(self):
+        return self.config.get("repr", "echo")
+
     def result(self, request):
         yield self.data