Skip to content

Commit 6149fa7

Browse files
authored
Merge pull request #45 from ecmwf/develop
Release 0.8.7
2 parents 93c2a43 + 5794a9b commit 6149fa7

24 files changed

+1045
-44
lines changed

Dockerfile

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ ARG ecbuild_version=3.8.2
108108
ARG eccodes_version=2.33.1
109109
ARG eckit_version=1.28.0
110110
ARG fdb_version=5.13.2
111-
ARG pyfdb_version=0.0.3
111+
ARG pyfdb_version=0.1.0
112112
RUN apt update
113113
# COPY polytope-deployment/common/default_fdb_schema /polytope/config/fdb/default
114114

@@ -173,6 +173,7 @@ RUN set -eux && \
173173
# Install pyfdb \
174174
RUN set -eux \
175175
&& git clone --single-branch --branch ${pyfdb_version} https://github.com/ecmwf/pyfdb.git \
176+
&& python -m pip install "numpy<2.0" --user\
176177
&& python -m pip install ./pyfdb --user
177178

178179
#######################################################
@@ -200,7 +201,7 @@ RUN set -eux \
200201
ls -R /opt
201202

202203
RUN set -eux \
203-
&& git clone --single-branch --branch develop https://github.com/ecmwf/gribjump.git
204+
&& git clone --single-branch --branch ${gribjump_version} https://github.com/ecmwf/gribjump.git
204205
# Install pygribjump
205206
RUN set -eux \
206207
&& cd /gribjump \
@@ -229,9 +230,13 @@ FROM mars-base AS mars-base-c
229230
RUN apt update && apt install -y liblapack3 mars-client=${mars_client_c_version} mars-client-cloud
230231

231232
FROM mars-base AS mars-base-cpp
233+
ARG pyfdb_version=0.1.0
232234
RUN apt update && apt install -y mars-client-cpp=${mars_client_cpp_version}
233235
RUN set -eux \
234-
&& python3 -m pip install git+https://github.com/ecmwf/pyfdb.git@master --user
236+
&& git clone --single-branch --branch ${pyfdb_version} https://github.com/ecmwf/pyfdb.git \
237+
&& python -m pip install "numpy<2.0" --user\
238+
&& python -m pip install ./pyfdb --user
239+
235240

236241
FROM blank-base AS blank-base-c
237242
FROM blank-base AS blank-base-cpp
@@ -342,7 +347,6 @@ COPY --chown=polytope --from=gribjump-base-final /root/.local /home/polytope/.lo
342347
# Copy python requirements
343348
COPY --chown=polytope --from=worker-base /root/.venv /home/polytope/.local
344349

345-
346350
# Install the server source
347351
COPY --chown=polytope . /polytope/
348352

polytope_server/common/auth.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -132,16 +132,16 @@ def authenticate(self, auth_header) -> User:
132132
def is_authorized(self, user, roles):
133133
"""Checks if the user has any of the provided roles"""
134134

135-
# roles can be a single value; convert to a list
136-
if not isinstance(roles, (tuple, list, set)):
137-
roles = [roles]
138-
139135
# roles can be a dict of realm:[roles] mapping; find the relevant realm.
140136
if isinstance(roles, dict):
141137
if user.realm not in roles:
142138
raise ForbiddenRequest("Not authorized to access this resource.")
143139
roles = roles[user.realm]
144140

141+
# roles can be a single value; convert to a list
142+
if not isinstance(roles, (tuple, list, set)):
143+
roles = [roles]
144+
145145
for required_role in roles:
146146
if required_role in user.roles:
147147
return True

polytope_server/common/authorization/ldap_authorization.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
from ldap3 import SUBTREE, Connection, Server
2424

2525
from ..auth import User
26+
from ..caching import cache
2627
from . import authorization
2728

2829

@@ -40,6 +41,7 @@ def __init__(self, name, realm, config):
4041
self.username_attribute = config.get("username-attribute", None)
4142
super().__init__(name, realm, config)
4243

44+
@cache(lifetime=120)
4345
def get_roles(self, user: User) -> list:
4446
if user.realm != self.realm():
4547
raise ValueError(
Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
import copy
2+
from datetime import datetime, timedelta
3+
from typing import Any, Dict
4+
5+
6+
class CoercionError(Exception):
7+
pass
8+
9+
10+
class Coercion:
11+
12+
allow_ranges = [
13+
"number",
14+
"step",
15+
"date",
16+
]
17+
allow_lists = ["class", "stream", "type", "expver", "param", "number", "date", "step"]
18+
19+
@staticmethod
20+
def coerce(request: Dict[str, Any]) -> Dict[str, Any]:
21+
request = copy.deepcopy(request)
22+
for key, value in request.items():
23+
request[key] = Coercion.coerce_value(key, value)
24+
return request
25+
26+
@staticmethod
27+
def coerce_value(key: str, value: Any):
28+
if key in Coercion.coercer:
29+
coercer_func = Coercion.coercer[key]
30+
31+
if isinstance(value, list):
32+
# Coerce each item in the list
33+
coerced_values = [Coercion.coerce_value(key, v) for v in value]
34+
return "/".join(coerced_values)
35+
elif isinstance(value, str):
36+
37+
if "/to/" in value and key in Coercion.allow_ranges:
38+
# Handle ranges with possible "/by/" suffix
39+
start_value, rest = value.split("/to/", 1)
40+
if not rest:
41+
raise CoercionError(f"Invalid range format for key {key}.")
42+
43+
if "/by/" in rest:
44+
end_value, suffix = rest.split("/by/", 1)
45+
suffix = "/by/" + suffix # Add back the '/by/'
46+
else:
47+
end_value = rest
48+
suffix = ""
49+
50+
# Coerce start_value and end_value
51+
start_coerced = coercer_func(start_value)
52+
end_coerced = coercer_func(end_value)
53+
54+
return f"{start_coerced}/to/{end_coerced}{suffix}"
55+
elif "/" in value and key in Coercion.allow_lists:
56+
# Handle lists
57+
coerced_values = [coercer_func(v) for v in value.split("/")]
58+
return "/".join(coerced_values)
59+
else:
60+
# Single value
61+
return coercer_func(value)
62+
else: # not list or string
63+
return coercer_func(value)
64+
else:
65+
if isinstance(value, list):
66+
# Join list into '/' separated string
67+
coerced_values = [str(v) for v in value]
68+
return "/".join(coerced_values)
69+
else:
70+
return value
71+
72+
@staticmethod
73+
def coerce_date(value: Any) -> str:
74+
try:
75+
# Attempt to convert the value to an integer
76+
int_value = int(value)
77+
if int_value > 0:
78+
# Positive integers are assumed to be dates in YYYYMMDD format
79+
date_str = str(int_value)
80+
try:
81+
datetime.strptime(date_str, "%Y%m%d")
82+
return date_str
83+
except ValueError:
84+
raise CoercionError("Invalid date format, expected YYYYMMDD or YYYY-MM-DD.")
85+
else:
86+
# Zero or negative integers represent relative days from today
87+
target_date = datetime.today() + timedelta(days=int_value)
88+
return target_date.strftime("%Y%m%d")
89+
except (ValueError, TypeError):
90+
# The value is not an integer or cannot be converted to an integer
91+
pass
92+
93+
if isinstance(value, str):
94+
value_stripped = value.strip()
95+
# Try parsing as YYYYMMDD
96+
try:
97+
datetime.strptime(value_stripped, "%Y%m%d")
98+
return value_stripped
99+
except ValueError:
100+
# Try parsing as YYYY-MM-DD
101+
try:
102+
date_obj = datetime.strptime(value_stripped, "%Y-%m-%d")
103+
return date_obj.strftime("%Y%m%d")
104+
except ValueError:
105+
raise CoercionError("Invalid date format, expected YYYYMMDD or YYYY-MM-DD.")
106+
else:
107+
raise CoercionError("Invalid date format, expected YYYYMMDD or YYYY-MM-DD.")
108+
109+
@staticmethod
110+
def coerce_step(value: Any) -> str:
111+
112+
if isinstance(value, int):
113+
if value < 0:
114+
raise CoercionError("Step must be greater than or equal to 0.")
115+
else:
116+
return str(value)
117+
elif isinstance(value, str):
118+
if not value.isdigit() or int(value) < 0:
119+
raise CoercionError("Step must be greater than or equal to 0.")
120+
return value
121+
else:
122+
raise CoercionError("Invalid type, expected integer or string.")
123+
124+
@staticmethod
125+
def coerce_number(value: Any) -> str:
126+
127+
if isinstance(value, int):
128+
if value <= 0:
129+
raise CoercionError("Number must be a positive value.")
130+
else:
131+
return str(value)
132+
elif isinstance(value, str):
133+
if not value.isdigit() or int(value) <= 0:
134+
raise CoercionError("Number must be a positive integer.")
135+
return value
136+
else:
137+
raise CoercionError("Invalid type, expected integer or string.")
138+
139+
@staticmethod
140+
def coerce_param(value: Any) -> str:
141+
if isinstance(value, int):
142+
return str(value)
143+
elif isinstance(value, str):
144+
return value
145+
else:
146+
raise CoercionError("Invalid param type, expected integer or string.")
147+
148+
@staticmethod
149+
def coerce_time(value: Any) -> str:
150+
if isinstance(value, int):
151+
if value < 0:
152+
raise CoercionError("Invalid time format, expected HHMM or HH greater than zero.")
153+
elif value < 24:
154+
# Treat as hour with minute=0
155+
hour = value
156+
minute = 0
157+
elif 100 <= value <= 2359:
158+
# Possible HHMM format
159+
hour = value // 100
160+
minute = value % 100
161+
else:
162+
raise CoercionError("Invalid time format, expected HHMM or HH.")
163+
elif isinstance(value, str):
164+
value_stripped = value.strip()
165+
# Check for colon-separated time (e.g., "12:00")
166+
if ":" in value_stripped:
167+
parts = value_stripped.split(":")
168+
if len(parts) != 2:
169+
raise CoercionError("Invalid time format, expected HHMM or HH.")
170+
hour_str, minute_str = parts
171+
if not (hour_str.isdigit() and minute_str.isdigit()):
172+
raise CoercionError("Invalid time format, expected HHMM or HH.")
173+
hour = int(hour_str)
174+
minute = int(minute_str)
175+
else:
176+
if value_stripped.isdigit():
177+
num_digits = len(value_stripped)
178+
if num_digits == 4:
179+
# Format is "HHMM"
180+
hour = int(value_stripped[:2])
181+
minute = int(value_stripped[2:])
182+
elif num_digits <= 2:
183+
# Format is "H" or "HH"
184+
hour = int(value_stripped)
185+
minute = 0
186+
else:
187+
raise CoercionError("Invalid time format, expected HHMM or HH.")
188+
else:
189+
raise CoercionError("Invalid time format, expected HHMM or HH.")
190+
else:
191+
raise CoercionError("Invalid type for time, expected string or integer.")
192+
193+
# Validate hour and minute
194+
if not (0 <= hour <= 23):
195+
raise CoercionError("Invalid time format, expected HHMM or HH.")
196+
if not (0 <= minute <= 59):
197+
raise CoercionError("Invalid time format, expected HHMM or HH.")
198+
if minute != 0:
199+
raise CoercionError("Invalid time format, expected HHMM or HH.")
200+
201+
# Format time as HHMM
202+
time_str = f"{hour:02d}{minute:02d}"
203+
return time_str
204+
205+
# Validate hour and minute
206+
if not (0 <= hour <= 23):
207+
raise CoercionError("Hour must be between 0 and 23.")
208+
if not (0 <= minute <= 59):
209+
raise CoercionError("Minute must be between 0 and 59.")
210+
if minute != 0:
211+
# In your test cases, minute must be zero
212+
raise CoercionError("Minute must be zero.")
213+
214+
# Format time as HHMM
215+
time_str = f"{hour:02d}{minute:02d}"
216+
return time_str
217+
218+
@staticmethod
219+
def coerce_expver(value: Any) -> str:
220+
221+
# Integers accepted, converted to 4-length strings
222+
if isinstance(value, int):
223+
if 0 <= value <= 9999:
224+
return f"{value:0>4d}"
225+
else:
226+
raise CoercionError("expver integer must be between 0 and 9999 inclusive.")
227+
228+
# Strings accepted if they are convertible to integer or exactly 4 characters long
229+
elif isinstance(value, str):
230+
if value.isdigit():
231+
int_value = int(value.lstrip("0") or "0")
232+
if 0 <= int_value <= 9999:
233+
return f"{int_value:0>4d}"
234+
else:
235+
raise CoercionError("expver integer string must represent a number between 0 and 9999 inclusive.")
236+
elif len(value) == 4:
237+
return value
238+
else:
239+
raise CoercionError("expver string length must be 4 characters exactly.")
240+
241+
else:
242+
raise CoercionError("expver must be an integer or a string.")
243+
244+
coercer = {
245+
"date": coerce_date,
246+
"step": coerce_step,
247+
"number": coerce_number,
248+
"param": coerce_param,
249+
"time": coerce_time,
250+
"expver": coerce_expver,
251+
}

polytope_server/common/datasource/datasource.py

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,10 @@ def match(self, request: str) -> None:
5050
"""Checks if the request matches the datasource, raises on failure"""
5151
raise NotImplementedError()
5252

53+
def repr(self) -> str:
54+
"""Returns a string name of the datasource, presented to the user on error"""
55+
raise NotImplementedError
56+
5357
def get_type(self) -> str:
5458
"""Returns a string stating the type of this object (e.g. fdb, mars, echo)"""
5559
raise NotImplementedError()
@@ -84,9 +88,7 @@ def dispatch(self, request, input_data) -> bool:
8488
if hasattr(self, "silent_match") and self.silent_match:
8589
pass
8690
else:
87-
request.user_message += "Skipping datasource {} due to match error: {}\n".format(
88-
self.get_type(), repr(e)
89-
)
91+
request.user_message += "Skipping datasource {}: {}\n".format(self.repr(), str(e))
9092
tb = traceback.format_exception(None, e, e.__traceback__)
9193
logging.info(tb)
9294

@@ -97,7 +99,7 @@ def dispatch(self, request, input_data) -> bool:
9799
datasource_role_rules = self.config.get("roles", None)
98100
if datasource_role_rules is not None:
99101
if not any(role in request.user.roles for role in datasource_role_rules.get(request.user.realm, [])):
100-
request.user_message += "Skipping datasource {}. User is forbidden.\n".format(self.get_type())
102+
request.user_message += "Skipping datasource {}: user is not authorised.\n".format(self.repr())
101103
return False
102104

103105
# Retrieve/Archive/etc.
@@ -111,8 +113,8 @@ def dispatch(self, request, input_data) -> bool:
111113
raise NotImplementedError()
112114

113115
except NotImplementedError as e:
114-
request.user_message += "Skipping datasource {}. Verb {} not available: {}\n".format(
115-
self.get_type(), request.verb, repr(e)
116+
request.user_message += "Skipping datasource {}: method '{}' not available: {}\n".format(
117+
self.repr(), request.verb, repr(e)
116118
)
117119
return False
118120

polytope_server/common/datasource/dummy.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ def retrieve(self, request):
4343

4444
return True
4545

46+
def repr(self):
47+
return self.config.get("repr", "dummy")
48+
4649
def result(self, request):
4750
chunk_size = 2 * 1024 * 1024
4851
data_generated = 0

polytope_server/common/datasource/echo.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ def retrieve(self, request):
4040
self.data = request.user_request
4141
return True
4242

43+
def repr(self):
44+
return self.config.get("repr", "echo")
45+
4346
def result(self, request):
4447
yield self.data
4548

0 commit comments

Comments
 (0)