Merge pull request #56 from ecmwf/develop

1.0.1

Author: peshence

.github/workflows/check-publish.yaml

@@ -8,7 +8,7 @@ on:
     tags:
     - '**'
 
-  pull_request: ~
+  # pull_request: ~
 
   workflow_dispatch: ~
 
@@ -24,7 +24,20 @@ jobs:
       - run: isort --check .
       - run: black --check .
       - run: flake8 .
-
+  unit-tests:
+    name: Unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+      - name: Run tests
+        run: pytest tests/unit
   docs:
     if: |
       (github.event_name == 'push' && contains(github.ref, 'refs/tags/')) || 

conftest.py

@@ -72,6 +72,8 @@ def __init__(self, config):
     def authenticate(self, auth_header) -> User:
         """Returns authenticated User, or raises UnauthorizedRequest"""
 
+        logging.info("Authenticating user with header:\n{}".format(auth_header))
+
         headers = auth_header.split(",")
         if len(headers) == 0:
             raise UnauthorizedRequest(
@@ -125,7 +127,7 @@ def authenticate(self, auth_header) -> User:
                 user.roles = list(set.union(set(user.roles), set(authorizer.get_roles(user))))
                 user.attributes.update(authorizer.get_attributes(user))
 
-        logging.debug("User authenticated:\n {}".format(user.serialize()))
+        logging.info("User authenticated:\n {}".format(user.serialize()))
 
         return user
 

polytope_server/common/auth.py

@@ -279,20 +279,20 @@ def parse_relativedelta(self, time_str):
 
         return relativedelta(days=time_dict["d"], hours=time_dict["h"], minutes=time_dict["m"])
 
-    def date_check(self, date, offsets, after=False):
+    def date_check(self, date, offset, after=False):
         """Process special match rules for DATE constraints"""
 
         date = str(date)
 
         # Default date is -1
-        if len(str(date)) == 0:
+        if len(date) == 0:
             date = "-1"
 
         now = datetime.today()
-        offset = now - self.parse_relativedelta(offsets)
+        offset = now - self.parse_relativedelta(offset)
         offset_fmted = offset.strftime("%Y%m%d")
 
-        split = str(date).split("/")
+        split = date.split("/")
 
         # YYYYMMDD
         if len(split) == 1:

polytope_server/common/datasource/mars.py

@@ -170,26 +170,7 @@ def match(self, request):
 
         # Check data released
         if SCHEDULE_READER is not None and self.obey_schedule:
-            # Check if step is in feature
-            if "step" in r:
-                step = r["step"]
-            elif r["feature"]["type"] == "timeseries":
-                step = r["feature"]["range"]["end"]
-            elif r["feature"]["type"] == "trajectory" and "step" in r["feature"]["axes"]:
-                # get index of step in axes, then get max step from trajectory
-                step = r["feature"]["axes"].index("step")
-                step = r["feature"]["points"][step].max()
-            else:
-                raise PolytopeError("'step' not found in request")
-            SCHEDULE_READER.check_released(
-                r["date"],
-                r["class"],
-                r["stream"],
-                r.get("domain", "g"),
-                r["time"],
-                str(step),
-                r["type"],
-            )
+            SCHEDULE_READER.check_released_polytope_request(r)
 
     def destroy(self, request) -> None:
         pass

polytope_server/common/datasource/polytope.py

@@ -59,8 +59,6 @@ def data(self, buffer_size=2 * 1024 * 1024):
         if buffer != b"":
             yield buffer
 
-        # self.delete()
-
     def delete(self):
         """Close and delete FIFO"""
         logging.info("Deleting FIFO.")

polytope_server/common/io/fifo.py

@@ -40,7 +40,7 @@
 
 # Indexable fields
 INDEXABLE_FIELDS = {"request_id": str}
-DEFAULT_LOGGING_MODE = "prettyprint"  # Changed from "json" to "prettyprint" or "console"
+DEFAULT_LOGGING_MODE = "json"
 DEFAULT_LOGGING_LEVEL = "INFO"
 
 
@@ -120,12 +120,14 @@ def format(self, record):
 
         if self.mode == "logserver":
             return self.format_for_logserver(record, result)
-        elif self.mode == "prettyprint":
+        if self.mode == "prettyprint":
             return json.dumps(
                 result, indent=2, ensure_ascii=False
             )  # Added ensure_ascii=False for correct Unicode display
-        else:
+        if self.mode == "json":
             return json.dumps(result, indent=None)
+        # default to json
+        return json.dumps(result, indent=None)
 
 
 def setup(config, source_name):
@@ -142,7 +144,4 @@ def setup(config, source_name):
     logger.addHandler(handler)
     logger.setLevel(level)
 
-    # Lower the logging level for pymongo
-    logging.getLogger("pymongo").setLevel(logging.WARNING)
-
     logger.info("Logging Initialized")

polytope_server/common/logging.py

@@ -45,6 +45,9 @@ def __init__(self, config=None):
         uri = config.get("uri", "mongodb://localhost:27017")
         metric_collection = config.get("collection", "metrics")
 
+        log_level = config.get("log_level", logging.WARNING)
+        logging.getLogger("pymongo").setLevel(log_level)
+
         username = config.get("username")
         password = config.get("password")
 

polytope_server/common/metric_store/mongodb_metric_store.py

@@ -37,6 +37,8 @@ class MongoRequestStore(request_store.RequestStore):
     def __init__(self, config=None, metric_store_config=None):
         uri = config.get("uri", "mongodb://localhost:27017")
         request_collection = config.get("collection", "requests")
+        log_level = config.get("log_level", logging.WARNING)
+        logging.getLogger("pymongo").setLevel(log_level)
         username = config.get("username")
         password = config.get("password")
 

polytope_server/common/request_store/mongodb_request_store.py

@@ -95,6 +95,30 @@ def check_released(
                     # f"Release time is {release_date}."
                 )
 
+    def check_released_polytope_request(self, req):
+        # Check if step is in feature
+        if "step" in req:
+            step = req["step"]
+        elif "feature" not in req:
+            raise PolytopeError("'step' not found in request")
+        elif req["feature"]["type"] == "timeseries":
+            step = req["feature"]["range"]["end"]
+        elif req["feature"]["type"] == "trajectory" and "step" in req["feature"]["axes"]:
+            # get index of step in axes, then get max step from trajectory
+            step = req["feature"]["axes"].index("step")
+            step = max([p[step] for p in req["feature"]["points"]])
+        else:
+            raise PolytopeError("'step' not found in request")
+        self.check_released(
+            req["date"],
+            req["class"],
+            req["stream"],
+            req.get("domain", "g"),
+            req["time"],
+            str(step),
+            req["type"],
+        )
+
     def get_release_time_and_delta_day(
         self, cclass: str, stream: str, domain: str, time_in: str, step: str, ttype: str
     ) -> Tuple[Optional[str], Optional[int]]:

polytope_server/common/schedule.py

@@ -20,4 +20,4 @@
 
 
 # Single-source Polytope version number
-__version__ = "0.8.7"
+__version__ = "1.0.1"

polytope_server/version.py

@@ -71,4 +71,4 @@ def test_datasource_dummy_contains_pattern(self):
         for x in self.ds.result(self.request):
             data += x
         assert len(data) == 13
-        assert data == b"polytopepolyt"
+        assert data == b"xxxxxxxxxxxxx"

tests/unit/test_datasource_dummy.py

@@ -29,9 +29,9 @@ class Test:
     def setup_method(self, method):
         pass
 
-    def test_fifo(self):
+    def test_fifo(self, tmp_path):
 
-        fifo = FIFO("test-fifo")
+        fifo = FIFO("test-fifo", str(tmp_path))
 
         assert not fifo.ready()
 
@@ -62,21 +62,22 @@ def test_fifo(self):
 
         assert count == 6
 
-        # the fifo has been deleted because we finished reading it
+        fifo.delete()
+        # the fifo has been deleted
         with pytest.raises(OSError):
-            assert fifo.ready()
+            assert not fifo.ready()
 
     def read_all(self, fifo, result):
         while not fifo.ready():
             pass
         for x in fifo.data():
             result[0] += x
 
-    def test_fifo_buffered(self):
+    def test_fifo_buffered(self, tmp_path):
 
         # write 1 MiB of data, we need two threads
 
-        fifo = FIFO("test-fifo")
+        fifo = FIFO("test-fifo", str(tmp_path))
 
         data = [b""]
 

tests/unit/test_fifo.py

@@ -18,8 +18,8 @@
 # does it submit to any jurisdiction.
 #
 
+import json
 import logging
-import re
 
 import pytest
 
@@ -41,33 +41,45 @@ def test_logging_format(self):
         import polytope_server.common.logging as mylogging
 
         formatter = mylogging.LogFormatter(mode="json")
-        record = logging.LogRecord("polytope_server.tests.unit", 10, "/hello/world", 500, "Test Message", None, None)
 
         # Normal record
-        result = formatter.format(record)
-        assert re.match(
-            r'[\s\S]*{[\s\S]*"lvl"[\s\S]*:[\s\S]*"DEBUG"[\s\S]*,[\s\S]*"msg"[\s\S]*:[\s\S]*"Test Message"[\s\S]*,[\s\S]*"pth"[\s\S]*:[\s\S]*"/hello/world:500"[\s\S]*,[\s\S]*"src"[\s\S]*:[\s\S]*"polytope_server.tests.unit"[\s\S]*}[\s\S]*',  # noqa
-            result,
+        log_message = {
+            "name": "polytope_server.tests.unit",
+            "filename": "world",
+            "lineno": 500,
+            "levelname": "DEBUG",
+            "message": "Test Message",
+        }
+        record = logging.LogRecord(
+            log_message["name"],
+            logging.getLevelNamesMapping()[log_message["levelname"]],
+            log_message["filename"],
+            log_message["lineno"],
+            log_message["message"],
+            None,
+            None,
         )
+        result = json.loads(formatter.format(record))
+        for k in log_message.keys():
+            assert result[k] == log_message[k]
 
         # Record with valid extra info
         record.request_id = "hello"
-        result = formatter.format(record)
-        assert re.match(
-            r'[\s\S]*{[\s\S]*"lvl"[\s\S]*:[\s\S]*"DEBUG"[\s\S]*,[\s\S]*"msg"[\s\S]*:[\s\S]*"Test Message"[\s\S]*,[\s\S]*"pth"[\s\S]*:[\s\S]*"/hello/world:500"[\s\S]*,[\s\S]*"request_id"[\s\S]*:[\s\S]*"hello"[\s\S]*,[\s\S]*"src"[\s\S]*:[\s\S]*"polytope_server.tests.unit"[\s\S]*}[\s\S]*',  # noqa
-            result,
-        )
+        log_message["request_id"] = "hello"
+        result = json.loads(formatter.format(record))
+        for k in log_message.keys():
+            assert result[k] == log_message[k]
 
         # Record with invalid extra info (wrong type)
         record.request_id = 1234
         with pytest.raises(TypeError):
             result = formatter.format(record)
         del record.request_id
+        del log_message["request_id"]
 
         # Record with unknown extra info (silently ignores extra)
         record.unknown_extra_arg = "hello"
-        result = formatter.format(record)
-        assert re.match(
-            r'[\s\S]*{[\s\S]*"lvl"[\s\S]*:[\s\S]*"DEBUG"[\s\S]*,[\s\S]*"msg"[\s\S]*:[\s\S]*"Test Message"[\s\S]*,[\s\S]*"pth"[\s\S]*:[\s\S]*"/hello/world:500"[\s\S]*,[\s\S]*"src"[\s\S]*:[\s\S]*"polytope_server.tests.unit"[\s\S]*}[\s\S]*',  # noqa
-            result,
-        )
+        result = json.loads(formatter.format(record))
+        for k in log_message.keys():
+            assert result[k] == log_message[k]
+        assert "unknown_extra_arg" not in result

tests/unit/test_logging.py

@@ -40,7 +40,7 @@ def setup_method(self, method):
 
         self.mars_config = {
             "name": "mars",
-            "match": {"class": ["od"], "stream": ["oper", "enfo"], "date": {"months": -1}},
+            "match": {"class": ["od"], "stream": ["oper", "enfo"], "date": "> 30d"},
         }
         self.request = Request()
         self.request.user_request = yaml.dump(
@@ -50,7 +50,7 @@ def setup_method(self, method):
                 "param": "165.128/166.128/167.128",
                 "step": "0",
                 "time": "00",
-                "date": "20200225/to/20200227/by/3",
+                "date": "-32",
                 "type": "an",
                 "class": "od",
                 "expver": "0001",

tests/unit/test_mars_match.py

@@ -37,7 +37,7 @@ def _verify(
 ):
     mock_mongo.assert_called_once()
     args, kwargs = mock_mongo.call_args
-    assert args[0] == endpoint
+    assert kwargs["host"] == endpoint
     if username:
         assert kwargs["username"] == username
     if password:

tests/unit/test_mongo_client_factory.py

@@ -57,9 +57,12 @@ def test_coerce():
             "foo": "bar",
         },
     }
-
-    assert Coercion.coerce(request_mars) == request_out
-    assert Coercion.coerce(request_json) == request_out
+    for r in [request_json, request_mars]:
+        r = Coercion.coerce(r)
+        for key in r:
+            if isinstance(r[key], list):
+                r[key] = "/".join(r[key])
+        assert r == request_out
 
 
 def test_date_coercion():