Merge pull request #18 from ess-dmsc/service

extracted server for service.

README.md

@@ -12,6 +12,27 @@ pip install -r requirements-dev.txt
 pre-commit install
 ```
 
+## Usage
+
+There are two parts to the program: a service for downloading the data from the proposal system and storing it locally,
+and a client for extracting information from the local store based on queries.
+
+Running the service:
+```
+python bin/yuos.py -u https://useroffice-test.esss.lu.se/graphql -i YMIR -c /opt/yuos/cached_proposals.json
+```
+Typically this would be run as a service.
+NOTE: requires an authenticating token environment variable.
+
+Using the client:
+```
+from yuos_query.yuos_client import YuosCacheClient
+
+client = YuosCacheClient.create('/opt/yuos/cached_proposals.json')
+client.update_cache()
+proposal = client.proposal_by_id("199842")
+```
+
 ## Tests
 ### Programmer tests
 These tests live in the `tests` directory and can be run directly from the main directory using pytest.
@@ -39,14 +60,4 @@ changed.**
 
 **Jenkins will run the tests against the real system automatically for pull requests.**
 
-**Jenkins will also run the tests on main daily, so if there are breaking any API changes we will know about it.**
-
-## Example usage
-
-```
-import os
-from yuos_query.yuos_client import YuosClient
-
-client = YuosClient("some url", os.environ["YUOS_TOKEN"], instrument_name, "/path/to/cache.json")
-proposal_info = client.proposal_by_id(proposal_id)
-```
+**Jenkins will also run the tests on main daily, so if there are any breaking API changes we will know about it.**

bin/yuos.py

@@ -0,0 +1,68 @@
+import argparse
+import logging
+import os
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+from yuos_query import YuosServer
+
+
+def main(url, instrument, cache_filepath, update_interval=3200):
+    while True:
+        try:
+            YuosServer.create(
+                url, os.environ.get("YUOS_TOKEN"), instrument, cache_filepath
+            ).update_cache()
+            logging.info("updated cache")
+            time.sleep(update_interval)
+        except RuntimeError as error:
+            logging.error(f"failed to update cache {error}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    required_args = parser.add_argument_group("required arguments")
+    required_args.add_argument(
+        "-u",
+        "--url",
+        type=str,
+        help="the URL for the proposal system",
+        required=True,
+    )
+
+    required_args.add_argument(
+        "-i", "--instrument", type=str, help="the instrument name", required=True
+    )
+
+    parser.add_argument(
+        "-c",
+        "--cache-filepath",
+        type=str,
+        help="where to write the data",
+        required=True,
+    )
+
+    parser.add_argument(
+        "-l",
+        "--log-level",
+        type=int,
+        default=3,
+        help="sets the logging level: debug=1, info=2, warning=3, error=4, critical=5.",
+    )
+
+    args = parser.parse_args()
+
+    if 1 <= args.log_level <= 5:
+        logging.basicConfig(
+            format="%(asctime)s - %(message)s", level=args.log_level * 10
+        )
+    else:
+        logging.basicConfig(format="%(asctime)s - %(message)s", level=logging.INFO)
+
+    main(
+        args.url,
+        args.instrument,
+        args.cache_filepath,
+    )

integration_tests/test_end_to_end.py

@@ -3,7 +3,7 @@
 
 import pytest
 
-from yuos_query.yuos_client import YuosClient
+from yuos_query.yuos_client import YuosCacheClient, YuosServer
 
 # These tests are skipped if the YUOS_TOKEN environment variable is not defined
 SKIP_TEST = True
@@ -20,9 +20,13 @@
 )
 def test_get_proposals_and_sample_for_specific_id_on_ymir_instrument():
     with TemporaryDirectory() as directory:
-        client = YuosClient(
+        server = YuosServer.create(
             SERVER_URL, YUOS_TOKEN, "YMIR", os.path.join(directory, "cache.json")
         )
+        server.update_cache()
+
+        client = YuosCacheClient.create(os.path.join(directory, "cache.json"))
+        client.update_cache()
 
         result = client.proposal_by_id(KNOWN_PROPOSAL)
 
@@ -61,9 +65,13 @@ def test_get_proposals_and_sample_for_specific_id_on_ymir_instrument():
 )
 def test_get_proposals_for_specific_fed_id_on_ymir_instrument():
     with TemporaryDirectory() as directory:
-        client = YuosClient(
+        server = YuosServer.create(
             SERVER_URL, YUOS_TOKEN, "YMIR", os.path.join(directory, "cache.json")
         )
+        server.update_cache()
+
+        client = YuosCacheClient.create(os.path.join(directory, "cache.json"))
+        client.update_cache()
 
         results = client.proposals_for_user("jonathantaylor")
         assert len(results) == 6

tests/test_yuos_client.py

@@ -2,16 +2,17 @@
 
 import pytest
 
+from example_data import get_ymir_example_data
 from yuos_query.data_classes import ProposalInfo, SampleInfo, User
 from yuos_query.exceptions import (
     DataUnavailableException,
-    ImportCacheException,
     InvalidIdException,
     ServerException,
 )
 from yuos_query.file_cache import FileCache
 from yuos_query.proposal_system import ProposalRequester
-from yuos_query.yuos_client import YuosClient
+from yuos_query.utils import serialise_proposals_to_json
+from yuos_query.yuos_client import YuosCacheClient, YuosServer
 
 VALID_PROPOSAL_DATA = {
     "471120": ProposalInfo(
@@ -54,58 +55,85 @@
 }
 
 
-class TestYuosClient:
+class TestYuosServer:
     @pytest.fixture(autouse=True)
     def prepare(self):
         self.cache = mock.create_autospec(FileCache)
         self.system = mock.create_autospec(ProposalRequester)
 
-    def create_client(self, cache=None, update_cache=True):
+    def create_server(self, cache=None):
         if not cache:
             cache = self.cache
 
-        return YuosClient(
-            ":: url ::",
-            ":: token ::",
-            "YMIR",
-            ":: file ::",
-            update_cache=update_cache,
-            cache=cache,
-            system=self.system,
-        )
+        return YuosServer("YMIR", cache, self.system)
 
-    def test_on_construction_proposal_system_called_and_cache_updated(self):
-        _ = self.create_client()
+    def test_on_update_proposal_system_called_and_cache_updated(self):
+        server = self.create_server()
+        server.update_cache()
 
         self.system.get_proposals_for_instrument.assert_called_once()
         self.cache.update.assert_called_once()
         self.cache.export_to_file.assert_called_once()
 
-    def test_on_refreshing_cache_proposal_system_called_and_cache_updated(self):
-        _ = self.create_client()
+    def test_if_proposal_system_unavailable_then_raises(self):
+        server = self.create_server()
+        server.update_cache()
 
-        self.system.get_proposals_for_instrument.assert_called_once()
-        self.cache.update.assert_called_once()
+        self.system.get_proposals_for_instrument.side_effect = ServerException("oops")
+
+        with pytest.raises(DataUnavailableException):
+            _ = server.update_cache()
+
+
+class TestYuosCacheClient:
+    class InMemoryCache(FileCache):
+        def _read_file(self):
+            return get_ymir_example_data()
+
+    @pytest.fixture(autouse=True)
+    def prepare(self):
+        self.cache = FileCache("::filepath::")
+        self.cache._read_file = lambda: serialise_proposals_to_json(VALID_PROPOSAL_DATA)
+
+    def create_client(
+        self,
+        cache=None,
+    ):
+        if not cache:
+            cache = self.cache
+
+        return YuosCacheClient(
+            cache,
+        )
 
     def test_querying_with_id_that_does_not_conform_to_pattern_raises(
         self,
     ):
         client = self.create_client()
+        client.update_cache()
 
         with pytest.raises(InvalidIdException):
             client.proposal_by_id("abc")
 
     def test_querying_for_unknown_proposal_id_returns_nothing(self):
-        self.cache.proposals = VALID_PROPOSAL_DATA
-
         client = self.create_client()
+        client.update_cache()
 
         assert client.proposal_by_id("00000") is None
 
-    def test_querying_for_proposal_by_id_gives_proposal_info(self):
-        self.cache.proposals = VALID_PROPOSAL_DATA
+    def test_if_cache_file_missing_then_raises(
+        self,
+    ):
+        client = self.create_client()
+        self.cache._read_file = lambda: (_ for _ in ()).throw(FileNotFoundError())
+
+        with pytest.raises(DataUnavailableException):
+            client.update_cache()
 
+    def test_querying_for_proposal_by_id_gives_proposal_info(self):
         client = self.create_client()
+        client.update_cache()
+
         proposal_info = client.proposal_by_id("471120")
 
         assert (
@@ -124,56 +152,19 @@ def test_querying_for_proposal_by_id_gives_proposal_info(self):
             "University A",
         )
 
-    def test_if_proposal_system_unavailable_load_from_cache(self):
-        self.system.get_proposals_for_instrument.side_effect = ServerException("oops")
-
-        _ = self.create_client()
-        self.cache.import_from_file.assert_called_once()
-
-    def test_if_proposal_system_unavailable_and_load_from_cache_raises(self):
-        self.system.get_proposals_for_instrument.side_effect = ServerException("oops")
-        self.cache.import_from_file.side_effect = ImportCacheException("oops")
-
-        with pytest.raises(DataUnavailableException):
-            _ = self.create_client()
-
-    def test_if_proposal_system_unavailable_and_cache_not_empty_then_do_not_import(
-        self,
-    ):
-        self.system.get_proposals_for_instrument.side_effect = ServerException("oops")
-        self.cache.is_empty.return_value = False
-
-        _ = self.create_client()
-        self.cache.import_from_file.assert_not_called()
-
-    def test_on_refresh_proposal_system_called_and_cache_updated(self):
+    def test_can_get_proposals_by_fed_id(self):
         client = self.create_client()
-        self.cache.reset_mock()
-        self.system.reset_mock()
-
         client.update_cache()
 
-        self.system.get_proposals_for_instrument.assert_called_once()
-        self.cache.update.assert_called_once()
-        self.cache.export_to_file.assert_called_once()
-
-    def test_can_get_proposals_by_fed_id(self):
-        cache = FileCache(":: filepath ::")
-        cache.update(VALID_PROPOSAL_DATA)
-        self.system.get_proposals_for_instrument.return_value = VALID_PROPOSAL_DATA
-
-        client = self.create_client(cache, update_cache=False)
         proposals = client.proposals_for_user("jonathantaylor")
 
         assert len(proposals) == 2
         assert {p.id for p in proposals} == {"471120", "871067"}
 
     def test_unrecognised_fed_id(self):
-        cache = FileCache(":: filepath ::")
-        cache.update(VALID_PROPOSAL_DATA)
-        self.system.get_proposals_for_instrument.return_value = VALID_PROPOSAL_DATA
+        client = self.create_client()
+        client.update_cache()
 
-        client = self.create_client(cache, update_cache=False)
         proposals = client.proposals_for_user("not_a_fed_id")
 
         assert len(proposals) == 0

yuos_query/__init__.py

@@ -1,3 +1,4 @@
 from yuos_query.version import version
+from yuos_query.yuos_client import YuosCacheClient, YuosServer  # noqa: F401
 
 __version__ = version

yuos_query/file_cache.py

@@ -28,8 +28,7 @@ def export_to_file(self):
 
     def import_from_file(self):
         try:
-            with open(self.cache_filepath, "r") as file:
-                self.update(deserialise_proposals_from_json(file.read()))
+            self.update(deserialise_proposals_from_json(self._read_file()))
         except FileNotFoundError as error:
             raise ImportCacheException(
                 f"Cache file ({self.cache_filepath}) not found"
@@ -39,6 +38,11 @@ def import_from_file(self):
                 f"Could not extract data from cache file ({self.cache_filepath}): {error}"
             ) from error
 
+    def _read_file(self):
+        with open(self.cache_filepath, "r") as file:
+            data = file.read()
+        return data
+
     def clear_cache(self):
         self.proposals = {}
         self.proposals_by_fed_id = {}

yuos_query/proposal_system.py

@@ -108,7 +108,7 @@ def request(self, query):
 
 class ProposalRequester:
     """
-    Don't use this directly, use YuosClient
+    Don't use this directly, use YuosServer
     """
 
     def __init__(self, url, token, wrapper=None):

yuos_query/version.py

@@ -1 +1 @@
-version = "0.1.17"
+version = "0.1.18"