diff --git a/README.md b/README.md index f92773d..be4b063 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,27 @@ pip install -r requirements-dev.txt pre-commit install ``` +## Usage + +There are two parts to the program: a service for downloading the data from the proposal system and storing it locally, +and a client for extracting information from the local store based on queries. + +Running the service: +``` +python bin/yuos.py -u https://useroffice-test.esss.lu.se/graphql -i YMIR -c /opt/yuos/cached_proposals.json +``` +Typically this would be run as a service. +NOTE: requires an authenticating token environment variable. + +Using the client: +``` +from yuos_query.yuos_client import YuosCacheClient + +client = YuosCacheClient.create('/opt/yuos/cached_proposals.json') +client.update_cache() +proposal = client.proposal_by_id("199842") +``` + ## Tests ### Programmer tests These tests live in the `tests` directory and can be run directly from the main directory using pytest. @@ -39,14 +60,4 @@ changed.** **Jenkins will run the tests against the real system automatically for pull requests.** -**Jenkins will also run the tests on main daily, so if there are breaking any API changes we will know about it.** - -## Example usage - -``` -import os -from yuos_query.yuos_client import YuosClient - -client = YuosClient("some url", os.environ["YUOS_TOKEN"], instrument_name, "/path/to/cache.json") -proposal_info = client.proposal_by_id(proposal_id) -``` +**Jenkins will also run the tests on main daily, so if there are any breaking API changes we will know about it.** diff --git a/bin/yuos.py b/bin/yuos.py new file mode 100644 index 0000000..04b79cc --- /dev/null +++ b/bin/yuos.py @@ -0,0 +1,68 @@ +import argparse +import logging +import os +import sys +import time + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) +from yuos_query import YuosServer + + +def main(url, instrument, cache_filepath, update_interval=3200): + while True: + try: + YuosServer.create( + url, os.environ.get("YUOS_TOKEN"), instrument, cache_filepath + ).update_cache() + logging.info("updated cache") + time.sleep(update_interval) + except RuntimeError as error: + logging.error(f"failed to update cache {error}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + required_args = parser.add_argument_group("required arguments") + required_args.add_argument( + "-u", + "--url", + type=str, + help="the URL for the proposal system", + required=True, + ) + + required_args.add_argument( + "-i", "--instrument", type=str, help="the instrument name", required=True + ) + + parser.add_argument( + "-c", + "--cache-filepath", + type=str, + help="where to write the data", + required=True, + ) + + parser.add_argument( + "-l", + "--log-level", + type=int, + default=3, + help="sets the logging level: debug=1, info=2, warning=3, error=4, critical=5.", + ) + + args = parser.parse_args() + + if 1 <= args.log_level <= 5: + logging.basicConfig( + format="%(asctime)s - %(message)s", level=args.log_level * 10 + ) + else: + logging.basicConfig(format="%(asctime)s - %(message)s", level=logging.INFO) + + main( + args.url, + args.instrument, + args.cache_filepath, + ) diff --git a/integration_tests/test_end_to_end.py b/integration_tests/test_end_to_end.py index 413a8ea..dc20960 100644 --- a/integration_tests/test_end_to_end.py +++ b/integration_tests/test_end_to_end.py @@ -3,7 +3,7 @@ import pytest -from yuos_query.yuos_client import YuosClient +from yuos_query.yuos_client import YuosCacheClient, YuosServer # These tests are skipped if the YUOS_TOKEN environment variable is not defined SKIP_TEST = True @@ -20,9 +20,13 @@ ) def test_get_proposals_and_sample_for_specific_id_on_ymir_instrument(): with TemporaryDirectory() as directory: - client = YuosClient( + server = YuosServer.create( SERVER_URL, YUOS_TOKEN, "YMIR", os.path.join(directory, "cache.json") ) + server.update_cache() + + client = YuosCacheClient.create(os.path.join(directory, "cache.json")) + client.update_cache() result = client.proposal_by_id(KNOWN_PROPOSAL) @@ -61,9 +65,13 @@ def test_get_proposals_and_sample_for_specific_id_on_ymir_instrument(): ) def test_get_proposals_for_specific_fed_id_on_ymir_instrument(): with TemporaryDirectory() as directory: - client = YuosClient( + server = YuosServer.create( SERVER_URL, YUOS_TOKEN, "YMIR", os.path.join(directory, "cache.json") ) + server.update_cache() + + client = YuosCacheClient.create(os.path.join(directory, "cache.json")) + client.update_cache() results = client.proposals_for_user("jonathantaylor") assert len(results) == 6 diff --git a/tests/test_yuos_client.py b/tests/test_yuos_client.py index fc00e23..09f26d4 100644 --- a/tests/test_yuos_client.py +++ b/tests/test_yuos_client.py @@ -2,16 +2,17 @@ import pytest +from example_data import get_ymir_example_data from yuos_query.data_classes import ProposalInfo, SampleInfo, User from yuos_query.exceptions import ( DataUnavailableException, - ImportCacheException, InvalidIdException, ServerException, ) from yuos_query.file_cache import FileCache from yuos_query.proposal_system import ProposalRequester -from yuos_query.yuos_client import YuosClient +from yuos_query.utils import serialise_proposals_to_json +from yuos_query.yuos_client import YuosCacheClient, YuosServer VALID_PROPOSAL_DATA = { "471120": ProposalInfo( @@ -54,58 +55,85 @@ } -class TestYuosClient: +class TestYuosServer: @pytest.fixture(autouse=True) def prepare(self): self.cache = mock.create_autospec(FileCache) self.system = mock.create_autospec(ProposalRequester) - def create_client(self, cache=None, update_cache=True): + def create_server(self, cache=None): if not cache: cache = self.cache - return YuosClient( - ":: url ::", - ":: token ::", - "YMIR", - ":: file ::", - update_cache=update_cache, - cache=cache, - system=self.system, - ) + return YuosServer("YMIR", cache, self.system) - def test_on_construction_proposal_system_called_and_cache_updated(self): - _ = self.create_client() + def test_on_update_proposal_system_called_and_cache_updated(self): + server = self.create_server() + server.update_cache() self.system.get_proposals_for_instrument.assert_called_once() self.cache.update.assert_called_once() self.cache.export_to_file.assert_called_once() - def test_on_refreshing_cache_proposal_system_called_and_cache_updated(self): - _ = self.create_client() + def test_if_proposal_system_unavailable_then_raises(self): + server = self.create_server() + server.update_cache() - self.system.get_proposals_for_instrument.assert_called_once() - self.cache.update.assert_called_once() + self.system.get_proposals_for_instrument.side_effect = ServerException("oops") + + with pytest.raises(DataUnavailableException): + _ = server.update_cache() + + +class TestYuosCacheClient: + class InMemoryCache(FileCache): + def _read_file(self): + return get_ymir_example_data() + + @pytest.fixture(autouse=True) + def prepare(self): + self.cache = FileCache("::filepath::") + self.cache._read_file = lambda: serialise_proposals_to_json(VALID_PROPOSAL_DATA) + + def create_client( + self, + cache=None, + ): + if not cache: + cache = self.cache + + return YuosCacheClient( + cache, + ) def test_querying_with_id_that_does_not_conform_to_pattern_raises( self, ): client = self.create_client() + client.update_cache() with pytest.raises(InvalidIdException): client.proposal_by_id("abc") def test_querying_for_unknown_proposal_id_returns_nothing(self): - self.cache.proposals = VALID_PROPOSAL_DATA - client = self.create_client() + client.update_cache() assert client.proposal_by_id("00000") is None - def test_querying_for_proposal_by_id_gives_proposal_info(self): - self.cache.proposals = VALID_PROPOSAL_DATA + def test_if_cache_file_missing_then_raises( + self, + ): + client = self.create_client() + self.cache._read_file = lambda: (_ for _ in ()).throw(FileNotFoundError()) + + with pytest.raises(DataUnavailableException): + client.update_cache() + def test_querying_for_proposal_by_id_gives_proposal_info(self): client = self.create_client() + client.update_cache() + proposal_info = client.proposal_by_id("471120") assert ( @@ -124,56 +152,19 @@ def test_querying_for_proposal_by_id_gives_proposal_info(self): "University A", ) - def test_if_proposal_system_unavailable_load_from_cache(self): - self.system.get_proposals_for_instrument.side_effect = ServerException("oops") - - _ = self.create_client() - self.cache.import_from_file.assert_called_once() - - def test_if_proposal_system_unavailable_and_load_from_cache_raises(self): - self.system.get_proposals_for_instrument.side_effect = ServerException("oops") - self.cache.import_from_file.side_effect = ImportCacheException("oops") - - with pytest.raises(DataUnavailableException): - _ = self.create_client() - - def test_if_proposal_system_unavailable_and_cache_not_empty_then_do_not_import( - self, - ): - self.system.get_proposals_for_instrument.side_effect = ServerException("oops") - self.cache.is_empty.return_value = False - - _ = self.create_client() - self.cache.import_from_file.assert_not_called() - - def test_on_refresh_proposal_system_called_and_cache_updated(self): + def test_can_get_proposals_by_fed_id(self): client = self.create_client() - self.cache.reset_mock() - self.system.reset_mock() - client.update_cache() - self.system.get_proposals_for_instrument.assert_called_once() - self.cache.update.assert_called_once() - self.cache.export_to_file.assert_called_once() - - def test_can_get_proposals_by_fed_id(self): - cache = FileCache(":: filepath ::") - cache.update(VALID_PROPOSAL_DATA) - self.system.get_proposals_for_instrument.return_value = VALID_PROPOSAL_DATA - - client = self.create_client(cache, update_cache=False) proposals = client.proposals_for_user("jonathantaylor") assert len(proposals) == 2 assert {p.id for p in proposals} == {"471120", "871067"} def test_unrecognised_fed_id(self): - cache = FileCache(":: filepath ::") - cache.update(VALID_PROPOSAL_DATA) - self.system.get_proposals_for_instrument.return_value = VALID_PROPOSAL_DATA + client = self.create_client() + client.update_cache() - client = self.create_client(cache, update_cache=False) proposals = client.proposals_for_user("not_a_fed_id") assert len(proposals) == 0 diff --git a/yuos_query/__init__.py b/yuos_query/__init__.py index 11b5990..6ee2188 100644 --- a/yuos_query/__init__.py +++ b/yuos_query/__init__.py @@ -1,3 +1,4 @@ from yuos_query.version import version +from yuos_query.yuos_client import YuosCacheClient, YuosServer # noqa: F401 __version__ = version diff --git a/yuos_query/file_cache.py b/yuos_query/file_cache.py index b5f892b..0b84eee 100644 --- a/yuos_query/file_cache.py +++ b/yuos_query/file_cache.py @@ -28,8 +28,7 @@ def export_to_file(self): def import_from_file(self): try: - with open(self.cache_filepath, "r") as file: - self.update(deserialise_proposals_from_json(file.read())) + self.update(deserialise_proposals_from_json(self._read_file())) except FileNotFoundError as error: raise ImportCacheException( f"Cache file ({self.cache_filepath}) not found" @@ -39,6 +38,11 @@ def import_from_file(self): f"Could not extract data from cache file ({self.cache_filepath}): {error}" ) from error + def _read_file(self): + with open(self.cache_filepath, "r") as file: + data = file.read() + return data + def clear_cache(self): self.proposals = {} self.proposals_by_fed_id = {} diff --git a/yuos_query/proposal_system.py b/yuos_query/proposal_system.py index 41b8ce3..1193fdb 100644 --- a/yuos_query/proposal_system.py +++ b/yuos_query/proposal_system.py @@ -108,7 +108,7 @@ def request(self, query): class ProposalRequester: """ - Don't use this directly, use YuosClient + Don't use this directly, use YuosServer """ def __init__(self, url, token, wrapper=None): diff --git a/yuos_query/version.py b/yuos_query/version.py index a543ebb..3a9ce5f 100644 --- a/yuos_query/version.py +++ b/yuos_query/version.py @@ -1 +1 @@ -version = "0.1.17" +version = "0.1.18" diff --git a/yuos_query/yuos_client.py b/yuos_query/yuos_client.py index 4c42b7b..95a1442 100644 --- a/yuos_query/yuos_client.py +++ b/yuos_query/yuos_client.py @@ -13,6 +13,7 @@ class YuosClient: + # Deprecated def __init__( self, url, @@ -65,3 +66,59 @@ def update_cache(self): ) from error except ExportCacheException: raise + + +class YuosServer: + def __init__(self, instrument, cache, requester): + self.instrument = instrument + self.cache = cache + self.system = requester + + @classmethod + def create(cls, url, token, instrument, cache_filepath): + return cls(instrument, FileCache(cache_filepath), ProposalRequester(url, token)) + + def update_cache(self): + try: + proposals = self.system.get_proposals_for_instrument(self.instrument) + self.cache.update(proposals) + self.cache.export_to_file() + except ServerException as error: + raise DataUnavailableException("Proposal system unavailable") from error + except ExportCacheException: + raise + + +class YuosCacheClient: + def __init__(self, cache): + self.cache = cache + + @classmethod + def create(cls, cache_filepath): + return cls(FileCache(cache_filepath)) + + def proposal_by_id(self, proposal_id: str) -> Optional[ProposalInfo]: + """ + Find the proposal. + + :param proposal_id: proposal ID + :return: the proposal information or None if not found + """ + if not self._does_proposal_id_conform(proposal_id): + raise InvalidIdException() + return self.cache.proposals.get(proposal_id) + + def proposals_for_user(self, fed_id: str) -> List[ProposalInfo]: + if fed_id not in self.cache.proposals_by_fed_id: + return [] + return self.cache.proposals_by_fed_id[fed_id] + + def _does_proposal_id_conform(self, proposal_id: str) -> bool: + # Does proposal_id conform to the expected pattern? + return all(c.isdigit() for c in proposal_id) + + def update_cache(self): + try: + self.cache.import_from_file() + except ImportCacheException as error: + raise DataUnavailableException("Could not import cached data") from error