David/use dependency injection (#174)

* restructure Catprez Endpoints
add annotation listings to root endpoint /

* blacked code

* Test passes locally - xfail pending change to pyoxigraph testing

* Black code
Specify namespace to avoid race condition
Add print to identify why GH action failing but local passing
Add further ref data (SKOS + SDO HTTPS version)
update GH actions; add init.py for tests
add oxrdflib depedency
QuerySender -> Repo
Use Fastapi's dependency injection to allow switchable backends
Add caching specifically to link generation

* rename query_sender (+ variants) -> repo

* remove commented code
reformat with black

* restructure Catprez Endpoints
add annotation listings to root endpoint /

* blacked code

* Black code
Specify namespace to avoid race condition
Add print to identify why GH action failing but local passing
Add further ref data (SKOS + SDO HTTPS version)
update GH actions; add init.py for tests
add oxrdflib depedency
QuerySender -> Repo
Use Fastapi's dependency injection to allow switchable backends
Add caching specifically to link generation

* rename query_sender (+ variants) -> repo

* remove commented code
reformat with black

.github/workflows/on_pr_to_main.yaml

@@ -57,13 +57,4 @@ jobs:
       #              run test suite
       #----------------------------------------------
       - name: Run tests
-        run: |
-          cd tests/spaceprez && poetry run pytest
-          cd ../vocprez && poetry run pytest
-          cd ../catprez && poetry run pytest
-          cd ../profiles && poetry run pytest
-          cd ../services && poetry run pytest
-          cd ../identifier && poetry run pytest
-          cd ../object && poetry run pytest
-          cd ../caching && poetry run pytest
-          cd ../dd_profile && poetry run pytest
+        run: poetry run pytest tests

.github/workflows/on_push_to_feature.yaml

@@ -55,13 +55,5 @@ jobs:
       #              run test suite
       #----------------------------------------------
       - name: Run tests
-        run: |
-          cd tests/spaceprez && poetry run pytest
-          cd ../vocprez && poetry run pytest
-          cd ../catprez && poetry run pytest
-          cd ../profiles && poetry run pytest
-          cd ../services && poetry run pytest
-          cd ../identifier && poetry run pytest
-          cd ../object && poetry run pytest
-          cd ../caching && poetry run pytest
-          cd ../dd_profile && poetry run pytest
+        run: poetry run pytest tests
+

prez/app.py

@@ -3,19 +3,23 @@
 from textwrap import dedent
 
 import uvicorn
-from fastapi import FastAPI, Request
+from fastapi import FastAPI
 from fastapi.openapi.utils import get_openapi
-from rdflib import Graph, Literal, URIRef
+from rdflib import Graph
 from starlette.middleware.cors import CORSMiddleware
 
 from prez.config import settings
+from prez.dependencies import (
+    get_async_http_client,
+    get_pyoxi_store,
+    load_local_data_to_oxigraph,
+    get_oxrdflib_store,
+)
 from prez.models.model_exceptions import (
     ClassNotFoundException,
     URINotFoundException,
     NoProfilesException,
 )
-from prez.reference_data.prez_ns import PREZ
-from prez.renderers.renderer import return_rdf
 from prez.routers.catprez import router as catprez_router
 from prez.routers.cql import router as cql_router
 from prez.routers.identifier import router as identifier_router
@@ -45,6 +49,7 @@
 from prez.services.generate_profiles import create_profiles_graph
 from prez.services.prez_logging import setup_logger
 from prez.services.search_methods import get_all_search_methods
+from prez.sparql.methods import RemoteSparqlRepo, PyoxigraphRepo, OxrdflibRepo
 
 app = FastAPI(
     exception_handlers={
@@ -116,12 +121,27 @@ async def app_startup():
     log = logging.getLogger("prez")
     log.info("Starting up")
 
-    await healthcheck_sparql_endpoints()
-    await add_prefixes_to_prefix_graph()
-    await get_all_search_methods()
-    await create_profiles_graph()
-    await create_endpoints_graph()
-    await count_objects()
+    if settings.sparql_repo_type == "pyoxigraph":
+        app.state.pyoxi_store = get_pyoxi_store()
+        app.state.repo = PyoxigraphRepo(app.state.pyoxi_store)
+        await load_local_data_to_oxigraph(app.state.pyoxi_store)
+    elif settings.sparql_repo_type == "oxrdflib":
+        app.state.oxrdflib_store = get_oxrdflib_store()
+        app.state.repo = OxrdflibRepo(app.state.oxrdflib_store)
+    elif settings.sparql_repo_type == "remote":
+        app.state.http_async_client = await get_async_http_client()
+        app.state.repo = RemoteSparqlRepo(app.state.http_async_client)
+        await healthcheck_sparql_endpoints()
+    else:
+        raise ValueError(
+            "SPARQL_REPO_TYPE must be one of 'pyoxigraph', 'oxrdflib' or 'remote'"
+        )
+
+    await add_prefixes_to_prefix_graph(app.state.repo)
+    await get_all_search_methods(app.state.repo)
+    await create_profiles_graph(app.state.repo)
+    await create_endpoints_graph(app.state.repo)
+    await count_objects(app.state.repo)
     await populate_api_info()
     await add_common_context_ontologies_to_tbox_cache()
 
@@ -136,10 +156,8 @@ async def app_shutdown():
     log.info("Shutting down...")
 
     # close all SPARQL async clients
-    if not os.getenv("TEST_MODE") == "true":
-        from prez.sparql.methods import async_client
-
-        await async_client.aclose()
+    if not settings.sparql_repo_type:
+        await app.state.http_async_client.aclose()
 
 
 def _get_sparql_service_description(request, format):

prez/cache.py

@@ -1,4 +1,5 @@
-from rdflib import Graph, ConjunctiveGraph
+from pyoxigraph.pyoxigraph import Store
+from rdflib import Graph, ConjunctiveGraph, Dataset
 
 tbox_cache = Graph()
 
@@ -16,4 +17,11 @@
 # TODO can probably merge counts graph
 counts_graph = Graph()
 
+links_ids_graph_cache = Dataset()
+links_ids_graph_cache.bind("prez", "https://prez.dev/")
+
 search_methods = {}
+
+store = Store()
+
+oxrdflib_store = Graph(store="Oxigraph")

prez/config.py

@@ -29,7 +29,7 @@ class Settings(BaseSettings):
     prez_version:
     """
 
-    sparql_endpoint: str
+    sparql_endpoint: Optional[str]
     sparql_username: Optional[str]
     sparql_password: Optional[str]
     sparql_auth: Optional[tuple]
@@ -48,6 +48,7 @@ class Settings(BaseSettings):
     provenance_predicates = [DCTERMS.provenance]
     other_predicates = [SDO.color, REG.status]
     sparql_timeout = 30.0
+    sparql_repo_type: str = "remote"
 
     log_level = "INFO"
     log_output = "stdout"
@@ -59,14 +60,6 @@ class Settings(BaseSettings):
     prez_version: Optional[str]
     disable_prefix_generation: bool = False
 
-    @root_validator()
-    def check_endpoint_enabled(cls, values):
-        if not values.get("sparql_endpoint"):
-            raise ValueError(
-                'A SPARQL endpoint must be specified using the "SPARQL_ENDPOINT" environment variable'
-            )
-        return values
-
     @root_validator()
     def get_version(cls, values):
         version = environ.get("PREZ_VERSION")

prez/dependencies.py

@@ -0,0 +1,46 @@
+from pathlib import Path
+
+import httpx
+from fastapi import Depends
+from pyoxigraph import Store
+
+from prez.cache import store, oxrdflib_store
+from prez.config import settings
+from prez.sparql.methods import PyoxigraphRepo, RemoteSparqlRepo, OxrdflibRepo
+
+
+async def get_async_http_client():
+    return httpx.AsyncClient(
+        auth=(settings.sparql_username, settings.sparql_password)
+        if settings.sparql_username
+        else None,
+        timeout=settings.sparql_timeout,
+    )
+
+
+def get_pyoxi_store():
+    return store
+
+
+def get_oxrdflib_store():
+    return oxrdflib_store
+
+
+async def get_repo(
+    http_async_client: httpx.AsyncClient = Depends(get_async_http_client),
+    pyoxi_store: Store = Depends(get_pyoxi_store),
+):
+    if settings.sparql_repo_type == "pyoxigraph":
+        return PyoxigraphRepo(pyoxi_store)
+    elif settings.sparql_repo_type == "oxrdflib":
+        return OxrdflibRepo(oxrdflib_store)
+    elif settings.sparql_repo_type == "remote":
+        return RemoteSparqlRepo(http_async_client)
+
+
+async def load_local_data_to_oxigraph(store: Store):
+    """
+    Loads all the data from the local data directory into the local SPARQL endpoint
+    """
+    for file in (Path(__file__).parent.parent / "rdf").glob("*.ttl"):
+        store.load(file.read_bytes(), "text/turtle")

prez/models/object_item.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Optional, FrozenSet, Tuple
 from typing import Set
 
 from pydantic import BaseModel, root_validator
@@ -13,46 +13,10 @@
 
 class ObjectItem(BaseModel):
     uri: Optional[URIRef] = None
-    object_curie: Optional[str] = None
-    classes: Optional[Set[URIRef]] = frozenset([PROF.Profile])
+    classes: Optional[FrozenSet[URIRef]] = frozenset([PROF.Profile])
     selected_class: Optional[URIRef] = None
-    profile: Optional[URIRef] = PREZ["profile/open"]
-    link_constructor: Optional[str] = None  # TODO remove when no longer being used
-    endpoint_uri: Optional[URIRef] = None
+    profile: Optional[URIRef] = None
+    top_level_listing: Optional[bool] = False
 
     def __hash__(self):
         return hash(self.uri)
-
-    @root_validator
-    def populate(cls, values):
-        values["top_level_listing"] = False  # this class is for objects, not listings.
-        uri_str = values.get("uri")
-        endpoint_uri_str = values.get("endpoint_uri")
-        if endpoint_uri_str:
-            endpoint_uri = URIRef(endpoint_uri_str)
-            values["classes"] = frozenset(
-                [
-                    klass
-                    for klass in endpoints_graph_cache.objects(
-                        endpoint_uri, ONT.deliversClasses, None
-                    )
-                ]
-            )
-            values["base_class"] = endpoints_graph_cache.value(
-                endpoint_uri, ONT.baseClass
-            )
-        else:
-            try:
-                values["classes"] = frozenset(
-                    tup[1] for tup in get_classes([values["uri"]])
-                )
-            except ClassNotFoundException:
-                # TODO return a generic DESCRIBE on the object - we can't use any of prez's profiles/endpoints to render
-                # information about the object, but we can provide any RDF we have for it.
-                pass
-        if uri_str:
-            values["uri"] = URIRef(uri_str)
-        else:
-            values["uri"] = get_uri_for_curie_id(values["object_curie"])
-
-        return values

prez/queries/vocprez.py

@@ -50,6 +50,8 @@ def get_concept_scheme_query(iri: str, bnode_depth: int) -> str:
     return dedent(query)
 
 
+# TODO query appears to erroneously create TopConcepts where they don't exist - perhaps from the optional statements
+#  see test_concept_scheme_top_concepts test w/ borehole-purpose-no-children
 def get_concept_scheme_top_concepts_query(iri: str, page: int, per_page: int) -> str:
     query = Template(
         """