Merge pull request #189 from RDFLib/david/improve-performance-link-ge…

…neration

Improve performance for link generation

prez/app.py

@@ -15,7 +15,7 @@
     load_local_data_to_oxigraph,
     get_oxrdflib_store,
     get_system_store,
-    load_profile_data_to_oxigraph,
+    load_system_data_to_oxigraph,
 )
 from prez.models.model_exceptions import (
     ClassNotFoundException,
@@ -148,7 +148,7 @@ async def app_startup():
     await add_common_context_ontologies_to_tbox_cache()
 
     app.state.pyoxi_system_store = get_system_store()
-    await load_profile_data_to_oxigraph(app.state.pyoxi_system_store)
+    await load_system_data_to_oxigraph(app.state.pyoxi_system_store)
 
 
 @app.on_event("shutdown")

prez/config.py

@@ -59,6 +59,7 @@ class Settings(BaseSettings):
     )
     prez_version: Optional[str]
     disable_prefix_generation: bool = False
+    local_rdf_dir: str = "rdf"
 
     @root_validator()
     def get_version(cls, values):

prez/dependencies.py

@@ -4,7 +4,13 @@
 from fastapi import Depends
 from pyoxigraph import Store
 
-from prez.cache import store, oxrdflib_store, system_store, profiles_graph_cache
+from prez.cache import (
+    store,
+    oxrdflib_store,
+    system_store,
+    profiles_graph_cache,
+    endpoints_graph_cache,
+)
 from prez.config import settings
 from prez.sparql.methods import PyoxigraphRepo, RemoteSparqlRepo, OxrdflibRepo
 
@@ -57,14 +63,17 @@ async def load_local_data_to_oxigraph(store: Store):
     """
     Loads all the data from the local data directory into the local SPARQL endpoint
     """
-    for file in (Path(__file__).parent.parent / "rdf").glob("*.ttl"):
+    for file in (Path(__file__).parent.parent / settings.local_rdf_dir).glob("*.ttl"):
         store.load(file.read_bytes(), "text/turtle")
 
 
-async def load_profile_data_to_oxigraph(store: Store):
+async def load_system_data_to_oxigraph(store: Store):
     """
     Loads all the data from the local data directory into the local SPARQL endpoint
     """
     # TODO refactor to use the local files directly
-    graph_bytes = profiles_graph_cache.serialize(format="nt", encoding="utf-8")
-    store.load(graph_bytes, "application/n-triples")
+    profiles_bytes = profiles_graph_cache.serialize(format="nt", encoding="utf-8")
+    store.load(profiles_bytes, "application/n-triples")
+
+    endpoints_bytes = endpoints_graph_cache.serialize(format="nt", encoding="utf-8")
+    store.load(endpoints_bytes, "application/n-triples")

prez/routers/catprez.py

@@ -3,7 +3,7 @@
 from fastapi import APIRouter, Request, Depends
 from starlette.responses import PlainTextResponse
 
-from prez.dependencies import get_repo
+from prez.dependencies import get_repo, get_system_repo
 from prez.services.objects import object_function
 from prez.services.listings import listing_function
 from prez.services.curie_functions import get_uri_for_curie_id
@@ -27,9 +27,14 @@ async def catalog_list(
     page: Optional[int] = 1,
     per_page: Optional[int] = 20,
     repo: Repo = Depends(get_repo),
+    system_repo: Repo = Depends(get_system_repo),
 ):
     return await listing_function(
-        request=request, page=page, per_page=per_page, repo=repo
+        request=request,
+        page=page,
+        per_page=per_page,
+        repo=repo,
+        system_repo=system_repo,
     )
 
 
@@ -42,6 +47,7 @@ async def resource_list(
     request: Request,
     catalog_curie: str,
     repo: Repo = Depends(get_repo),
+    system_repo: Repo = Depends(get_system_repo),
     page: Optional[int] = 1,
     per_page: Optional[int] = 20,
 ):
@@ -51,6 +57,7 @@ async def resource_list(
         page=page,
         per_page=per_page,
         repo=repo,
+        system_repo=system_repo,
         uri=catalog_uri,
     )
 
@@ -65,9 +72,10 @@ async def resource_item(
     catalog_curie: str,
     resource_curie: str,
     repo: Repo = Depends(get_repo),
+    system_repo: Repo = Depends(get_system_repo),
 ):
     return await object_function(
-        request=request, object_curie=resource_curie, repo=repo
+        request=request, object_curie=resource_curie, repo=repo, system_repo=system_repo
     )
 
 
@@ -80,5 +88,8 @@ async def catalog_item(
     request: Request,
     catalog_curie: str,
     repo: Repo = Depends(get_repo),
+    system_repo: Repo = Depends(get_system_repo),
 ):
-    return await object_function(request=request, object_curie=catalog_curie, repo=repo)
+    return await object_function(
+        request=request, object_curie=catalog_curie, repo=repo, system_repo=system_repo
+    )

prez/routers/object.py

@@ -11,7 +11,7 @@
     profiles_graph_cache,
     links_ids_graph_cache,
 )
-from prez.dependencies import get_repo
+from prez.dependencies import get_repo, get_system_repo
 from prez.models.listing import ListingModel
 from prez.models.object_item import ObjectItem
 from prez.models.profiles_and_mediatypes import ProfilesMediatypesInfo
@@ -93,5 +93,5 @@ async def count_route(
 
 
 @router.get("/object", summary="Object", name="https://prez.dev/endpoint/object")
-async def object_route(request: Request, repo=Depends(get_repo)):
-    return await object_function(request, repo=repo)
+async def object_route(request: Request, repo=Depends(get_repo), system_repo=Depends(get_system_repo)):
+    return await object_function(request, repo=repo, system_repo=system_repo)

prez/routers/profiles.py

@@ -34,7 +34,7 @@ async def profiles(
     repo=Depends(get_system_repo),
 ):
     return await listing_function(
-        request=request, page=page, per_page=per_page, repo=repo
+        request=request, page=page, per_page=per_page, repo=repo, system_repo=repo
     )
 
 
@@ -44,4 +44,4 @@ async def profiles(
     name="https://prez.dev/endpoint/profile",
 )
 async def profile(request: Request, profile_curie: str, repo=Depends(get_system_repo)):
-    return await object_function(request, object_curie=profile_curie, repo=repo)
+    return await object_function(request, object_curie=profile_curie, repo=repo, system_repo=repo)

prez/routers/search.py

@@ -6,7 +6,7 @@
 
 from prez.cache import search_methods
 from prez.config import settings
-from prez.dependencies import get_repo
+from prez.dependencies import get_repo, get_system_repo
 from prez.models.profiles_and_mediatypes import ProfilesMediatypesInfo
 from prez.reference_data.prez_ns import PREZ
 from prez.renderers.renderer import return_from_graph
@@ -22,6 +22,7 @@
 async def search(
     request: Request,
     repo: Repo = Depends(get_repo),
+    system_repo: Repo = Depends(get_system_repo),
 ):
     term = request.query_params.get("term")
     limit = request.query_params.get("limit", 20)
@@ -81,7 +82,7 @@ async def search(
         request=request, classes=frozenset([PREZ.SearchResult])
     )
     if "anot+" in prof_and_mt_info.mediatype:
-        await _add_prez_links(graph, repo)
+        await _add_prez_links(graph, repo, system_repo)
 
     return await return_from_graph(
         graph,

prez/routers/spaceprez.py

@@ -3,7 +3,7 @@
 from fastapi import APIRouter, Request, Depends
 from starlette.responses import PlainTextResponse
 
-from prez.dependencies import get_repo
+from prez.dependencies import get_repo, get_system_repo
 from prez.services.objects import object_function
 from prez.services.listings import listing_function
 from prez.services.curie_functions import get_uri_for_curie_id
@@ -25,11 +25,12 @@ async def spaceprez_profiles():
 async def list_datasets(
     request: Request,
     repo: Repo = Depends(get_repo),
-    page: Optional[int] = 1,
+        system_repo: Repo = Depends(get_system_repo),
+        page: Optional[int] = 1,
     per_page: Optional[int] = 20,
 ):
     return await listing_function(
-        request=request, page=page, per_page=per_page, repo=repo
+        request=request, page=page, per_page=per_page, repo=repo, system_repo=system_repo
     )
 
 
@@ -42,7 +43,8 @@ async def list_feature_collections(
     request: Request,
     dataset_curie: str,
     repo: Repo = Depends(get_repo),
-    page: Optional[int] = 1,
+        system_repo: Repo = Depends(get_system_repo),
+        page: Optional[int] = 1,
     per_page: Optional[int] = 20,
 ):
     dataset_uri = get_uri_for_curie_id(dataset_curie)
@@ -52,6 +54,7 @@ async def list_feature_collections(
         per_page=per_page,
         uri=dataset_uri,
         repo=repo,
+        system_repo=system_repo,
     )
 
 
@@ -65,7 +68,8 @@ async def list_features(
     dataset_curie: str,
     collection_curie: str,
     repo: Repo = Depends(get_repo),
-    page: Optional[int] = 1,
+        system_repo: Repo = Depends(get_system_repo),
+        page: Optional[int] = 1,
     per_page: Optional[int] = 20,
 ):
     collection_uri = get_uri_for_curie_id(collection_curie)
@@ -75,6 +79,7 @@ async def list_features(
         per_page=per_page,
         uri=collection_uri,
         repo=repo,
+        system_repo=system_repo,
     )
 
 
@@ -87,8 +92,9 @@ async def dataset_item(
     request: Request,
     dataset_curie: str,
     repo: Repo = Depends(get_repo),
+        system_repo: Repo = Depends(get_system_repo),
 ):
-    return await object_function(request, object_curie=dataset_curie, repo=repo)
+    return await object_function(request, object_curie=dataset_curie, repo=repo, system_repo=system_repo)
 
 
 @router.get(
@@ -101,8 +107,9 @@ async def feature_collection_item(
     dataset_curie: str,
     collection_curie: str,
     repo: Repo = Depends(get_repo),
+        system_repo: Repo = Depends(get_system_repo),
 ):
-    return await object_function(request, object_curie=collection_curie, repo=repo)
+    return await object_function(request, object_curie=collection_curie, repo=repo, system_repo=system_repo)
 
 
 @router.get(
@@ -116,5 +123,6 @@ async def feature_item(
     collection_curie: str,
     feature_curie: str,
     repo: Repo = Depends(get_repo),
+        system_repo: Repo = Depends(get_system_repo),
 ):
-    return await object_function(request=request, object_curie=feature_curie, repo=repo)
+    return await object_function(request=request, object_curie=feature_curie, repo=repo, system_repo=system_repo)