diff --git a/prez/app.py b/prez/app.py index 01c0436a..ebcb58f2 100644 --- a/prez/app.py +++ b/prez/app.py @@ -15,7 +15,7 @@ load_local_data_to_oxigraph, get_oxrdflib_store, get_system_store, - load_profile_data_to_oxigraph, + load_system_data_to_oxigraph, ) from prez.models.model_exceptions import ( ClassNotFoundException, @@ -148,7 +148,7 @@ async def app_startup(): await add_common_context_ontologies_to_tbox_cache() app.state.pyoxi_system_store = get_system_store() - await load_profile_data_to_oxigraph(app.state.pyoxi_system_store) + await load_system_data_to_oxigraph(app.state.pyoxi_system_store) @app.on_event("shutdown") diff --git a/prez/config.py b/prez/config.py index 88db3282..db02415c 100644 --- a/prez/config.py +++ b/prez/config.py @@ -59,6 +59,7 @@ class Settings(BaseSettings): ) prez_version: Optional[str] disable_prefix_generation: bool = False + local_rdf_dir: str = "rdf" @root_validator() def get_version(cls, values): diff --git a/prez/dependencies.py b/prez/dependencies.py index 72c7c123..d351c411 100644 --- a/prez/dependencies.py +++ b/prez/dependencies.py @@ -4,7 +4,13 @@ from fastapi import Depends from pyoxigraph import Store -from prez.cache import store, oxrdflib_store, system_store, profiles_graph_cache +from prez.cache import ( + store, + oxrdflib_store, + system_store, + profiles_graph_cache, + endpoints_graph_cache, +) from prez.config import settings from prez.sparql.methods import PyoxigraphRepo, RemoteSparqlRepo, OxrdflibRepo @@ -57,14 +63,17 @@ async def load_local_data_to_oxigraph(store: Store): """ Loads all the data from the local data directory into the local SPARQL endpoint """ - for file in (Path(__file__).parent.parent / "rdf").glob("*.ttl"): + for file in (Path(__file__).parent.parent / settings.local_rdf_dir).glob("*.ttl"): store.load(file.read_bytes(), "text/turtle") -async def load_profile_data_to_oxigraph(store: Store): +async def load_system_data_to_oxigraph(store: Store): """ Loads all the data from the local data directory into the local SPARQL endpoint """ # TODO refactor to use the local files directly - graph_bytes = profiles_graph_cache.serialize(format="nt", encoding="utf-8") - store.load(graph_bytes, "application/n-triples") + profiles_bytes = profiles_graph_cache.serialize(format="nt", encoding="utf-8") + store.load(profiles_bytes, "application/n-triples") + + endpoints_bytes = endpoints_graph_cache.serialize(format="nt", encoding="utf-8") + store.load(endpoints_bytes, "application/n-triples") diff --git a/prez/routers/catprez.py b/prez/routers/catprez.py index 48bed64e..5f318cf8 100644 --- a/prez/routers/catprez.py +++ b/prez/routers/catprez.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Request, Depends from starlette.responses import PlainTextResponse -from prez.dependencies import get_repo +from prez.dependencies import get_repo, get_system_repo from prez.services.objects import object_function from prez.services.listings import listing_function from prez.services.curie_functions import get_uri_for_curie_id @@ -27,9 +27,14 @@ async def catalog_list( page: Optional[int] = 1, per_page: Optional[int] = 20, repo: Repo = Depends(get_repo), + system_repo: Repo = Depends(get_system_repo), ): return await listing_function( - request=request, page=page, per_page=per_page, repo=repo + request=request, + page=page, + per_page=per_page, + repo=repo, + system_repo=system_repo, ) @@ -42,6 +47,7 @@ async def resource_list( request: Request, catalog_curie: str, repo: Repo = Depends(get_repo), + system_repo: Repo = Depends(get_system_repo), page: Optional[int] = 1, per_page: Optional[int] = 20, ): @@ -51,6 +57,7 @@ async def resource_list( page=page, per_page=per_page, repo=repo, + system_repo=system_repo, uri=catalog_uri, ) @@ -65,9 +72,10 @@ async def resource_item( catalog_curie: str, resource_curie: str, repo: Repo = Depends(get_repo), + system_repo: Repo = Depends(get_system_repo), ): return await object_function( - request=request, object_curie=resource_curie, repo=repo + request=request, object_curie=resource_curie, repo=repo, system_repo=system_repo ) @@ -80,5 +88,8 @@ async def catalog_item( request: Request, catalog_curie: str, repo: Repo = Depends(get_repo), + system_repo: Repo = Depends(get_system_repo), ): - return await object_function(request=request, object_curie=catalog_curie, repo=repo) + return await object_function( + request=request, object_curie=catalog_curie, repo=repo, system_repo=system_repo + ) diff --git a/prez/routers/object.py b/prez/routers/object.py index 18189dad..96728780 100644 --- a/prez/routers/object.py +++ b/prez/routers/object.py @@ -11,7 +11,7 @@ profiles_graph_cache, links_ids_graph_cache, ) -from prez.dependencies import get_repo +from prez.dependencies import get_repo, get_system_repo from prez.models.listing import ListingModel from prez.models.object_item import ObjectItem from prez.models.profiles_and_mediatypes import ProfilesMediatypesInfo @@ -93,5 +93,5 @@ async def count_route( @router.get("/object", summary="Object", name="https://prez.dev/endpoint/object") -async def object_route(request: Request, repo=Depends(get_repo)): - return await object_function(request, repo=repo) +async def object_route(request: Request, repo=Depends(get_repo), system_repo=Depends(get_system_repo)): + return await object_function(request, repo=repo, system_repo=system_repo) diff --git a/prez/routers/profiles.py b/prez/routers/profiles.py index 0b87bbf5..cbf5ba69 100644 --- a/prez/routers/profiles.py +++ b/prez/routers/profiles.py @@ -34,7 +34,7 @@ async def profiles( repo=Depends(get_system_repo), ): return await listing_function( - request=request, page=page, per_page=per_page, repo=repo + request=request, page=page, per_page=per_page, repo=repo, system_repo=repo ) @@ -44,4 +44,4 @@ async def profiles( name="https://prez.dev/endpoint/profile", ) async def profile(request: Request, profile_curie: str, repo=Depends(get_system_repo)): - return await object_function(request, object_curie=profile_curie, repo=repo) + return await object_function(request, object_curie=profile_curie, repo=repo, system_repo=repo) diff --git a/prez/routers/search.py b/prez/routers/search.py index 29e01af3..10c11567 100644 --- a/prez/routers/search.py +++ b/prez/routers/search.py @@ -6,7 +6,7 @@ from prez.cache import search_methods from prez.config import settings -from prez.dependencies import get_repo +from prez.dependencies import get_repo, get_system_repo from prez.models.profiles_and_mediatypes import ProfilesMediatypesInfo from prez.reference_data.prez_ns import PREZ from prez.renderers.renderer import return_from_graph @@ -22,6 +22,7 @@ async def search( request: Request, repo: Repo = Depends(get_repo), + system_repo: Repo = Depends(get_system_repo), ): term = request.query_params.get("term") limit = request.query_params.get("limit", 20) @@ -81,7 +82,7 @@ async def search( request=request, classes=frozenset([PREZ.SearchResult]) ) if "anot+" in prof_and_mt_info.mediatype: - await _add_prez_links(graph, repo) + await _add_prez_links(graph, repo, system_repo) return await return_from_graph( graph, diff --git a/prez/routers/spaceprez.py b/prez/routers/spaceprez.py index ce20ae60..31e475dd 100644 --- a/prez/routers/spaceprez.py +++ b/prez/routers/spaceprez.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Request, Depends from starlette.responses import PlainTextResponse -from prez.dependencies import get_repo +from prez.dependencies import get_repo, get_system_repo from prez.services.objects import object_function from prez.services.listings import listing_function from prez.services.curie_functions import get_uri_for_curie_id @@ -25,11 +25,12 @@ async def spaceprez_profiles(): async def list_datasets( request: Request, repo: Repo = Depends(get_repo), - page: Optional[int] = 1, + system_repo: Repo = Depends(get_system_repo), + page: Optional[int] = 1, per_page: Optional[int] = 20, ): return await listing_function( - request=request, page=page, per_page=per_page, repo=repo + request=request, page=page, per_page=per_page, repo=repo, system_repo=system_repo ) @@ -42,7 +43,8 @@ async def list_feature_collections( request: Request, dataset_curie: str, repo: Repo = Depends(get_repo), - page: Optional[int] = 1, + system_repo: Repo = Depends(get_system_repo), + page: Optional[int] = 1, per_page: Optional[int] = 20, ): dataset_uri = get_uri_for_curie_id(dataset_curie) @@ -52,6 +54,7 @@ async def list_feature_collections( per_page=per_page, uri=dataset_uri, repo=repo, + system_repo=system_repo, ) @@ -65,7 +68,8 @@ async def list_features( dataset_curie: str, collection_curie: str, repo: Repo = Depends(get_repo), - page: Optional[int] = 1, + system_repo: Repo = Depends(get_system_repo), + page: Optional[int] = 1, per_page: Optional[int] = 20, ): collection_uri = get_uri_for_curie_id(collection_curie) @@ -75,6 +79,7 @@ async def list_features( per_page=per_page, uri=collection_uri, repo=repo, + system_repo=system_repo, ) @@ -87,8 +92,9 @@ async def dataset_item( request: Request, dataset_curie: str, repo: Repo = Depends(get_repo), + system_repo: Repo = Depends(get_system_repo), ): - return await object_function(request, object_curie=dataset_curie, repo=repo) + return await object_function(request, object_curie=dataset_curie, repo=repo, system_repo=system_repo) @router.get( @@ -101,8 +107,9 @@ async def feature_collection_item( dataset_curie: str, collection_curie: str, repo: Repo = Depends(get_repo), + system_repo: Repo = Depends(get_system_repo), ): - return await object_function(request, object_curie=collection_curie, repo=repo) + return await object_function(request, object_curie=collection_curie, repo=repo, system_repo=system_repo) @router.get( @@ -116,5 +123,6 @@ async def feature_item( collection_curie: str, feature_curie: str, repo: Repo = Depends(get_repo), + system_repo: Repo = Depends(get_system_repo), ): - return await object_function(request=request, object_curie=feature_curie, repo=repo) + return await object_function(request=request, object_curie=feature_curie, repo=repo, system_repo=system_repo) diff --git a/prez/routers/vocprez.py b/prez/routers/vocprez.py index a2e4fbe2..61d56278 100644 --- a/prez/routers/vocprez.py +++ b/prez/routers/vocprez.py @@ -7,7 +7,7 @@ from starlette.responses import PlainTextResponse from prez.bnode import get_bnode_depth -from prez.dependencies import get_repo +from prez.dependencies import get_repo, get_system_repo from prez.models.profiles_and_mediatypes import ProfilesMediatypesInfo from prez.queries.vocprez import ( get_concept_scheme_query, @@ -44,11 +44,16 @@ async def vocprez_home(): async def vocab_endpoint( request: Request, repo: Repo = Depends(get_repo), + system_repo: Repo = Depends(get_system_repo), page: int = 1, per_page: int = 20, ): return await listing_function( - request=request, page=page, per_page=per_page, repo=repo + request=request, + page=page, + per_page=per_page, + repo=repo, + system_repo=system_repo, ) @@ -60,11 +65,16 @@ async def vocab_endpoint( async def collection_endpoint( request: Request, repo: Repo = Depends(get_repo), + system_repo: Repo = Depends(get_system_repo), page: int = 1, per_page: int = 20, ): return await listing_function( - request=request, page=page, per_page=per_page, repo=repo + request=request, + page=page, + per_page=per_page, + repo=repo, + system_repo=system_repo, ) @@ -74,13 +84,18 @@ async def collection_endpoint( name="https://prez.dev/endpoint/vocprez/vocab", ) async def vocprez_scheme( - request: Request, scheme_curie: str, repo: Repo = Depends(get_repo) + request: Request, + scheme_curie: str, + repo: Repo = Depends(get_repo), + system_repo: Repo = Depends(get_system_repo), ): """Get a SKOS Concept Scheme and all of its concepts. Note: This may be a very expensive operation depending on the size of the concept scheme. """ - return await object_function(request, object_curie=scheme_curie, repo=repo) + return await object_function( + request, object_curie=scheme_curie, repo=repo, system_repo=system_repo + ) @router.get( @@ -147,6 +162,7 @@ async def concept_scheme_top_concepts_route( page: int = 1, per_page: int = 20, repo: Repo = Depends(get_repo), + system_repo: Repo = Depends(get_system_repo), ): """Get a SKOS Concept Scheme's top concepts. @@ -166,7 +182,7 @@ async def concept_scheme_top_concepts_route( if isinstance(concept, URIRef): concept_curie = get_curie_id_for_uri(concept) if "anot+" in profiles_mediatypes_info.mediatype: - await _add_prez_links(graph, repo) + await _add_prez_links(graph, repo, system_repo) return await return_from_graph( graph, profiles_mediatypes_info.mediatype, @@ -192,6 +208,7 @@ async def concept_narrowers_route( concept_scheme_curie: str, concept_curie: str, repo: Repo = Depends(get_repo), + system_repo: Repo = Depends(get_system_repo), page: int = 1, per_page: int = 20, ): @@ -208,7 +225,7 @@ async def concept_narrowers_route( graph, _ = await repo.send_queries([concept_narrowers_query], []) if "anot+" in profiles_mediatypes_info.mediatype: - await _add_prez_links(graph, repo) + await _add_prez_links(graph, repo, system_repo) return await return_from_graph( graph, profiles_mediatypes_info.mediatype, @@ -235,9 +252,12 @@ async def concept_route( concept_scheme_curie: str, concept_curie: str, repo: Repo = Depends(get_repo), + system_repo=Depends(get_system_repo), ): """Get a SKOS Concept.""" - return await object_function(request, object_curie=concept_curie, repo=repo) + return await object_function( + request, object_curie=concept_curie, repo=repo, system_repo=system_repo + ) @router.get( @@ -249,8 +269,11 @@ async def vocprez_collection( request: Request, collection_curie: str, repo: Repo = Depends(get_repo), + system_repo: Repo = Depends(get_system_repo), ): - return await object_function(request, object_curie=collection_curie, repo=repo) + return await object_function( + request, object_curie=collection_curie, repo=repo, system_repo=system_repo + ) @router.get( @@ -263,5 +286,8 @@ async def vocprez_collection_concept( collection_curie: str, concept_curie: str, repo: Repo = Depends(get_repo), + system_repo: Repo = Depends(get_system_repo), ): - return await object_function(request, object_curie=concept_curie, repo=repo) + return await object_function( + request, object_curie=concept_curie, repo=repo, system_repo=system_repo + ) diff --git a/prez/services/link_generation.py b/prez/services/link_generation.py index 349b6713..7f1b3ea6 100644 --- a/prez/services/link_generation.py +++ b/prez/services/link_generation.py @@ -1,9 +1,11 @@ from string import Template from typing import FrozenSet -from rdflib import Graph, Literal, URIRef, DCTERMS +from fastapi import Depends +from rdflib import Graph, Literal, URIRef, DCTERMS, BNode from prez.cache import endpoints_graph_cache, links_ids_graph_cache +from prez.dependencies import get_system_repo from prez.reference_data.prez_ns import PREZ from prez.services.curie_functions import get_curie_id_for_uri from prez.services.model_methods import get_classes @@ -14,14 +16,18 @@ ) -async def _add_prez_links(graph: Graph, repo): +async def _add_prez_links(graph: Graph, repo: Repo, system_repo: Repo): # get all URIRefs - if Prez can find a class and endpoint for them, an internal link will be generated. uris = [uri for uri in graph.all_nodes() if isinstance(uri, URIRef)] + uri_to_klasses = {} for uri in uris: - await _create_internal_links_graph(uri, graph, repo) + uri_to_klasses[uri] = await get_classes(uri, repo) + for uri, klasses in uri_to_klasses.items(): + await _create_internal_links_graph(uri, graph, repo, klasses, system_repo) -async def _create_internal_links_graph(uri, graph, repo: Repo): + +async def _create_internal_links_graph(uri, graph, repo: Repo, klasses, system_repo): quads = list( links_ids_graph_cache.quads((None, None, None, uri)) ) # context required as not all triples that relate to links or identifiers for a particular object have that object's URI as the subject @@ -29,9 +35,10 @@ async def _create_internal_links_graph(uri, graph, repo: Repo): for quad in quads: graph.add(quad[:3]) else: - klasses = await get_classes(uri, repo) for klass in klasses: - endpoint_to_relations = get_endpoint_info_for_classes(frozenset([klass])) + endpoint_to_relations = await get_endpoint_info_for_classes( + frozenset([klass]), system_repo + ) relationship_query = generate_relationship_query(uri, endpoint_to_relations) if relationship_query: _, tabular_results = await repo.send_queries( @@ -44,7 +51,9 @@ async def _create_internal_links_graph(uri, graph, repo: Repo): links_ids_graph_cache.add(quad) # add the quad to the cache -def get_endpoint_info_for_classes(classes: FrozenSet[URIRef]) -> dict: +async def get_endpoint_info_for_classes( + classes: FrozenSet[URIRef], system_repo +) -> dict: """ Queries Prez's in memory reference data for endpoints to determine which endpoints are relevant for the classes an object has, along with information about "parent" objects included in the URL path for the object. This information @@ -52,13 +61,17 @@ def get_endpoint_info_for_classes(classes: FrozenSet[URIRef]) -> dict: and the predicate used for the relationship. """ endpoint_query = get_endpoint_template_queries(classes) - results = endpoints_graph_cache.query(endpoint_query) + results = await system_repo.send_queries([], [(None, endpoint_query)]) endpoint_to_relations = {} - if results.bindings != [{}]: - for result in results.bindings: - endpoint_template = result["endpoint_template"] + if results[1][0][1] != [{}]: + for result in results[1][0][1]: + endpoint_template = result["endpoint_template"]["value"] relation = result.get("relation_predicate") + if relation: + relation = URIRef(relation["value"]) direction = result.get("relation_direction") + if direction: + direction = URIRef(direction["value"]) if endpoint_template not in endpoint_to_relations: endpoint_to_relations[endpoint_template] = [(relation, direction)] else: diff --git a/prez/services/listings.py b/prez/services/listings.py index 5b993d21..a07e2d28 100644 --- a/prez/services/listings.py +++ b/prez/services/listings.py @@ -16,6 +16,7 @@ async def listing_function( request: Request, repo: Repo, + system_repo: Repo, page: int = 1, per_page: int = 20, uri: str = None, @@ -61,7 +62,7 @@ async def listing_function( else: item_graph, _ = await repo.send_queries([count_query, item_members_query], []) if "anot+" in prof_and_mt_info.mediatype: - await _add_prez_links(item_graph, repo) + await _add_prez_links(item_graph, repo, system_repo) return await return_from_graph( item_graph, prof_and_mt_info.mediatype, diff --git a/prez/services/objects.py b/prez/services/objects.py index 9f975452..b096bb69 100644 --- a/prez/services/objects.py +++ b/prez/services/objects.py @@ -14,6 +14,7 @@ from prez.services.curie_functions import get_uri_for_curie_id from prez.services.model_methods import get_classes from prez.services.link_generation import _add_prez_links +from prez.sparql.methods import Repo from prez.sparql.objects_listings import ( generate_item_construct, generate_listing_construct, @@ -22,7 +23,8 @@ async def object_function( request: Request, - repo=Depends(get_repo), + repo: Repo, + system_repo=Repo, object_curie: Optional[str] = None, ): endpoint_uri = URIRef(request.scope["route"].name) @@ -81,7 +83,7 @@ async def object_function( else: item_graph, _ = await repo.send_queries([item_query, item_members_query], []) if "anot+" in prof_and_mt_info.mediatype: - await _add_prez_links(item_graph, repo) + await _add_prez_links(item_graph, repo, system_repo) return await return_from_graph( item_graph, prof_and_mt_info.mediatype,