Merge branch 'develop' into dependabot/pip/develop/pytest-env-1.0.1

.github/workflows/build-docker.yml

@@ -11,30 +11,30 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ghcr.io/investigativedata/ftmstore-fastapi
           tags: |
             type=ref,event=branch
             type=semver,pattern={{version}}
             type=sha
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           install: true
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and push release
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           # platforms: linux/amd64,linux/arm64

.github/workflows/python.yml

@@ -18,7 +18,7 @@ jobs:
         python-version: ["3.10", "3.11"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:

.pre-commit-config.yaml

@@ -68,7 +68,7 @@ repos:
       - id: rst-inline-touching-normal
 
   - repo: https://github.com/python-poetry/poetry
-    rev: 1.5.0
+    rev: 1.6.1
     hooks:
       - id: poetry-check
       - id: poetry-lock

Dockerfile

@@ -16,5 +16,4 @@ USER 1000
 ENV FTM_STORE_URI=sqlite:////data/followthemoney.store
 ENV CATALOG=/data/catalog.json
 
-# Run a single unicorn, scale on container level
-CMD ["uvicorn", "ftmstore_fastapi.api:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "8000"]
+ENTRYPOINT ["gunicorn", "ftmstore_fastapi.api:app", "--bind", "0.0.0.0:8000", "--worker-class", "uvicorn.workers.UvicornWorker"]

Makefile

@@ -11,7 +11,7 @@ followthemoney.store:
 	poetry run ftmq --store-dataset gdho -i ./tests/fixtures/gdho.ftm.json -o $(FTM_STORE_URI)
 
 test: followthemoney.store
-	poetry run pytest -s --cov=ftmstore_fastapi --cov-report term-missing
+	poetry run pytest -s --cov=ftmstore_fastapi --cov-report term-missing -v
 
 typecheck:
 	# pip install types-python-jose

docker-compose.yml

@@ -1,19 +1,21 @@
 services:
   api:
-    # use 1 worker per service, scale api services instead of gunicorn workers!
     build: .
+    command: "--workers 4"
     restart: unless-stopped
     ports:
       - 127.0.0.1:8000:8000
     links:
       - redis
     environment:
+      CATALOG_URI: /data/catalog.json
       FTM_STORE_URI: sqlite:////data/followthemoney.store
       REDIS_URL: redis://redis
       CACHE: 1
       PRELOAD_DATASETS: 1
     volumes:
       - ${DATA_ROOT:-.}/followthemoney.store:/data/followthemoney.store
+      - ${DATA_ROOT:-.}/tests/fixtures/catalog.json:/data/catalog.json
 
   redis:
     image: redis

ftmstore_fastapi/api.py

@@ -1,10 +1,8 @@
 import secrets
-from typing import Literal
 
 from fastapi import Depends, FastAPI, Query, Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import RedirectResponse
-from ftmq.settings import DB_STORE_URI
 
 from ftmstore_fastapi import settings, views
 from ftmstore_fastapi.logging import get_logger
@@ -17,7 +15,8 @@
     EntityResponse,
     ErrorResponse,
 )
-from ftmstore_fastapi.store import get_catalog
+from ftmstore_fastapi.settings import FTM_STORE_URI
+from ftmstore_fastapi.store import Datasets
 
 log = get_logger(__name__)
 
@@ -34,7 +33,7 @@
     allow_methods=["OPTIONS", "GET"],
 )
 
-log.info("Ftm store: %s" % DB_STORE_URI)
+log.info("Ftm store: %s" % FTM_STORE_URI)
 
 
 @app.get(
@@ -54,13 +53,8 @@ async def dataset_list(request: Request) -> CatalogResponse:
     return views.dataset_list(request)
 
 
-# cache at boot time
-catalog = get_catalog()
-Datasets = Literal[tuple(catalog.names)]
-
-
 @app.get(
-    "/{dataset}",
+    "/catalog/{dataset}",
     response_model=DatasetResponse,
     responses={
         500: {"model": ErrorResponse, "description": "Server error"},
@@ -86,16 +80,14 @@ def get_authenticated(
 
 
 @app.get(
-    "/{dataset}/entities",
+    "/entities",
     response_model=EntitiesResponse,
     responses={
         500: {"model": ErrorResponse, "description": "Server error"},
     },
 )
-async def list_entities(
+async def entities(
     request: Request,
-    dataset: Datasets,
-    q: str = Query(None, title="Search string"),
     params: QueryParams = Depends(QueryParams),
     retrieve_params: views.RetrieveParams = Depends(views.get_retrieve_params),
     authenticated: bool = Depends(get_authenticated),
@@ -110,9 +102,15 @@ async def list_entities(
     returned. This is e.g. useful for static site builders to reduce the data
     amount.
 
-    ## filter
+    ## dataset scope
+
+    Limit entities filter to one or more datasets from the catalog:
 
-    `/{dataset}/entities?schema=Company?country=de`
+    `/entities?dataset=my_dataset&dataset=another_dataset`
+
+    ## filter by schema and properties
+
+    `/entities?schema=Company?country=de`
 
     Filtering works for all [FollowTheMoney](https://followthemoney.tech/explorer/)
     properties
@@ -129,7 +127,7 @@ async def list_entities(
 
     Could be queried like this:
 
-    `/{dataset}/entities?name__ilike=%Jane%`
+    `/entities?name__ilike=%Jane%`
 
     ## sorting
 
@@ -143,21 +141,15 @@ async def list_entities(
 
     ## searching
 
-    Search entities in the [FTS5-Index](https://www.sqlite.org/fts5.html)
+    Search entities via the configured search backend.
 
     Use optional `q` parameter for a search term.
     """
-    return views.entity_list(
-        request,
-        dataset,
-        retrieve_params,
-        q=q,
-        authenticated=authenticated,
-    )
+    return views.entity_list(request, retrieve_params, authenticated=authenticated)
 
 
 @app.get(
-    "/{dataset}/entities/{entity_id}",
+    "/entities/{entity_id}",
     response_model=EntityResponse,
     responses={
         307: {"description": "The entity was merged into another ID"},
@@ -167,7 +159,6 @@ async def list_entities(
 )
 async def detail_entity(
     request: Request,
-    dataset: Datasets,
     entity_id: str,
     retrieve_params: views.RetrieveParams = Depends(views.get_retrieve_params),
 ) -> EntityResponse | RedirectResponse | ErrorResponse:
@@ -183,20 +174,18 @@ async def detail_entity(
         `x-entity-id` - the new entity id
         `x-entity-schema` - the new entity schema
     """
-    return views.entity_detail(request, dataset, entity_id, retrieve_params)
+    return views.entity_detail(request, entity_id, retrieve_params)
 
 
 @app.get(
-    "/{dataset}/aggregate",
+    "/aggregate",
     response_model=AggregationResponse,
     responses={
         500: {"model": ErrorResponse, "description": "Server error"},
     },
 )
 async def aggregation(
     request: Request,
-    dataset: Datasets,
-    q: str = Query(None, title="Search string"),
     params: QueryParams = Depends(QueryParams),
     aggregation_params: views.AggregationParams = Depends(views.get_aggregation_params),
     authenticated: bool = Depends(get_authenticated),
@@ -213,4 +202,4 @@ async def aggregation(
 
         ?aggMax=amount&aggMax=date
     """
-    return views.aggregation(request, dataset, q)
+    return views.aggregation(request)

ftmstore_fastapi/cache.py

@@ -28,7 +28,7 @@ def __init__(self):
             else:
                 con = redis.from_url(settings.REDIS_URL)
                 con.ping()
-                log.info("Redis connected: `{settings.REDIS_URL}`")
+                log.info(f"Redis connected: `{settings.REDIS_URL}`")
             self.cache = con
         else:
             self.cache = None

ftmstore_fastapi/query.py

@@ -1,13 +1,16 @@
-from typing import Any
+from typing import Annotated, Any
 
 from banal import clean_dict
-from fastapi import HTTPException, Request
+from fastapi import HTTPException
+from fastapi import Query as FastQuery
+from fastapi import Request
 from ftmq.aggregations import Aggregator
-from ftmq.enums import Properties, Schemata
+from ftmq.enums import Schemata
 from ftmq.query import Query as _Query
 from pydantic import BaseModel, Field, validator
 
 from ftmstore_fastapi import settings
+from ftmstore_fastapi.store import Datasets
 
 
 class RetrieveParams(BaseModel):
@@ -25,6 +28,13 @@ class AggregationParams(BaseModel):
 
 
 class QueryParams(BaseModel):
+    q: Annotated[
+        str | None, FastQuery(description="Optional search query for fuzzy search")
+    ] = None
+    dataset: Annotated[
+        list[Datasets] | None,
+        FastQuery(description="One or more dataset names to limit scope to"),
+    ] = []
     limit: int | None = settings.DEFAULT_LIMIT
     page: int | None = 1
     schema_: Schemata | None = Field(
@@ -33,21 +43,11 @@ class QueryParams(BaseModel):
         alias="schema",
     )
     order_by: str | None = Field(None, example="-date")
-    prop: str | None = Field(None, example="country")
-    value: str | None = Field(None, example="de")
+    reverse: str | None = Field(None, example="eu-id-1234")
 
     class Config:
         allow_population_by_field_name = True
 
-    @validator("prop")
-    def validate_prop(cls, prop: str | None) -> bool:
-        if prop is not None:
-            if prop == "reverse":
-                return prop
-            if prop not in Properties:
-                raise HTTPException(400, detail=[f"Invalid ftm property: `{prop}`"])
-        return prop
-
     @validator("schema_")
     def validate_schema(cls, value: str | None) -> bool:
         if value is not None and value not in Schemata:
@@ -61,7 +61,7 @@ def to_where_lookup_dict(self) -> dict[str, Any]:
 META_FIELDS = (
     set(AggregationParams.__fields__)
     | set(RetrieveParams.__fields__)  # noqa: W503
-    | set(QueryParams.__fields__) - {"prop", "value", "operator"}  # noqa: W503
+    | set(QueryParams.__fields__)  # noqa: W503
 )
 
 
@@ -78,7 +78,9 @@ def __init__(self, **data):
     def from_request(
         cls, request: Request, authenticated: bool | None = False
     ) -> "ViewQueryParams":
-        params = cls(**request.query_params)
+        params = dict(request.query_params)
+        params["dataset"] = request.query_params.getlist("dataset")
+        params = cls(**params)
         if not authenticated and params.limit > settings.DEFAULT_LIMIT:
             params.limit = settings.DEFAULT_LIMIT
         return params
@@ -95,12 +97,10 @@ def to_aggregator(self) -> Aggregator:
 
 class Query(_Query):
     @classmethod
-    def from_params(
-        cls: "Query", params: ViewQueryParams, dataset: str | None = None
-    ) -> "Query":
+    def from_params(cls: "Query", params: ViewQueryParams) -> "Query":
         q = cls()[(params.page - 1) * params.limit : params.page * params.limit]
-        if dataset is not None:
-            q = q.where(dataset=dataset)
+        if params.dataset:
+            q = q.where(dataset__in=params.dataset)
         if params.order_by:
             ascending = True
             if params.order_by.startswith("-"):
@@ -109,11 +109,11 @@ def from_params(
             q = q.order_by(params.order_by, ascending=ascending)
         if params.schema_:
             q = q.where(schema=params.schema_)
+        if params.reverse:
+            q = q.where(reverse=params.reverse)
         q = q.where(**params.to_where_lookup_dict())
+        if params.q:
+            q = q.search(params.q)
         aggregator = params.to_aggregator()
         q.aggregations = aggregator.aggregations
         return q
-
-
-class SearchQuery(_Query):
-    pass

ftmstore_fastapi/serialize.py

@@ -86,7 +86,7 @@ def from_view(
         query_data = clean_dict(query.dict())
         query_data.pop("schema_", None)
         url.args.update(query_data)
-        entities = [EntityResponse.from_entity(e) for e in entities]
+        entities = [EntityResponse.from_entity(e, adjacents) for e in entities]
         response = cls(
             total=coverage.entities,
             items=len(entities),
@@ -147,7 +147,7 @@ class DatasetResponse(Dataset):
     def from_dataset(cls, request: Request, dataset: Dataset) -> "DatasetResponse":
         return cls(
             **dataset.dict(),
-            entities_url=f"{request.base_url}{dataset.name}/entities",
+            entities_url=f"{request.base_url}entities?dataset={dataset.name}",
         )