From e1ee2e4ed12bdbb6f2bcc3d0448e6b158915635b Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Fri, 2 Feb 2024 18:16:56 +0100 Subject: [PATCH 01/44] First caching with redis working --- Docker/docker-compose.yml | 10 ++ python/python_fastapi_server/main.py | 12 +++ .../routers/cachingmiddleware.py | 93 +++++++++++++++++++ requirements.in | 4 +- requirements.txt | 3 + 5 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 python/python_fastapi_server/routers/cachingmiddleware.py diff --git a/Docker/docker-compose.yml b/Docker/docker-compose.yml index 6983b03f..faca171f 100755 --- a/Docker/docker-compose.yml +++ b/Docker/docker-compose.yml @@ -36,6 +36,7 @@ services: - "ADAGUC_AUTOWMS_DIR=/data/adaguc-autowms" - "ADAGUC_DATA_DIR=/data/adaguc-data" - "ADAGUC_DATASET_DIR=/data/adaguc-datasets" + - "ADAGUC_REDIS=redis://adaguc-redis:${REDIS_PORT}" env_file: - .env restart: unless-stopped @@ -43,6 +44,7 @@ services: - adaguc-db depends_on: - adaguc-db + - adaguc-redis logging: driver: "json-file" options: @@ -91,6 +93,14 @@ services: options: max-size: "200k" max-file: "10" + adaguc-redis: + image: redis:alpine + container_name: adaguc-redis + hostname: adaguc-redis + networks: + - adaguc-network + expose: + - "${REDIS_PORT}" volumes: adaguc-server-compose-adagucdb: networks: diff --git a/python/python_fastapi_server/main.py b/python/python_fastapi_server/main.py index 4a61b9cd..e35d6068 100644 --- a/python/python_fastapi_server/main.py +++ b/python/python_fastapi_server/main.py @@ -21,6 +21,7 @@ from routers.ogcapi import ogcApiApp from routers.opendap import opendapRouter from routers.wmswcs import testadaguc, wmsWcsRouter +from routers.cachingmiddleware import CachingMiddleware logger = logging.getLogger(__name__) @@ -32,6 +33,15 @@ app.add_middleware(AccessLoggerMiddleware, format=access_log_format) logging.getLogger("access").propagate = False +@app.middleware("http") +async def add_process_time_header(request: Request, call_next): + start_time = time.time() + response = await call_next(request) + process_time = time.time() - start_time + print("Adding X-Process-Time") + response.headers["X-Process-Time"] = str(process_time) + return response + @app.middleware("http") async def add_process_time_header(request: Request, call_next): start_time = time.time() @@ -65,6 +75,8 @@ async def add_hsts_header(request: Request, call_next): app.add_middleware(BrotliMiddleware, gzip_fallback=True) +app.add_middleware(CachingMiddleware) + if "EXTERNALADDRESS" in os.environ: app.add_middleware(FixSchemeMiddleware) diff --git a/python/python_fastapi_server/routers/cachingmiddleware.py b/python/python_fastapi_server/routers/cachingmiddleware.py new file mode 100644 index 00000000..f01580c0 --- /dev/null +++ b/python/python_fastapi_server/routers/cachingmiddleware.py @@ -0,0 +1,93 @@ +import os +from urllib.parse import urlsplit + +from starlette.concurrency import iterate_in_threadpool +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.responses import StreamingResponse, Response +from starlette.background import BackgroundTask + +import calendar +from datetime import datetime, timedelta +import time +from redis import asyncio +import json + +ADAGUC_REDIS=os.environ.get('ADAGUC_REDIS', "redis://localhost:6379") +redis = asyncio.from_url(ADAGUC_REDIS) + +async def get_cached_response(request): + key = generate_key(request) + cached = await redis.get(key) + if not cached: + print("Cache miss") + return None, None, None + print("Cache hit") + + entrytime=int(cached[:10]) + currenttime=calendar.timegm(datetime.utcnow().utctimetuple()) + age=currenttime-entrytime + + headers_len=int(cached[10:16]) + print("HL:", headers_len) + print("C",type(cached), cached[16:16+headers_len]) + headers=json.loads(cached[16:16+headers_len]) + + data = cached[16+headers_len:] + return age, headers, data + +skip_headers=["x-process-time"] + +async def cache_response(request, headers, data, ex: int=60): + key=generate_key(request) + + allheaders={} + for k in headers.keys(): + if k not in skip_headers: + allheaders[k]=headers[k] + else: + print("skipping header", k) + allheaders_json=json.dumps(allheaders) + + entrytime="%10d"%calendar.timegm(datetime.utcnow().utctimetuple()) + print("ENTRY:", entrytime) + + print("Caching ", key, allheaders_json, "<><><>", type(data), data[:80]) + await redis.set(key, bytes(entrytime, 'utf-8')+bytes("%06d"%len(allheaders_json), 'utf-8')+bytes(allheaders_json, 'utf-8')+data, ex=ex) + +def generate_key(request): + key = request['query_string'] + return key + +class CachingMiddleware(BaseHTTPMiddleware): + def __init__(self, app): + super().__init__(app) + + async def dispatch(self, request, call_next): + #Check if request is in cache, if so return that + expire, headers, data = await get_cached_response(request) + print("AGE:", expire, data[:20] if data else None) + + if data: + #Fix Age header + headers["Age"]="%1d"%(expire) + return Response(content=data, status_code=200, headers=headers, media_type="text/xml") + + response: Response = await call_next(request) + + if response.status_code == 200: + if "cache-control" in response.headers and response.headers['cache-control']!="no-store": + print("CachingMiddleware:", response.headers['cache-control']) + age_terms = response.headers['cache-control'].split("=") + if age_terms[0].lower()!="max-age": + return + age=int(age_terms[1]) + response_body = b"" + async for chunk in response.body_iterator: + response_body+=chunk + task = BackgroundTask(cache_response, request=request, headers=response.headers, data=response_body, ex=age) + # await cache_response(request, response.headers, response_body, age) + response.headers['age']="0" + print("HDRS:",response.headers) + return Response(content=response_body, status_code=200, headers=response.headers, media_type=response.media_type, background=task) + print("NOT CACHING ", generate_key(request)) + return response diff --git a/requirements.in b/requirements.in index 557ad11f..5151a16f 100644 --- a/requirements.in +++ b/requirements.in @@ -24,4 +24,6 @@ pytz~=2023.3 uvicorn~=0.23.2 covjson-pydantic~=0.2.0 geomet~=1.0 -edr-pydantic~=0.2.0 \ No newline at end of file +edr-pydantic~=0.2.0 +redis~=5.0 +aioredis~=2.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 9ada52ff..c78156fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ # # pip-compile --no-emit-index-url # +aioredis==2.0.1 annotated-types==0.6.0 # via pydantic anyio==3.7.1 @@ -15,6 +16,7 @@ asgi-logger==0.1.0 # via -r requirements.in asgiref==3.7.2 # via asgi-logger +async-timeout==4.0.3 brotli==1.1.0 # via # -r requirements.in @@ -107,6 +109,7 @@ pytz==2023.4 # owslib pyyaml==6.0.1 # via owslib +redis==5.0.1 requests==2.31.0 # via owslib six==1.16.0 From 2a99e583e61840fefb522a8e28e325cd752cb3a0 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Fri, 2 Feb 2024 21:59:42 +0100 Subject: [PATCH 02/44] Cleaned printing --- python/python_fastapi_server/routers/cachingmiddleware.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python/python_fastapi_server/routers/cachingmiddleware.py b/python/python_fastapi_server/routers/cachingmiddleware.py index f01580c0..c787adf1 100644 --- a/python/python_fastapi_server/routers/cachingmiddleware.py +++ b/python/python_fastapi_server/routers/cachingmiddleware.py @@ -28,8 +28,6 @@ async def get_cached_response(request): age=currenttime-entrytime headers_len=int(cached[10:16]) - print("HL:", headers_len) - print("C",type(cached), cached[16:16+headers_len]) headers=json.loads(cached[16:16+headers_len]) data = cached[16+headers_len:] @@ -70,7 +68,7 @@ async def dispatch(self, request, call_next): if data: #Fix Age header headers["Age"]="%1d"%(expire) - return Response(content=data, status_code=200, headers=headers, media_type="text/xml") + return Response(content=data, status_code=200, headers=headers, media_type=headers['content-type']) response: Response = await call_next(request) @@ -87,7 +85,7 @@ async def dispatch(self, request, call_next): task = BackgroundTask(cache_response, request=request, headers=response.headers, data=response_body, ex=age) # await cache_response(request, response.headers, response_body, age) response.headers['age']="0" - print("HDRS:",response.headers) + # print("HDRS:",response.headers) return Response(content=response_body, status_code=200, headers=response.headers, media_type=response.media_type, background=task) print("NOT CACHING ", generate_key(request)) return response From 4675cdb800a0a4789aec92e64c7cb5ed4dd0d9ab Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Fri, 2 Feb 2024 22:22:43 +0100 Subject: [PATCH 03/44] async response streaming --- .../python_fastapi_server/routers/cachingmiddleware.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/python_fastapi_server/routers/cachingmiddleware.py b/python/python_fastapi_server/routers/cachingmiddleware.py index c787adf1..5dd58352 100644 --- a/python/python_fastapi_server/routers/cachingmiddleware.py +++ b/python/python_fastapi_server/routers/cachingmiddleware.py @@ -80,12 +80,16 @@ async def dispatch(self, request, call_next): return age=int(age_terms[1]) response_body = b"" - async for chunk in response.body_iterator: - response_body+=chunk + # async for chunk in response.body_iterator: + # response_body+=chunk + async def response_chunks(): + async for chunk in response.body_iterator: + yield chunk task = BackgroundTask(cache_response, request=request, headers=response.headers, data=response_body, ex=age) # await cache_response(request, response.headers, response_body, age) response.headers['age']="0" # print("HDRS:",response.headers) - return Response(content=response_body, status_code=200, headers=response.headers, media_type=response.media_type, background=task) + # return Response(content=response_body, status_code=200, headers=response.headers, media_type=response.media_type, background=task) + return StreamingResponse(content=response_chunks(), status_code=200, headers=response.headers, media_type=response.media_type, background=task) print("NOT CACHING ", generate_key(request)) return response From 13b5899de0ca2c9f4bd103b234c3c39ab967e57d Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Fri, 2 Feb 2024 22:48:34 +0100 Subject: [PATCH 04/44] Adding headers from call_adaguc --- python/python_fastapi_server/main.py | 1 - python/python_fastapi_server/routers/edr.py | 11 +++++++---- python/python_fastapi_server/routers/ogcapi_tools.py | 7 +++++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/python/python_fastapi_server/main.py b/python/python_fastapi_server/main.py index e35d6068..50e9cc8f 100644 --- a/python/python_fastapi_server/main.py +++ b/python/python_fastapi_server/main.py @@ -38,7 +38,6 @@ async def add_process_time_header(request: Request, call_next): start_time = time.time() response = await call_next(request) process_time = time.time() - start_time - print("Adding X-Process-Time") response.headers["X-Process-Time"] = str(process_time) return response diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index ce099966..7e8aa063 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -195,7 +195,7 @@ def get_point_value( if custom_dims: urlrequest += custom_dims - status, response = call_adaguc(url=urlrequest.encode("UTF-8")) + status, response, headers = call_adaguc(url=urlrequest.encode("UTF-8")) if status == 0: return response.getvalue() return None @@ -217,10 +217,12 @@ async def get_collection_position( collection_name: str, request: Request, coords: str, + response: CovJSONResponse, instance: Union[str, None] = None, datetime_par: str = Query(default=None, alias="datetime"), parameter_name: str = Query(alias="parameter-name"), z_par: str = Query(alias="z", default=None), + ) -> Coverage: """ returns data for the EDR /position endpoint @@ -237,7 +239,7 @@ async def get_collection_position( latlons = wkt.loads(coords) logger.info("latlons:%s", latlons) coord = {"lat": latlons["coordinates"][1], "lon": latlons["coordinates"][0]} - resp = get_point_value( + resp, headers = get_point_value( edr_collections[collection_name], instance, [coord["lon"], coord["lat"]], @@ -247,6 +249,7 @@ async def get_collection_position( custom_dims, ) if resp: + response.headers['X-Special']='OK' dat = json.loads(resp) return covjson_from_resp(dat, edr_collections[collection_name]["vertical_name"]) @@ -628,7 +631,7 @@ def get_capabilities(collname): urlrequest = ( f"dataset={dataset}&service=wms&version=1.3.0&request=getcapabilities" ) - status, response = call_adaguc(url=urlrequest.encode("UTF-8")) + status, response, headers = call_adaguc(url=urlrequest.encode("UTF-8")) logger.info("status: %d", status) if status == 0: xml = response.getvalue() @@ -649,7 +652,7 @@ def get_ref_times_for_coll(edr_collectioninfo: dict, layer: str) -> list[str]: dataset = edr_collectioninfo["dataset"] url = f"?DATASET={dataset}&SERVICE=WMS&VERSION=1.3.0&request=getreferencetimes&LAYER={layer}" logger.info("getreftime_url(%s,%s): %s", dataset, layer, url) - status, response = call_adaguc(url=url.encode("UTF-8")) + status, response, headers = call_adaguc(url=url.encode("UTF-8")) if status == 0: ref_times = json.loads(response.getvalue()) instance_ids = [parse_iso(reft).strftime("%Y%m%d%H%M") for reft in ref_times] diff --git a/python/python_fastapi_server/routers/ogcapi_tools.py b/python/python_fastapi_server/routers/ogcapi_tools.py index b1ad4637..22fabc04 100644 --- a/python/python_fastapi_server/routers/ogcapi_tools.py +++ b/python/python_fastapi_server/routers/ogcapi_tools.py @@ -116,7 +116,7 @@ def call_adaguc(url): if len(logfile) > 0: logger.info(logfile) - return status, data + return status, data, headers @cached(cache=cache) @@ -131,7 +131,10 @@ def get_capabilities(collname): urlrequest = ( f"dataset={dataset}&service=wms&version=1.3.0&request=getcapabilities" ) - status, response = call_adaguc(url=urlrequest.encode("UTF-8")) + status, response, headers = call_adaguc(url=urlrequest.encode("UTF-8")) + for hdr in headers.keys(): + if hdr.lower()=="cache-control": + logger.info("%s: %s", hdr, headers[hdr]) if status == 0: xml = response.getvalue() wms = WebMapService(coll["service"], xml=xml, version="1.3.0") From 569f00afde63c55d4fd38f0643aa1109ca87a43a Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Mon, 5 Feb 2024 22:34:48 +0100 Subject: [PATCH 05/44] Added caching (redis or local) to fastapi/EDR --- adagucserverEC/CImageDataWriter.cpp | 11 +- adagucserverEC/CRequest.cpp | 4 +- .../edr_pydantic_classes/__init__.py | 0 .../edr_pydantic_classes/capabilities.py | 65 +++++++ .../edr_pydantic_classes/generic_models.py | 119 +++++++++++++ .../edr_pydantic_classes/instances.py | 164 ++++++++++++++++++ .../edr_pydantic_classes/my_base_model.py | 37 ++++ .../edr_pydantic_classes/py.typed | 0 python/edr_package.MINE/setup.py | 31 ++++ .../edr_pydantic_classes/__init__.py | 0 .../edr_pydantic_classes/capabilities.py | 65 +++++++ .../edr_pydantic_classes/generic_models.py | 114 ++++++++++++ .../edr_pydantic_classes/instances.py | 164 ++++++++++++++++++ .../edr_pydantic_classes/my_base_model.py | 37 ++++ .../edr_pydantic_classes/py.typed | 0 python/edr_package.ORG/setup.py | 31 ++++ python/python_fastapi_server/main.py | 4 +- .../routers/cachingmiddleware.py | 36 ++-- python/python_fastapi_server/routers/edr.py | 69 +++++--- .../routers/ogcapi_tools.py | 7 +- requirements.in | 3 +- requirements.txt | 2 + 22 files changed, 903 insertions(+), 60 deletions(-) create mode 100644 python/edr_package.MINE/edr_pydantic_classes/__init__.py create mode 100644 python/edr_package.MINE/edr_pydantic_classes/capabilities.py create mode 100644 python/edr_package.MINE/edr_pydantic_classes/generic_models.py create mode 100644 python/edr_package.MINE/edr_pydantic_classes/instances.py create mode 100644 python/edr_package.MINE/edr_pydantic_classes/my_base_model.py create mode 100644 python/edr_package.MINE/edr_pydantic_classes/py.typed create mode 100755 python/edr_package.MINE/setup.py create mode 100644 python/edr_package.ORG/edr_pydantic_classes/__init__.py create mode 100644 python/edr_package.ORG/edr_pydantic_classes/capabilities.py create mode 100644 python/edr_package.ORG/edr_pydantic_classes/generic_models.py create mode 100644 python/edr_package.ORG/edr_pydantic_classes/instances.py create mode 100644 python/edr_package.ORG/edr_pydantic_classes/my_base_model.py create mode 100644 python/edr_package.ORG/edr_pydantic_classes/py.typed create mode 100755 python/edr_package.ORG/setup.py diff --git a/adagucserverEC/CImageDataWriter.cpp b/adagucserverEC/CImageDataWriter.cpp index d73476b3..6fe91b02 100644 --- a/adagucserverEC/CImageDataWriter.cpp +++ b/adagucserverEC/CImageDataWriter.cpp @@ -2414,10 +2414,10 @@ int CImageDataWriter::end() { CT::string resultJSON; if (srvParam->JSONP.length() == 0) { CDBDebug("CREATING JSON"); - printf("%s%c%c\n", "Content-Type: application/json", 13, 10); + printf("%s%s%c%c\n", "Content-Type: application/json", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); } else { CDBDebug("CREATING JSONP %s", srvParam->JSONP.c_str()); - printf("%s%c%c\n%s(", "Content-Type: application/javascript", 13, 10, srvParam->JSONP.c_str()); + printf("%s%s%c%c\n", "Content-Type: application/javascript", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); } puts(data.c_str()); @@ -2438,9 +2438,9 @@ int CImageDataWriter::end() { if (resultFormat == textplain || resultFormat == texthtml) { CT::string resultHTML; if (resultFormat == textplain) { - resultHTML.print("%s%c%c\n", "Content-Type:text/plain", 13, 10); + resultHTML.print("%s%s%c%c\n", "Content-Type: text/plain", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); } else { - resultHTML.print("%s%c%c\n", "Content-Type:text/html", 13, 10); + resultHTML.print("%s%s%c%c\n", "Content-Type: text/html", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); } if (resultFormat == texthtml) resultHTML.printconcat("\n"); @@ -2568,7 +2568,8 @@ int CImageDataWriter::end() { if (resultFormat == applicationvndogcgml) { CDBDebug("CREATING GML"); CT::string resultXML; - resultXML.print("%s%c%c\n", "Content-Type:text/xml", 13, 10); + resultXML.print("%s%s%c%c\n", "Content-Type: text/xml", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); + resultXML.printconcat("\n"); resultXML.printconcat(" JSONP.length() == 0) { - printf("%s%c%c\n", "Content-Type: application/json ", 13, 10); + printf("%s%s%c%c\n", "Content-Type: application/json ", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); printf("%s", XMLdocument.c_str()); } else { - printf("%s%c%c\n", "Content-Type: application/javascript ", 13, 10); + printf("%s%s%c%c\n", "Content-Type: application/javascript ", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); printf("%s(%s)", srvParam->JSONP.c_str(), XMLdocument.c_str()); } diff --git a/python/edr_package.MINE/edr_pydantic_classes/__init__.py b/python/edr_package.MINE/edr_pydantic_classes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/edr_package.MINE/edr_pydantic_classes/capabilities.py b/python/edr_package.MINE/edr_pydantic_classes/capabilities.py new file mode 100644 index 00000000..5a000490 --- /dev/null +++ b/python/edr_package.MINE/edr_pydantic_classes/capabilities.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from typing import List +from typing import Optional + +from pydantic import Field + +from .generic_models import Link +from .my_base_model import MyBaseModel + + +class Provider(MyBaseModel): + name: str + url: Optional[str] + + +class Contact(MyBaseModel): + email: Optional[str] + phone: Optional[str] + fax: Optional[str] + hours: Optional[str] + instructions: Optional[str] + address: Optional[str] + postalCode: Optional[str] # noqa: N815 + city: Optional[str] + stateorprovince: Optional[str] + country: Optional[str] + + +class LandingPageModel(MyBaseModel): + links: List[Link] = Field( + ..., + example=[ + { + "href": "http://www.example.org/edr", + "hreflang": "en", + "rel": "service-desc", + "type": "application/vnd.oai.openapi+json;version=3.0", + "title": "", + }, + { + "href": "http://www.example.org/edr/conformance", + "hreflang": "en", + "rel": "data", + "type": "application/json", + "title": "", + }, + { + "href": "http://www.example.org/edr/collections", + "hreflang": "en", + "rel": "data", + "type": "application/json", + "title": "", + }, + ], + ) + title: Optional[str] + description: Optional[str] + keywords: Optional[List[str]] + provider: Optional[Provider] + contact: Optional[Contact] + + +class ConformanceModel(MyBaseModel): + conformsTo: List[str] # noqa: N815 diff --git a/python/edr_package.MINE/edr_pydantic_classes/generic_models.py b/python/edr_package.MINE/edr_pydantic_classes/generic_models.py new file mode 100644 index 00000000..b6917886 --- /dev/null +++ b/python/edr_package.MINE/edr_pydantic_classes/generic_models.py @@ -0,0 +1,119 @@ +from typing import List +from typing import Optional, Union +from enum import Enum + +from datetime import datetime + +from covjson_pydantic.domain import DomainType +from pydantic import Field + +from .my_base_model import MyBaseModel + + +class CRSOptions(str, Enum): + wgs84 = "WGS84" + + +class Spatial(MyBaseModel): + bbox: list[list[float]] + crs: CRSOptions + name: Optional[str] + + +class Temporal(MyBaseModel): + interval: list[list[datetime]] + values: list[str] + trs: str + name: Optional[str] + + +class Vertical(MyBaseModel): + interval: List[List[float]] + vrs: str + values: list[str] + name: Optional[str] + + +class Custom(MyBaseModel): + interval: List[str] + id: str + values: List[str] + reference: Optional[str] = None + + +class Extent(MyBaseModel): + spatial: Optional[Spatial] + temporal: Optional[Temporal] + vertical: Optional[Vertical] + custom: Optional[List[Custom]] + + +class CrsObject(MyBaseModel): + crs: str + wkt: str + + +class ObservedPropertyCollection(MyBaseModel): + id: Optional[str] = None + label: str + description: Optional[str] = None + # categories + + + +class Symbol(MyBaseModel, extra="allow"): + value: str + type: str + +class Unit(MyBaseModel): + label: Optional[str] + symbol: Optional[str] + id: Optional[Union[str, Symbol]] = None + + +class ParameterName(MyBaseModel): + id: Optional[str] = None + type: str = "Parameter" + label: Optional[str] = None + description: Optional[str] = None + data_type: Optional[str] = None + observedProperty: ObservedPropertyCollection + extent: Optional[Extent] = None + unit: Optional[Unit] = None + + +class Variables(MyBaseModel): + crs_details: list[CrsObject] + default_output_format: Optional[str] = None + output_formats: list[str] = [] + query_type: str = "" + title: str = "" + + +class Link(MyBaseModel): + href: str = Field( + ..., + example="http://data.example.com/collections/monitoringsites/locations/1234", + ) + rel: str = Field(..., example="alternate") + type: Optional[str] = Field(None, example="application/geo+json") + hreflang: Optional[str] = Field(None, example="en") + title: Optional[str] = Field(None, example="Monitoring site name") + length: Optional[int] = None + templated: Optional[bool] = Field( + False, + description="defines if the link href value is a template with values requiring replacement", + ) + variables: Optional[Variables] + + +class SupportedQueries(MyBaseModel): + domain_types: List[DomainType] = Field( + description="A list of domain types from which can be determined what endpoints are allowed.", + example="When [DomainType.point_series] is returned, " + "the /position endpoint is allowed but /cube is not allowed.", + ) + has_locations: bool = Field( + description="A boolean from which can be determined if the backend has the /locations endpoint.", + example="When True is returned, the backend has the /locations endpoint.", + ) diff --git a/python/edr_package.MINE/edr_pydantic_classes/instances.py b/python/edr_package.MINE/edr_pydantic_classes/instances.py new file mode 100644 index 00000000..40b532a9 --- /dev/null +++ b/python/edr_package.MINE/edr_pydantic_classes/instances.py @@ -0,0 +1,164 @@ +# generated by datamodel-codegen: +# filename: https://schemas.opengis.net/ogcapi/edr/1.0/openapi/schemas/instances.yaml +# timestamp: 2022-06-29T09:43:11+00:00 +from __future__ import annotations + + +from typing import Dict +from typing import List +from typing import Optional + +from pydantic import Field + +from .generic_models import Link, CrsObject, Variables, ParameterName, Extent +from .my_base_model import MyBaseModel + + +class InstancesVariables(Variables): + query_type: str = "instances" + title: str = "Instances query" + + +class InstancesLink(Link): + variables: InstancesVariables + + +class InstancesDataQueryLink(MyBaseModel): + link: InstancesLink + + +class PositionVariables(Variables): + query_type: str = "position" + title: str = "Position query" + coords: str = "Well Known Text POINT value i.e. POINT(24.9384 60.1699)" + + +class PositionLink(Link): + variables: PositionVariables + + +class PositionDataQueryLink(MyBaseModel): + link: PositionLink + + +class DataQueries(MyBaseModel): + position: Optional[PositionDataQueryLink] = None + # radius: Optional[DataQueryLink] = None + # area: Optional[DataQueryLink] = None + # cube: Optional[DataQueryLink] = None + # trajectory: Optional[DataQueryLink] = None + # corridor: Optional[DataQueryLink] = None + # locations: Optional[DataQueryLink] = None + # items: Optional[DataQueryLink] = None + instances: Optional[InstancesDataQueryLink] = None + + +class Instance(MyBaseModel): + links: List[Link] = Field( + ..., + example=[ + { + "href": "https://wwww.example.org/service/description.html", + "hreflang": "en", + "rel": "service-doc", + "type": "text/html", + "title": "", + }, + { + "href": "https://www.example.org/service/licence.html", + "hreflang": "en", + "rel": "licence", + "type": "text/html", + "title": "", + }, + { + "href": "https://www.example.org/service/terms-and-conditions.html", + "hreflang": "en", + "rel": "restrictions", + "type": "text/html", + "title": "", + }, + { + "href": "http://www.example.org/edr/collections/the_collection_id/", + "hreflang": "en", + "rel": "collection", + "type": "collection", + "title": "Collection", + }, + { + "href": "http://www.example.org/edr/collections/the_collection_id/position", + "hreflang": "en", + "rel": "data", + "type": "position", + "title": "Position", + }, + ], + ) + id: str + title: Optional[str] + description: Optional[str] + keywords: Optional[List[str]] + extent: Extent + data_queries: Optional[DataQueries] + crs: Optional[str | List[str]] + output_formats: Optional[List[str]] + parameter_names: Optional[Dict[str, ParameterName]] + + +class InstancesModel(MyBaseModel): + links: List[Link] = Field( + ..., + example=[ + { + "href": "http://www.example.org/edr/collections/the_collection_id/instances", + "hreflang": "en", + "rel": "self", + "type": "application/json", + }, + { + "href": "http://www.example.org/edr/collections/the_collection_id/instances?f=html", + "hreflang": "en", + "rel": "alternate", + "type": "text/html", + }, + { + "href": "http://www.example.org/edr/collections/the_collection_id/instances?f=xml", + "hreflang": "en", + "rel": "alternate", + "type": "application/xml", + }, + ], + ) + instances: List[Instance] + + +# For now, the collection metadata corresponds to the first instance metadata. So they have equal classes +class Collection(Instance): + pass + + +class CollectionsModel(MyBaseModel): + links: List[Link] = Field( + ..., + example=[ + { + "href": "http://www.example.org/edr/collections", + "hreflang": "en", + "rel": "self", + "type": "application/json", + }, + { + "href": "http://www.example.org/edr/collections?f=html", + "hreflang": "en", + "rel": "alternate", + "type": "text/html", + }, + { + "href": "http://www.example.org/edr/collections?f=xml", + "hreflang": "en", + "rel": "alternate", + "type": "application/xml", + }, + ], + ) + collections: List[Collection] diff --git a/python/edr_package.MINE/edr_pydantic_classes/my_base_model.py b/python/edr_package.MINE/edr_pydantic_classes/my_base_model.py new file mode 100644 index 00000000..d6291d2d --- /dev/null +++ b/python/edr_package.MINE/edr_pydantic_classes/my_base_model.py @@ -0,0 +1,37 @@ +import orjson +from pydantic import BaseModel +from pydantic import Extra + +from datetime import datetime +from zoneinfo import ZoneInfo + + +def orjson_dumps(v, *, default): + # orjson.dumps returns bytes, to match standard json.dumps we need to decode + return orjson.dumps( + v, + default=default, + option=orjson.OPT_NON_STR_KEYS | orjson.OPT_UTC_Z | orjson.OPT_NAIVE_UTC, + ).decode() + + +def convert_datetime_to_gmt(dt: datetime) -> str: + if not dt.tzinfo: + dt = dt.replace(tzinfo=ZoneInfo("UTC")) + + return dt.strftime("%Y-%m-%dT%H:%M:%SZ") + + +class MyBaseModel(BaseModel): + class Config: + allow_population_by_field_name = True + anystr_strip_whitespace = True + extra = Extra.forbid + min_anystr_length = 1 + smart_union = True + validate_all = True + validate_assignment = True + + json_loads = orjson.loads + json_dumps = orjson_dumps + json_encoders = {datetime: convert_datetime_to_gmt} diff --git a/python/edr_package.MINE/edr_pydantic_classes/py.typed b/python/edr_package.MINE/edr_pydantic_classes/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/python/edr_package.MINE/setup.py b/python/edr_package.MINE/setup.py new file mode 100755 index 00000000..bdb32bf4 --- /dev/null +++ b/python/edr_package.MINE/setup.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import logging +import os + +import setuptools + +logging.basicConfig() +logger = logging.getLogger(__name__) +logger.setLevel(os.environ.get("LOG_LEVEL", "INFO")) + +# Package meta-data. +NAME = "edr-pydantic-classes" + +env_suffix = os.environ.get("ENVIRONMENT_SUFFIX", "") +logger.debug(f"Environment suffix: {env_suffix}") + +if env_suffix: + NAME += f"-{env_suffix}" +logger.debug(f"Package name: {NAME}") + +setuptools.setup( + name=NAME, + version="0.0.9", + description="The Pydantic models for EDR datatypes", + package_data={"edr_pydantic_classes": ["py.typed"]}, + packages=["edr_pydantic_classes"], + include_package_data=False, + license="MIT", + install_requires=["pydantic", "orjson", "covjson-pydantic"], + python_requires=">=3.8.0", +) diff --git a/python/edr_package.ORG/edr_pydantic_classes/__init__.py b/python/edr_package.ORG/edr_pydantic_classes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/edr_package.ORG/edr_pydantic_classes/capabilities.py b/python/edr_package.ORG/edr_pydantic_classes/capabilities.py new file mode 100644 index 00000000..5a000490 --- /dev/null +++ b/python/edr_package.ORG/edr_pydantic_classes/capabilities.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from typing import List +from typing import Optional + +from pydantic import Field + +from .generic_models import Link +from .my_base_model import MyBaseModel + + +class Provider(MyBaseModel): + name: str + url: Optional[str] + + +class Contact(MyBaseModel): + email: Optional[str] + phone: Optional[str] + fax: Optional[str] + hours: Optional[str] + instructions: Optional[str] + address: Optional[str] + postalCode: Optional[str] # noqa: N815 + city: Optional[str] + stateorprovince: Optional[str] + country: Optional[str] + + +class LandingPageModel(MyBaseModel): + links: List[Link] = Field( + ..., + example=[ + { + "href": "http://www.example.org/edr", + "hreflang": "en", + "rel": "service-desc", + "type": "application/vnd.oai.openapi+json;version=3.0", + "title": "", + }, + { + "href": "http://www.example.org/edr/conformance", + "hreflang": "en", + "rel": "data", + "type": "application/json", + "title": "", + }, + { + "href": "http://www.example.org/edr/collections", + "hreflang": "en", + "rel": "data", + "type": "application/json", + "title": "", + }, + ], + ) + title: Optional[str] + description: Optional[str] + keywords: Optional[List[str]] + provider: Optional[Provider] + contact: Optional[Contact] + + +class ConformanceModel(MyBaseModel): + conformsTo: List[str] # noqa: N815 diff --git a/python/edr_package.ORG/edr_pydantic_classes/generic_models.py b/python/edr_package.ORG/edr_pydantic_classes/generic_models.py new file mode 100644 index 00000000..ccbe666d --- /dev/null +++ b/python/edr_package.ORG/edr_pydantic_classes/generic_models.py @@ -0,0 +1,114 @@ +from typing import List +from typing import Optional +from enum import Enum + +from datetime import datetime + +from covjson_pydantic.domain import DomainType +from pydantic import Field + +from .my_base_model import MyBaseModel + + +class CRSOptions(str, Enum): + wgs84 = "WGS84" + + +class Spatial(MyBaseModel): + bbox: list[list[float]] + crs: CRSOptions + name: Optional[str] + + +class Temporal(MyBaseModel): + interval: list[list[datetime]] + values: list[str] + trs: str + name: Optional[str] + + +class Vertical(MyBaseModel): + interval: List[List[float]] + vrs: str + values: list[str] + name: Optional[str] + + +class Custom(MyBaseModel): + interval: List[str] + id: str + values: List[str] + reference: Optional[str] = None + + +class Extent(MyBaseModel): + spatial: Optional[Spatial] + temporal: Optional[Temporal] + vertical: Optional[Vertical] + custom: Optional[List[Custom]] + + +class CrsObject(MyBaseModel): + crs: str + wkt: str + + +class ObservedPropertyCollection(MyBaseModel): + id: Optional[str] = None + label: str + description: Optional[str] = None + # categories + + +class Units(MyBaseModel): + label: Optional[str] + symbol: Optional[str] + id: Optional[str] = None + + +class ParameterName(MyBaseModel): + id: Optional[str] = None + type: str = "Parameter" + label: Optional[str] = None + description: Optional[str] = None + data_type: Optional[str] = None + observedProperty: ObservedPropertyCollection + extent: Optional[Extent] = None + unit: Optional[Units] = None + + +class Variables(MyBaseModel): + crs_details: list[CrsObject] + default_output_format: Optional[str] = None + output_formats: list[str] = [] + query_type: str = "" + title: str = "" + + +class Link(MyBaseModel): + href: str = Field( + ..., + example="http://data.example.com/collections/monitoringsites/locations/1234", + ) + rel: str = Field(..., example="alternate") + type: Optional[str] = Field(None, example="application/geo+json") + hreflang: Optional[str] = Field(None, example="en") + title: Optional[str] = Field(None, example="Monitoring site name") + length: Optional[int] = None + templated: Optional[bool] = Field( + False, + description="defines if the link href value is a template with values requiring replacement", + ) + variables: Optional[Variables] + + +class SupportedQueries(MyBaseModel): + domain_types: List[DomainType] = Field( + description="A list of domain types from which can be determined what endpoints are allowed.", + example="When [DomainType.point_series] is returned, " + "the /position endpoint is allowed but /cube is not allowed.", + ) + has_locations: bool = Field( + description="A boolean from which can be determined if the backend has the /locations endpoint.", + example="When True is returned, the backend has the /locations endpoint.", + ) diff --git a/python/edr_package.ORG/edr_pydantic_classes/instances.py b/python/edr_package.ORG/edr_pydantic_classes/instances.py new file mode 100644 index 00000000..40b532a9 --- /dev/null +++ b/python/edr_package.ORG/edr_pydantic_classes/instances.py @@ -0,0 +1,164 @@ +# generated by datamodel-codegen: +# filename: https://schemas.opengis.net/ogcapi/edr/1.0/openapi/schemas/instances.yaml +# timestamp: 2022-06-29T09:43:11+00:00 +from __future__ import annotations + + +from typing import Dict +from typing import List +from typing import Optional + +from pydantic import Field + +from .generic_models import Link, CrsObject, Variables, ParameterName, Extent +from .my_base_model import MyBaseModel + + +class InstancesVariables(Variables): + query_type: str = "instances" + title: str = "Instances query" + + +class InstancesLink(Link): + variables: InstancesVariables + + +class InstancesDataQueryLink(MyBaseModel): + link: InstancesLink + + +class PositionVariables(Variables): + query_type: str = "position" + title: str = "Position query" + coords: str = "Well Known Text POINT value i.e. POINT(24.9384 60.1699)" + + +class PositionLink(Link): + variables: PositionVariables + + +class PositionDataQueryLink(MyBaseModel): + link: PositionLink + + +class DataQueries(MyBaseModel): + position: Optional[PositionDataQueryLink] = None + # radius: Optional[DataQueryLink] = None + # area: Optional[DataQueryLink] = None + # cube: Optional[DataQueryLink] = None + # trajectory: Optional[DataQueryLink] = None + # corridor: Optional[DataQueryLink] = None + # locations: Optional[DataQueryLink] = None + # items: Optional[DataQueryLink] = None + instances: Optional[InstancesDataQueryLink] = None + + +class Instance(MyBaseModel): + links: List[Link] = Field( + ..., + example=[ + { + "href": "https://wwww.example.org/service/description.html", + "hreflang": "en", + "rel": "service-doc", + "type": "text/html", + "title": "", + }, + { + "href": "https://www.example.org/service/licence.html", + "hreflang": "en", + "rel": "licence", + "type": "text/html", + "title": "", + }, + { + "href": "https://www.example.org/service/terms-and-conditions.html", + "hreflang": "en", + "rel": "restrictions", + "type": "text/html", + "title": "", + }, + { + "href": "http://www.example.org/edr/collections/the_collection_id/", + "hreflang": "en", + "rel": "collection", + "type": "collection", + "title": "Collection", + }, + { + "href": "http://www.example.org/edr/collections/the_collection_id/position", + "hreflang": "en", + "rel": "data", + "type": "position", + "title": "Position", + }, + ], + ) + id: str + title: Optional[str] + description: Optional[str] + keywords: Optional[List[str]] + extent: Extent + data_queries: Optional[DataQueries] + crs: Optional[str | List[str]] + output_formats: Optional[List[str]] + parameter_names: Optional[Dict[str, ParameterName]] + + +class InstancesModel(MyBaseModel): + links: List[Link] = Field( + ..., + example=[ + { + "href": "http://www.example.org/edr/collections/the_collection_id/instances", + "hreflang": "en", + "rel": "self", + "type": "application/json", + }, + { + "href": "http://www.example.org/edr/collections/the_collection_id/instances?f=html", + "hreflang": "en", + "rel": "alternate", + "type": "text/html", + }, + { + "href": "http://www.example.org/edr/collections/the_collection_id/instances?f=xml", + "hreflang": "en", + "rel": "alternate", + "type": "application/xml", + }, + ], + ) + instances: List[Instance] + + +# For now, the collection metadata corresponds to the first instance metadata. So they have equal classes +class Collection(Instance): + pass + + +class CollectionsModel(MyBaseModel): + links: List[Link] = Field( + ..., + example=[ + { + "href": "http://www.example.org/edr/collections", + "hreflang": "en", + "rel": "self", + "type": "application/json", + }, + { + "href": "http://www.example.org/edr/collections?f=html", + "hreflang": "en", + "rel": "alternate", + "type": "text/html", + }, + { + "href": "http://www.example.org/edr/collections?f=xml", + "hreflang": "en", + "rel": "alternate", + "type": "application/xml", + }, + ], + ) + collections: List[Collection] diff --git a/python/edr_package.ORG/edr_pydantic_classes/my_base_model.py b/python/edr_package.ORG/edr_pydantic_classes/my_base_model.py new file mode 100644 index 00000000..d6291d2d --- /dev/null +++ b/python/edr_package.ORG/edr_pydantic_classes/my_base_model.py @@ -0,0 +1,37 @@ +import orjson +from pydantic import BaseModel +from pydantic import Extra + +from datetime import datetime +from zoneinfo import ZoneInfo + + +def orjson_dumps(v, *, default): + # orjson.dumps returns bytes, to match standard json.dumps we need to decode + return orjson.dumps( + v, + default=default, + option=orjson.OPT_NON_STR_KEYS | orjson.OPT_UTC_Z | orjson.OPT_NAIVE_UTC, + ).decode() + + +def convert_datetime_to_gmt(dt: datetime) -> str: + if not dt.tzinfo: + dt = dt.replace(tzinfo=ZoneInfo("UTC")) + + return dt.strftime("%Y-%m-%dT%H:%M:%SZ") + + +class MyBaseModel(BaseModel): + class Config: + allow_population_by_field_name = True + anystr_strip_whitespace = True + extra = Extra.forbid + min_anystr_length = 1 + smart_union = True + validate_all = True + validate_assignment = True + + json_loads = orjson.loads + json_dumps = orjson_dumps + json_encoders = {datetime: convert_datetime_to_gmt} diff --git a/python/edr_package.ORG/edr_pydantic_classes/py.typed b/python/edr_package.ORG/edr_pydantic_classes/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/python/edr_package.ORG/setup.py b/python/edr_package.ORG/setup.py new file mode 100755 index 00000000..bdb32bf4 --- /dev/null +++ b/python/edr_package.ORG/setup.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import logging +import os + +import setuptools + +logging.basicConfig() +logger = logging.getLogger(__name__) +logger.setLevel(os.environ.get("LOG_LEVEL", "INFO")) + +# Package meta-data. +NAME = "edr-pydantic-classes" + +env_suffix = os.environ.get("ENVIRONMENT_SUFFIX", "") +logger.debug(f"Environment suffix: {env_suffix}") + +if env_suffix: + NAME += f"-{env_suffix}" +logger.debug(f"Package name: {NAME}") + +setuptools.setup( + name=NAME, + version="0.0.9", + description="The Pydantic models for EDR datatypes", + package_data={"edr_pydantic_classes": ["py.typed"]}, + packages=["edr_pydantic_classes"], + include_package_data=False, + license="MIT", + install_requires=["pydantic", "orjson", "covjson-pydantic"], + python_requires=">=3.8.0", +) diff --git a/python/python_fastapi_server/main.py b/python/python_fastapi_server/main.py index 50e9cc8f..4a6ba52b 100644 --- a/python/python_fastapi_server/main.py +++ b/python/python_fastapi_server/main.py @@ -72,13 +72,13 @@ async def add_hsts_header(request: Request, call_next): allow_headers=["*"], ) -app.add_middleware(BrotliMiddleware, gzip_fallback=True) - app.add_middleware(CachingMiddleware) if "EXTERNALADDRESS" in os.environ: app.add_middleware(FixSchemeMiddleware) +app.add_middleware(BrotliMiddleware, gzip_fallback=True) + @app.get("/") async def root(): diff --git a/python/python_fastapi_server/routers/cachingmiddleware.py b/python/python_fastapi_server/routers/cachingmiddleware.py index 5dd58352..57528c34 100644 --- a/python/python_fastapi_server/routers/cachingmiddleware.py +++ b/python/python_fastapi_server/routers/cachingmiddleware.py @@ -19,9 +19,9 @@ async def get_cached_response(request): key = generate_key(request) cached = await redis.get(key) if not cached: - print("Cache miss") + # print("Cache miss") return None, None, None - print("Cache hit") + # print("Cache hit", len(cached)) entrytime=int(cached[:10]) currenttime=calendar.timegm(datetime.utcnow().utctimetuple()) @@ -33,7 +33,7 @@ async def get_cached_response(request): data = cached[16+headers_len:] return age, headers, data -skip_headers=["x-process-time"] +skip_headers=["x-process-time", "age"] async def cache_response(request, headers, data, ex: int=60): key=generate_key(request) @@ -42,28 +42,28 @@ async def cache_response(request, headers, data, ex: int=60): for k in headers.keys(): if k not in skip_headers: allheaders[k]=headers[k] - else: - print("skipping header", k) allheaders_json=json.dumps(allheaders) entrytime="%10d"%calendar.timegm(datetime.utcnow().utctimetuple()) - print("ENTRY:", entrytime) - - print("Caching ", key, allheaders_json, "<><><>", type(data), data[:80]) await redis.set(key, bytes(entrytime, 'utf-8')+bytes("%06d"%len(allheaders_json), 'utf-8')+bytes(allheaders_json, 'utf-8')+data, ex=ex) def generate_key(request): - key = request['query_string'] + key = f"{request.url}?{request['query_string']}" return key class CachingMiddleware(BaseHTTPMiddleware): + shortcut = True def __init__(self, app): super().__init__(app) + if "ADAGUC_REDIS" in os.environ: + self.shortcut=False async def dispatch(self, request, call_next): + if self.shortcut: + return await call_next(request) + #Check if request is in cache, if so return that expire, headers, data = await get_cached_response(request) - print("AGE:", expire, data[:20] if data else None) if data: #Fix Age header @@ -74,22 +74,14 @@ async def dispatch(self, request, call_next): if response.status_code == 200: if "cache-control" in response.headers and response.headers['cache-control']!="no-store": - print("CachingMiddleware:", response.headers['cache-control']) age_terms = response.headers['cache-control'].split("=") if age_terms[0].lower()!="max-age": return age=int(age_terms[1]) response_body = b"" - # async for chunk in response.body_iterator: - # response_body+=chunk - async def response_chunks(): - async for chunk in response.body_iterator: - yield chunk - task = BackgroundTask(cache_response, request=request, headers=response.headers, data=response_body, ex=age) - # await cache_response(request, response.headers, response_body, age) + async for chunk in response.body_iterator: + response_body+=chunk + task = BackgroundTask(cache_response, request=request, headers=response.headers, data=response_body, ex=age) # await cache_response(request, response.headers, response_body, age) response.headers['age']="0" - # print("HDRS:",response.headers) - # return Response(content=response_body, status_code=200, headers=response.headers, media_type=response.media_type, background=task) - return StreamingResponse(content=response_chunks(), status_code=200, headers=response.headers, media_type=response.media_type, background=task) - print("NOT CACHING ", generate_key(request)) + return Response(content=response_body, status_code=200, headers=response.headers, media_type=response.media_type, background=task) return response diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index 7e8aa063..7495ec38 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -1,4 +1,4 @@ -""" +"""python/python_fastapi_server/routers/edr.py Adaguc-Server OGC EDR implementation This code uses Adaguc's OGC WMS and WCS endpoints to convert into an EDR service. @@ -16,6 +16,8 @@ from typing import Union from cachetools import TTLCache, cached +from cachetools.keys import hashkey +from CacheToolsUtils import PrefixedRedisCache from covjson_pydantic.coverage import Coverage from covjson_pydantic.domain import Domain, ValuesAxis from covjson_pydantic.observed_property import ( @@ -45,19 +47,31 @@ from edr_pydantic.variables import Variables -from fastapi import FastAPI, Query, Request +from fastapi import FastAPI, Query, Request, Response from fastapi.responses import JSONResponse +from fastapi.encoders import jsonable_encoder from fastapi.openapi.utils import get_openapi from geomet import wkt from owslib.wms import WebMapService from pydantic import AwareDatetime +from redis import from_url +from functools import partial + from .covjsonresponse import CovJSONResponse from .ogcapi_tools import call_adaguc +SHORT_CACHE_TIME=15 + logger = logging.getLogger(__name__) logger.debug("Starting EDR") +redis_url = os.environ.get("ADAGUC_REDIS", None) +if redis_url: + edr_cache = PrefixedRedisCache(from_url(os.environ.get("ADAGUC_REDIS")), "edr", SHORT_CACHE_TIME) +else: + edr_cache = TTLCache(maxsize=1024, ttl=SHORT_CACHE_TIME) + edrApiApp = FastAPI(debug=True) OWSLIB_DUMMY_URL = "http://localhost:8000" @@ -96,8 +110,6 @@ async def edr_exception_handler(_, exc: EdrException): content={"code": str(exc.code), "description": exc.description}, ) - -@cached(cache=TTLCache(maxsize=1024, ttl=60)) def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DIR"]): """ Return all possible OGCAPI EDR datasets, based on the dataset directory @@ -157,10 +169,14 @@ def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DI pass return edr_collections - +edr_collections = None def get_edr_collections(): """Returns all EDR collections""" - return init_edr_collections() + global edr_collections + if edr_collections is None: + edr_collections = init_edr_collections() + + return edr_collections def get_point_value( @@ -197,8 +213,8 @@ def get_point_value( status, response, headers = call_adaguc(url=urlrequest.encode("UTF-8")) if status == 0: - return response.getvalue() - return None + return response.getvalue(), headers + return None, None @edrApiApp.get( @@ -249,7 +265,6 @@ async def get_collection_position( custom_dims, ) if resp: - response.headers['X-Special']='OK' dat = json.loads(resp) return covjson_from_resp(dat, edr_collections[collection_name]["vertical_name"]) @@ -262,7 +277,6 @@ async def get_collection_position( } -@cached(cache=TTLCache(maxsize=1024, ttl=60)) def get_collectioninfo_for_id( edr_collection: str, instance: str = None, @@ -300,7 +314,6 @@ def get_collectioninfo_for_id( bbox = get_extent(edr_collectioninfo) if bbox is None: return None - print("bbox:", bbox, type(bbox)) crs = 'GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]' spatial = Spatial(bbox=bbox, crs=crs) (interval, time_values) = get_times_for_collection( @@ -500,8 +513,8 @@ def get_times_for_collection( else: layer = wms[list(wms)[0]] - if "time" in layer.dimensions: - time_dim = layer.dimensions["time"] + if "time" in layer["dimensions"]: + time_dim = layer["dimensions"]["time"] if "/" in time_dim["values"][0]: terms = time_dim["values"][0].split("/") interval = [ @@ -540,7 +553,7 @@ def get_custom_dims_for_collection(edr_collectioninfo: dict, parameter: str = No else: # default to first layer layer = wms[list(wms)[0]] - for dim_name in layer.dimensions: + for dim_name in layer["dimensions"]: # Not needed for non custom dims: if dim_name not in [ "reference_time", @@ -551,7 +564,7 @@ def get_custom_dims_for_collection(edr_collectioninfo: dict, parameter: str = No custom_dim = { "id": dim_name, "interval": [], - "values": layer.dimensions[dim_name]["values"], + "values": layer["dimensions"][dim_name]["values"], "reference": f"custom_{dim_name}", } custom.append(custom_dim) @@ -568,14 +581,14 @@ def get_vertical_dim_for_collection(edr_collectioninfo: dict, parameter: str = N else: layer = wms[list(wms)[0]] - for dim_name in layer.dimensions: + for dim_name in layer["dimensions"]: if dim_name in ["elevation"] or ( "vertical_name" in edr_collectioninfo and dim_name == edr_collectioninfo["vertical_name"] ): vertical_dim = { "interval": [], - "values": layer.dimensions[dim_name]["values"], + "values": layer["dimensions"][dim_name]["values"], "vrs": "customvrs", } return vertical_dim @@ -585,7 +598,7 @@ def get_vertical_dim_for_collection(edr_collectioninfo: dict, parameter: str = N @edrApiApp.get( "/collections", response_model=Collections, response_model_exclude_none=True ) -async def rest_get_edr_collections(request: Request): +async def rest_get_edr_collections(request: Request, response: Response): """ GET /collections, returns a list of available collections """ @@ -603,6 +616,7 @@ async def rest_get_edr_collections(request: Request): else: logger.warning("Unable to fetch WMS GetCapabilties for %s", edr_coll) collections_data = Collections(links=links, collections=collections) + response.headers["cache-control"] = f"max-age={SHORT_CACHE_TIME}" return collections_data @@ -611,15 +625,16 @@ async def rest_get_edr_collections(request: Request): response_model=Collection, response_model_exclude_none=True, ) -async def rest_get_edr_collection_by_id(collection_name: str): +async def rest_get_edr_collection_by_id(collection_name: str, response: Response): """ GET Returns collection information for given collection id """ collection = get_collectioninfo_for_id(collection_name) + response.headers['cache-control']=f"max-age={SHORT_CACHE_TIME}" return collection -@cached(cache=TTLCache(maxsize=1024, ttl=60)) +@cached(cache=edr_cache, key=partial(hashkey, "get_capabilities")) def get_capabilities(collname): """ Get the collectioninfo from the WMS GetCapabilities @@ -642,7 +657,11 @@ def get_capabilities(collname): else: logger.info("callADAGUC by service %s", dataset) wms = WebMapService(dataset["service"], version="1.3.0") - return wms.contents + + layers={} + for layername,layerinfo in wms.contents.items(): + layers[layername]={ "name": layername, "dimensions":{**layerinfo.dimensions}, "boundingBoxWGS84": layerinfo.boundingBoxWGS84} + return layers def get_ref_times_for_coll(edr_collectioninfo: dict, layer: str) -> list[str]: @@ -666,7 +685,7 @@ def get_extent(edr_collectioninfo: dict): """ contents = get_capabilities(edr_collectioninfo["name"]) if len(contents): - bbox = contents[next(iter(contents))].boundingBoxWGS84 + bbox = contents[next(iter(contents))]["boundingBoxWGS84"] return [[bbox[0], bbox[1]], [bbox[2], bbox[3]]] return None @@ -677,7 +696,7 @@ def get_extent(edr_collectioninfo: dict): response_model=Instances, response_model_exclude_none=True, ) -async def rest_get_edr_inst_for_coll(collection_name: str, request: Request): +async def rest_get_edr_inst_for_coll(collection_name: str, request: Request, response: Response): """ GET: Returns all available instances for the collection """ @@ -703,6 +722,7 @@ async def rest_get_edr_inst_for_coll(collection_name: str, request: Request): instances.append(instance_info) instances_data = Instances(instances=instances, links=links) + response.headers['cache-control'] = f"max-age={SHORT_CACHE_TIME}" return instances_data @@ -711,11 +731,12 @@ async def rest_get_edr_inst_for_coll(collection_name: str, request: Request): response_model=Collection, response_model_exclude_none=True, ) -async def rest_get_collection_info(collection_name: str, instance): +async def rest_get_collection_info(collection_name: str, instance, response: Response): """ GET "/collections/{collection_name}/instances/{instance}" """ coll = get_collectioninfo_for_id(collection_name, instance) + response.headers['cache-control'] = f"max-age={SHORT_CACHE_TIME}" return coll diff --git a/python/python_fastapi_server/routers/ogcapi_tools.py b/python/python_fastapi_server/routers/ogcapi_tools.py index 22fabc04..56f59814 100644 --- a/python/python_fastapi_server/routers/ogcapi_tools.py +++ b/python/python_fastapi_server/routers/ogcapi_tools.py @@ -120,11 +120,10 @@ def call_adaguc(url): @cached(cache=cache) -def get_capabilities(collname): +def get_capabilities(coll): """ Get the collectioninfo from the WMS GetCapabilities """ - coll = generate_collections().get(collname) if "dataset" in coll: logger.info("callADAGUC by dataset") dataset = coll["dataset"] @@ -175,11 +174,11 @@ def get_dimensions(layer, skip_dims=None): @cached(cache=cache) -def get_parameters(collname): +def get_parameters(coll): """ get_parameters """ - contents = get_capabilities(collname) + contents = get_capabilities(coll) layers = [] for layer in contents: dims = get_dimensions(contents[layer], ["time"]) diff --git a/requirements.in b/requirements.in index 5151a16f..9014f1a0 100644 --- a/requirements.in +++ b/requirements.in @@ -26,4 +26,5 @@ covjson-pydantic~=0.2.0 geomet~=1.0 edr-pydantic~=0.2.0 redis~=5.0 -aioredis~=2.0 \ No newline at end of file +aioredis~=2.0 +CacheToolsUtils~=8.5 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c78156fa..4ef8e1fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,6 +25,8 @@ brotli-asgi==1.4.0 # via -r requirements.in cachetools==5.3.2 # via -r requirements.in +CacheToolsUtils==8.5 + # via -r requirements.in certifi==2023.11.17 # via # httpcore From 7be69056d8bdc39dff47eb24b592ae717846e017 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Fri, 9 Feb 2024 11:30:59 +0100 Subject: [PATCH 06/44] Fixed tests failing becaue of cache --- python/python_fastapi_server/main.py | 11 +-- .../routers/cachingmiddleware.py | 45 +++++++---- python/python_fastapi_server/routers/edr.py | 78 ++++++++++++++----- .../python_fastapi_server/routers/ogcapi.py | 10 ++- .../routers/ogcapi_tools.py | 15 ++-- .../python_fastapi_server/test_ogc_api_edr.py | 24 +++--- .../test_ogc_api_features.py | 39 +++++----- 7 files changed, 136 insertions(+), 86 deletions(-) diff --git a/python/python_fastapi_server/main.py b/python/python_fastapi_server/main.py index 4a6ba52b..5c266bec 100644 --- a/python/python_fastapi_server/main.py +++ b/python/python_fastapi_server/main.py @@ -22,6 +22,7 @@ from routers.opendap import opendapRouter from routers.wmswcs import testadaguc, wmsWcsRouter from routers.cachingmiddleware import CachingMiddleware +# from routers.cachingmiddleware2 import CachingMiddleware2 logger = logging.getLogger(__name__) @@ -33,13 +34,6 @@ app.add_middleware(AccessLoggerMiddleware, format=access_log_format) logging.getLogger("access").propagate = False -@app.middleware("http") -async def add_process_time_header(request: Request, call_next): - start_time = time.time() - response = await call_next(request) - process_time = time.time() - start_time - response.headers["X-Process-Time"] = str(process_time) - return response @app.middleware("http") async def add_process_time_header(request: Request, call_next): @@ -72,7 +66,8 @@ async def add_hsts_header(request: Request, call_next): allow_headers=["*"], ) -app.add_middleware(CachingMiddleware) +if "ADAGUC_REDIS" in os.environ: + app.add_middleware(CachingMiddleware) if "EXTERNALADDRESS" in os.environ: app.add_middleware(FixSchemeMiddleware) diff --git a/python/python_fastapi_server/routers/cachingmiddleware.py b/python/python_fastapi_server/routers/cachingmiddleware.py index 57528c34..58e02574 100644 --- a/python/python_fastapi_server/routers/cachingmiddleware.py +++ b/python/python_fastapi_server/routers/cachingmiddleware.py @@ -1,23 +1,23 @@ import os from urllib.parse import urlsplit -from starlette.concurrency import iterate_in_threadpool +# from starlette.middleware.base import BaseHTTPMiddleware from starlette.middleware.base import BaseHTTPMiddleware -from starlette.responses import StreamingResponse, Response -from starlette.background import BackgroundTask +from starlette.responses import Response +from fastapi import BackgroundTasks import calendar from datetime import datetime, timedelta import time -from redis import asyncio +import redis.asyncio as redis import json -ADAGUC_REDIS=os.environ.get('ADAGUC_REDIS', "redis://localhost:6379") -redis = asyncio.from_url(ADAGUC_REDIS) +ADAGUC_REDIS=os.environ.get('ADAGUC_REDIS') -async def get_cached_response(request): +async def get_cached_response(redis_client, request): key = generate_key(request) - cached = await redis.get(key) + cached = await redis_client.get(key) + await redis_client.aclose() if not cached: # print("Cache miss") return None, None, None @@ -35,7 +35,7 @@ async def get_cached_response(request): skip_headers=["x-process-time", "age"] -async def cache_response(request, headers, data, ex: int=60): +async def cache_response(redis_client, request, headers, data, ex: int=60): key=generate_key(request) allheaders={} @@ -45,10 +45,12 @@ async def cache_response(request, headers, data, ex: int=60): allheaders_json=json.dumps(allheaders) entrytime="%10d"%calendar.timegm(datetime.utcnow().utctimetuple()) - await redis.set(key, bytes(entrytime, 'utf-8')+bytes("%06d"%len(allheaders_json), 'utf-8')+bytes(allheaders_json, 'utf-8')+data, ex=ex) + await redis_client.set(key, bytes(entrytime, 'utf-8')+bytes("%06d"%len(allheaders_json), 'utf-8')+bytes(allheaders_json, 'utf-8')+data, ex=ex) + await redis_client.aclose() def generate_key(request): - key = f"{request.url}?{request['query_string']}" + key = f"{request.url.path}?{request['query_string']}" + # print(f"generate_key({key})") return key class CachingMiddleware(BaseHTTPMiddleware): @@ -57,13 +59,16 @@ def __init__(self, app): super().__init__(app) if "ADAGUC_REDIS" in os.environ: self.shortcut=False + self.redis = None async def dispatch(self, request, call_next): + if self.redis is None: + self.redis = redis.from_url(ADAGUC_REDIS) if self.shortcut: return await call_next(request) #Check if request is in cache, if so return that - expire, headers, data = await get_cached_response(request) + expire, headers, data = None, None, None #await get_cached_response(self.redis, request) if data: #Fix Age header @@ -74,14 +79,20 @@ async def dispatch(self, request, call_next): if response.status_code == 200: if "cache-control" in response.headers and response.headers['cache-control']!="no-store": - age_terms = response.headers['cache-control'].split("=") - if age_terms[0].lower()!="max-age": + cache_control_terms = response.headers['cache-control'].split(",") + ttl = None + for term in cache_control_terms: + age_terms = term.split("=") + if age_terms[0].lower()=="max-age": + ttl=int(age_terms[1]) + break + if ttl is None: return - age=int(age_terms[1]) response_body = b"" async for chunk in response.body_iterator: response_body+=chunk - task = BackgroundTask(cache_response, request=request, headers=response.headers, data=response_body, ex=age) # await cache_response(request, response.headers, response_body, age) + tasks = BackgroundTasks() + tasks.add_task(cache_response, redis_client=self.redis, request=request, headers=response.headers, data=response_body, ex=ttl) response.headers['age']="0" - return Response(content=response_body, status_code=200, headers=response.headers, media_type=response.media_type, background=task) + return Response(content=response_body, status_code=200, headers=response.headers, media_type=response.media_type, background=tasks) return response diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index 7495ec38..18bb65fc 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -12,7 +12,7 @@ import logging import os import re -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta from typing import Union from cachetools import TTLCache, cached @@ -61,7 +61,7 @@ from .covjsonresponse import CovJSONResponse from .ogcapi_tools import call_adaguc -SHORT_CACHE_TIME=15 +SHORT_CACHE_TIME=60 logger = logging.getLogger(__name__) logger.debug("Starting EDR") @@ -266,6 +266,10 @@ async def get_collection_position( ) if resp: dat = json.loads(resp) + ttl = get_ttl_from_adaguc_call(headers) + if ttl is not None: + expires = (datetime.utcnow()+timedelta(seconds=ttl)).timestamp() + response.headers["cache-control"]=generate_max_age(expires) return covjson_from_resp(dat, edr_collections[collection_name]["vertical_name"]) raise EdrException(code=400, description="No data") @@ -280,7 +284,7 @@ async def get_collection_position( def get_collectioninfo_for_id( edr_collection: str, instance: str = None, -) -> Collection: +) -> tuple[Collection, datetime]: """ Returns collection information for a given collection id and or instance Is used to obtain metadata from the dataset configuration and WMS GetCapabilities document. @@ -407,7 +411,9 @@ def get_collectioninfo_for_id( crs=crs, output_formats=output_formats, ) - return collection + + get_cap = get_capabilities(edr_collection) + return collection, get_cap["expires"] def get_params_for_collection(edr_collection: str) -> dict[str, Parameter]: @@ -507,7 +513,7 @@ def get_times_for_collection( It does this for given parameter. When the parameter is not given it will do it for the first Layer in the GetCapabilities document. """ logger.info("get_times_for_dataset(%s,%s)", edr_collectioninfo["name"], parameter) - wms = get_capabilities(edr_collectioninfo["name"]) + wms = get_capabilities(edr_collectioninfo["name"])["layers"] if parameter and parameter in wms: layer = wms[parameter] else: @@ -546,7 +552,7 @@ def get_custom_dims_for_collection(edr_collectioninfo: dict, parameter: str = No """ Return the dimensions other then elevation or time from the WMS GetCapabilities document. """ - wms = get_capabilities(edr_collectioninfo["name"]) + wms = get_capabilities(edr_collectioninfo["name"])["layers"] custom = [] if parameter and parameter in list(wms): layer = wms[parameter] @@ -575,7 +581,7 @@ def get_vertical_dim_for_collection(edr_collectioninfo: dict, parameter: str = N """ Return the verticel dimension the WMS GetCapabilities document. """ - wms = get_capabilities(edr_collectioninfo["name"]) + wms = get_capabilities(edr_collectioninfo["name"])["layers"] if parameter and parameter in list(wms): layer = wms[parameter] else: @@ -610,13 +616,14 @@ async def rest_get_edr_collections(request: Request, response: Response): collections: list[Collection] = [] edr_collections = get_edr_collections() for edr_coll in edr_collections: - coll = get_collectioninfo_for_id(edr_coll) + coll, expires = get_collectioninfo_for_id(edr_coll) if coll: collections.append(coll) else: logger.warning("Unable to fetch WMS GetCapabilties for %s", edr_coll) collections_data = Collections(links=links, collections=collections) - response.headers["cache-control"] = f"max-age={SHORT_CACHE_TIME}" + if expires is not None: + response.headers["cache-control"] = generate_max_age(expires) return collections_data @@ -629,10 +636,23 @@ async def rest_get_edr_collection_by_id(collection_name: str, response: Response """ GET Returns collection information for given collection id """ - collection = get_collectioninfo_for_id(collection_name) - response.headers['cache-control']=f"max-age={SHORT_CACHE_TIME}" + collection,expires = get_collectioninfo_for_id(collection_name) + response.headers['cache-control']=generate_max_age(expires) return collection +def get_ttl_from_adaguc_call(headers): + ttl = None + for hdr in headers: + hdr_terms=hdr.split(":") + if hdr_terms[0].lower() == "cache-control": + for cache_control_terms in hdr_terms[1].split(","): + terms = cache_control_terms.split("=") + if terms[0]=='max-age': + ttl = int(terms[1]) + break + if ttl is not None: + break + return ttl @cached(cache=edr_cache, key=partial(hashkey, "get_capabilities")) def get_capabilities(collname): @@ -647,6 +667,8 @@ def get_capabilities(collname): f"dataset={dataset}&service=wms&version=1.3.0&request=getcapabilities" ) status, response, headers = call_adaguc(url=urlrequest.encode("UTF-8")) + ttl = get_ttl_from_adaguc_call(headers) + now=datetime.utcnow() logger.info("status: %d", status) if status == 0: xml = response.getvalue() @@ -657,11 +679,18 @@ def get_capabilities(collname): else: logger.info("callADAGUC by service %s", dataset) wms = WebMapService(dataset["service"], version="1.3.0") + now=datetime.utcnow() + ttl=None layers={} for layername,layerinfo in wms.contents.items(): layers[layername]={ "name": layername, "dimensions":{**layerinfo.dimensions}, "boundingBoxWGS84": layerinfo.boundingBoxWGS84} - return layers + if ttl is not None: + expires = (now + timedelta(seconds=ttl)).timestamp() + else: + expires=None + + return {"layers": layers, "expires": expires} def get_ref_times_for_coll(edr_collectioninfo: dict, layer: str) -> list[str]: @@ -683,9 +712,13 @@ def get_extent(edr_collectioninfo: dict): """ Get the boundingbox extent from the WMS GetCapabilities """ - contents = get_capabilities(edr_collectioninfo["name"]) + contents = get_capabilities(edr_collectioninfo["name"])["layers"] + first_layer = edr_collectioninfo["parameters"][0]["name"] if len(contents): - bbox = contents[next(iter(contents))]["boundingBoxWGS84"] + if first_layer in contents: + bbox = contents[first_layer]["boundingBoxWGS84"] + else: + bbox = contents[next(iter(contents))]["boundingBoxWGS84"] return [[bbox[0], bbox[1]], [bbox[2], bbox[3]]] return None @@ -713,16 +746,17 @@ async def rest_get_edr_inst_for_coll(collection_name: str, request: Request, res ) links: list(Link) = [] links.append(Link(href=instances_url, rel="collection")) + min_expires=(datetime.utcnow()+timedelta(days=1)).timestamp() # 24 hours for instance in list(ref_times): instance_links: list(Link) = [] instance_link = Link(href=f"{instances_url}/{instance}", rel="collection") instance_links.append(instance_link) - instance_info = get_collectioninfo_for_id(collection_name, instance) - + instance_info, expires = get_collectioninfo_for_id(collection_name, instance) + min_expires=min(min_expires, expires) instances.append(instance_info) instances_data = Instances(instances=instances, links=links) - response.headers['cache-control'] = f"max-age={SHORT_CACHE_TIME}" + response.headers['cache-control'] = generate_max_age(min_expires) return instances_data @@ -735,10 +769,16 @@ async def rest_get_collection_info(collection_name: str, instance, response: Res """ GET "/collections/{collection_name}/instances/{instance}" """ - coll = get_collectioninfo_for_id(collection_name, instance) - response.headers['cache-control'] = f"max-age={SHORT_CACHE_TIME}" + coll, expires = get_collectioninfo_for_id(collection_name, instance) + response.headers['cache-control'] = generate_max_age(expires) return coll +def generate_max_age(expires): + rest_age = int((datetime.fromtimestamp(expires)-datetime.utcnow()).total_seconds()) + if rest_age>0: + return f"max-age={rest_age}" + return f"max-age=0" + @edrApiApp.get("/", response_model=LandingPageModel, response_model_exclude_none=True) async def rest_get_edr_landing_page(request: Request): diff --git a/python/python_fastapi_server/routers/ogcapi.py b/python/python_fastapi_server/routers/ogcapi.py index c2175751..102071fc 100644 --- a/python/python_fastapi_server/routers/ogcapi.py +++ b/python/python_fastapi_server/routers/ogcapi.py @@ -75,7 +75,7 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE @ogcApiApp.get("", response_model=LandingPage, response_model_exclude_none=True) -async def handle_ogc_api_root(req: Request, f: str = "json"): +async def handle_ogc_api_root(req: Request, response: Response, f: str = "json"): links: List[Link] = [] links.append( Link( @@ -122,6 +122,9 @@ async def handle_ogc_api_root(req: Request, f: str = "json"): landing_page = LandingPage(title="ogcapi", description="ADAGUC OGCAPI-Features server", links=links) + + response.headers['cache-control']="max-age=18" + if request_type(f) == "HTML": return templates.TemplateResponse("landingpage.html", { "request": req, @@ -200,7 +203,7 @@ def request_type(wanted_format: str) -> str: @ogcApiApp.get("/collections", response_model=Collections, response_model_exclude_none=True) -async def get_collections(req: Request, f: str = "json"): +async def get_collections(req: Request, response: Response, f: str = "json"): collections: List[Collection] = [] parsed_collections = generate_collections() for parsed_collection in parsed_collections.values(): @@ -227,6 +230,9 @@ async def get_collections(req: Request, f: str = "json"): )) links = get_collections_links(req.url_for("get_collections")) + + response.headers["cache-control"]="max-age=18" + if request_type(f) == "HTML": collections_list = [c.dict() for c in collections] return templates.TemplateResponse("collections.html", { diff --git a/python/python_fastapi_server/routers/ogcapi_tools.py b/python/python_fastapi_server/routers/ogcapi_tools.py index 56f59814..03abf359 100644 --- a/python/python_fastapi_server/routers/ogcapi_tools.py +++ b/python/python_fastapi_server/routers/ogcapi_tools.py @@ -104,7 +104,7 @@ def call_adaguc(url): # Run adaguc-server # pylint: disable=unused-variable status, data, headers = adaguc_instance.runADAGUCServer( - url, env=adagucenv, showLogOnError=True) + url, env=adagucenv, showLogOnError=False) # Obtain logfile logfile = adaguc_instance.getLogFile() @@ -120,10 +120,11 @@ def call_adaguc(url): @cached(cache=cache) -def get_capabilities(coll): +def get_capabilities(collname): """ Get the collectioninfo from the WMS GetCapabilities """ + coll = generate_collections().get(collname) if "dataset" in coll: logger.info("callADAGUC by dataset") dataset = coll["dataset"] @@ -131,9 +132,9 @@ def get_capabilities(coll): f"dataset={dataset}&service=wms&version=1.3.0&request=getcapabilities" ) status, response, headers = call_adaguc(url=urlrequest.encode("UTF-8")) - for hdr in headers.keys(): - if hdr.lower()=="cache-control": - logger.info("%s: %s", hdr, headers[hdr]) + for hdr in headers: + if hdr.lower().startswith("cache-control"): + logger.info("%s", hdr) if status == 0: xml = response.getvalue() wms = WebMapService(coll["service"], xml=xml, version="1.3.0") @@ -174,11 +175,11 @@ def get_dimensions(layer, skip_dims=None): @cached(cache=cache) -def get_parameters(coll): +def get_parameters(collname): """ get_parameters """ - contents = get_capabilities(coll) + contents = get_capabilities(collname) layers = [] for layer in contents: dims = get_dimensions(contents[layer], ["time"]) diff --git a/python/python_fastapi_server/test_ogc_api_edr.py b/python/python_fastapi_server/test_ogc_api_edr.py index ceb48953..6370d627 100644 --- a/python/python_fastapi_server/test_ogc_api_edr.py +++ b/python/python_fastapi_server/test_ogc_api_edr.py @@ -3,8 +3,11 @@ import os import pytest +import pytest_asyncio from adaguc.AdagucTestTools import AdagucTestTools from fastapi.testclient import TestClient +from httpx import AsyncClient +import asyncio from main import app @@ -18,7 +21,6 @@ def set_environ(): def setup_test_data(): - print("About to ingest data") AdagucTestTools().cleanTempDir() for service in ["netcdf_5d.xml", "dataset_a.xml"]: _status, _data, _headers = AdagucTestTools().runADAGUCServer( @@ -32,7 +34,6 @@ def setup_test_data(): showLog=True, ) - @pytest.fixture(name="client") def fixture_client() -> TestClient: # Initialize adaguc-server @@ -40,17 +41,16 @@ def fixture_client() -> TestClient: setup_test_data() yield TestClient(app) - -def test_root(client: TestClient): +@pytest.mark.asyncio +async def test_root(client): resp = client.get("/edr/") root_info = resp.json() print("resp:", resp, json.dumps(root_info, indent=2)) - print() assert root_info["description"] == "EDR service for ADAGUC datasets" assert len(root_info["links"]) >= 4 - -def test_collections(client: TestClient): +@pytest.mark.asyncio +async def test_collections(client): resp = client.get("/edr/collections") colls = resp.json() assert len(colls["collections"]) == 1 @@ -74,9 +74,7 @@ def test_collections(client: TestClient): assert "position" in coll_5d["data_queries"] - -def test_coll_5d_position(client: TestClient): - resp = client.get( - "/edr/collections/data_5d/position?coords=POINT(5.2 50.0)¶meter-name=data" - ) - print(resp.json()) +@pytest.mark.asyncio +async def test_coll_5d_position(client): + resp = client.get("/edr/collections/data_5d/position?coords=POINT(5.2 50.0)¶meter-name=data") + print(resp.json()) \ No newline at end of file diff --git a/python/python_fastapi_server/test_ogc_api_features.py b/python/python_fastapi_server/test_ogc_api_features.py index 385b9878..49f0d9ce 100644 --- a/python/python_fastapi_server/test_ogc_api_features.py +++ b/python/python_fastapi_server/test_ogc_api_features.py @@ -1,11 +1,13 @@ import json import logging import os +from httpx import AsyncClient import pytest from adaguc.AdagucTestTools import AdagucTestTools from fastapi.testclient import TestClient +import pytest_asyncio from main import app logger = logging.getLogger(__name__) @@ -18,7 +20,6 @@ def set_environ(): def setup_test_data(): - print("About to ingest data") AdagucTestTools().cleanTempDir() for service in ["netcdf_5d.xml", "dataset_a.xml"]: _status, _data, _headers = AdagucTestTools().runADAGUCServer( @@ -33,27 +34,25 @@ def setup_test_data(): ) -@pytest.fixture(name="client") -def fixture_client() -> TestClient: - # Initialize adaguc-server +@pytest_asyncio.fixture(name="clientdata") +async def clientdata(): set_environ() setup_test_data() - yield TestClient(app) +@pytest.mark.asyncio() +async def test_root(clientdata): + async with AsyncClient(app=app, base_url="http://test") as client: + resp = await client.get( + "/adaguc-server?dataset=netcdf_5d&request=getcapabilities&service=wms&version=1.3.0" + ) -def test_root(client: TestClient): - resp = client.get( - "/adaguc-server?dataset=netcdf_5d&request=getcapabilities&service=wms&version=1.3.0" - ) - print("getcap:", resp.text) - - resp = client.get("/ogcapi/") - print("resp:", resp, resp.json()) - assert resp.json()["description"] == "ADAGUC OGCAPI-Features server" - + resp = await client.get("/ogcapi/") + assert resp.json()["description"] == "ADAGUC OGCAPI-Features server" -def test_collections(client: TestClient): - resp = client.get("/ogcapi/collections") - colls = resp.json() - print(json.dumps(colls["collections"][1], indent=2)) - assert len(colls["collections"]) == 2 +@pytest.mark.asyncio() +async def test_collections(clientdata): + async with AsyncClient(app=app, base_url="http://test") as client: + resp = await client.get("/ogcapi/collections") + colls = resp.json() + print(json.dumps(colls["collections"][1], indent=2)) + assert len(colls["collections"]) == 2 From 265ee59b175f57983c42a8ee973f3a104f1f1368 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Fri, 9 Feb 2024 12:10:14 +0100 Subject: [PATCH 07/44] Updated requirements; add pytest-asyncio --- requirements-dev.in | 1 + requirements-dev.txt | 4 ++++ requirements.in | 1 - requirements.txt | 21 ++++++++++++--------- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/requirements-dev.in b/requirements-dev.in index 044ffc26..78fba8d1 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -8,3 +8,4 @@ black~=23.7 isort~=5.12 pylint~=2.17 pytest~=7.4 +pytest-asyncio~=0.23 \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index ba2e4a19..f4b8fe36 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -46,6 +46,10 @@ pluggy==1.4.0 pylint==2.17.7 # via -r requirements-dev.in pytest==7.4.4 + # via + # -r requirements-dev.in + # pytest-asyncio +pytest-asyncio==0.23.4 # via -r requirements-dev.in tomli==2.0.1 # via diff --git a/requirements.in b/requirements.in index 9014f1a0..2d304dc4 100644 --- a/requirements.in +++ b/requirements.in @@ -26,5 +26,4 @@ covjson-pydantic~=0.2.0 geomet~=1.0 edr-pydantic~=0.2.0 redis~=5.0 -aioredis~=2.0 CacheToolsUtils~=8.5 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4ef8e1fa..24b09120 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,8 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --no-emit-index-url +# pip-compile --no-emit-index-url requirements.in # -aioredis==2.0.1 annotated-types==0.6.0 # via pydantic anyio==3.7.1 @@ -17,6 +16,7 @@ asgi-logger==0.1.0 asgiref==3.7.2 # via asgi-logger async-timeout==4.0.3 + # via redis brotli==1.1.0 # via # -r requirements.in @@ -24,10 +24,12 @@ brotli==1.1.0 brotli-asgi==1.4.0 # via -r requirements.in cachetools==5.3.2 + # via + # -r requirements.in + # cachetoolsutils +cachetoolsutils==8.5 # via -r requirements.in -CacheToolsUtils==8.5 - # via -r requirements.in -certifi==2023.11.17 +certifi==2024.2.2 # via # httpcore # httpx @@ -80,11 +82,11 @@ lxml==4.9.4 # via # -r requirements.in # owslib -markupsafe==2.1.4 +markupsafe==2.1.5 # via jinja2 netcdf4==1.6.5 # via -r requirements.in -numpy==1.26.3 +numpy==1.26.4 # via # cftime # netcdf4 @@ -94,14 +96,14 @@ packaging==23.2 # via gunicorn pillow==10.2.0 # via -r requirements.in -pydantic==2.6.0 +pydantic==2.6.1 # via # -r requirements.in # covjson-pydantic # edr-pydantic # fastapi # geojson-pydantic -pydantic-core==2.16.1 +pydantic-core==2.16.2 # via pydantic python-dateutil==2.8.2 # via owslib @@ -112,6 +114,7 @@ pytz==2023.4 pyyaml==6.0.1 # via owslib redis==5.0.1 + # via -r requirements.in requests==2.31.0 # via owslib six==1.16.0 From f1636fd5eb5d26a98abe532c0fd40a2c7b20ad69 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Fri, 9 Feb 2024 12:52:08 +0100 Subject: [PATCH 08/44] Set version to 2.16.0 --- Dockerfile | 4 ++-- adagucserverEC/Definitions.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7775e4f6..dd69aebc 100755 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ USER root LABEL maintainer="adaguc@knmi.nl" # Version should be same as in Definitions.h -LABEL version="2.15.1" +LABEL version="2.16.0" # Try to update image packages RUN apt-get -q -y update \ @@ -134,7 +134,7 @@ RUN bash -c "python3 /adaguc/adaguc-server-master/python/examples/runautowms/run WORKDIR /adaguc/adaguc-server-master # This checks if the test stage has ran without issues. -COPY --from=test /adaguc/adaguc-server-master/testsdone.txt /adaguc/adaguc-server-master/testsdone.txt +COPY --from=test /adaguc/adaguc-server-master/testsdone.txt /adaguc/adaguc-server-master/testsdone.txt USER adaguc diff --git a/adagucserverEC/Definitions.h b/adagucserverEC/Definitions.h index 5457611b..a5c706f3 100755 --- a/adagucserverEC/Definitions.h +++ b/adagucserverEC/Definitions.h @@ -28,7 +28,7 @@ #ifndef Definitions_H #define Definitions_H -#define ADAGUCSERVER_VERSION "2.15.1" // Please also update in the Dockerfile to the same version +#define ADAGUCSERVER_VERSION "2.16.0" // Please also update in the Dockerfile to the same version // CConfigReaderLayerType #define CConfigReaderLayerTypeUnknown 0 From 63e74976a128b9e7764d09c9201cf2b3edffb6b8 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Sun, 11 Feb 2024 17:29:17 +0100 Subject: [PATCH 09/44] Fixed incorrect return value --- python/python_fastapi_server/routers/edr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index 18bb65fc..a5a3007b 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -317,7 +317,7 @@ def get_collectioninfo_for_id( bbox = get_extent(edr_collectioninfo) if bbox is None: - return None + return None, None crs = 'GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]' spatial = Spatial(bbox=bbox, crs=crs) (interval, time_values) = get_times_for_collection( @@ -718,6 +718,7 @@ def get_extent(edr_collectioninfo: dict): if first_layer in contents: bbox = contents[first_layer]["boundingBoxWGS84"] else: + #Fallback to first layer in getcapabilities bbox = contents[next(iter(contents))]["boundingBoxWGS84"] return [[bbox[0], bbox[1]], [bbox[2], bbox[3]]] From 42a1a18b21fefd97c85df1b1a9bb4c6d950cd162 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Mon, 12 Feb 2024 08:49:47 +0100 Subject: [PATCH 10/44] EDR fix --- python/python_fastapi_server/routers/edr.py | 23 ++++++++++++--------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index a5a3007b..e041f35d 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -614,16 +614,18 @@ async def rest_get_edr_collections(request: Request, response: Response): links.append(self_link) collections: list[Collection] = [] + min_expires=None edr_collections = get_edr_collections() for edr_coll in edr_collections: coll, expires = get_collectioninfo_for_id(edr_coll) if coll: collections.append(coll) + min_expires=expires if min_expires is None else min(min_expires, expires) else: - logger.warning("Unable to fetch WMS GetCapabilties for %s", edr_coll) + logger.warning("Unable to fetch WMS GetCapabilities for %s", edr_coll) collections_data = Collections(links=links, collections=collections) - if expires is not None: - response.headers["cache-control"] = generate_max_age(expires) + if min_expires is not None: + response.headers["cache-control"] = generate_max_age(min_expires) return collections_data @@ -738,26 +740,27 @@ async def rest_get_edr_inst_for_coll(collection_name: str, request: Request, res get_base_url(request) + f"/edr/collections/{collection_name}/instances" ) - instances: list(Instance) = [] + instances: list[Instance] = [] edr_collections = get_edr_collections() ref_times = get_ref_times_for_coll( edr_collections[collection_name], edr_collections[collection_name]["parameters"][0]["name"], ) - links: list(Link) = [] + links: list[Link] = [] links.append(Link(href=instances_url, rel="collection")) - min_expires=(datetime.utcnow()+timedelta(days=1)).timestamp() # 24 hours + min_expires=None for instance in list(ref_times): - instance_links: list(Link) = [] + instance_links: list[Link] = [] instance_link = Link(href=f"{instances_url}/{instance}", rel="collection") instance_links.append(instance_link) instance_info, expires = get_collectioninfo_for_id(collection_name, instance) - min_expires=min(min_expires, expires) + min_expires=expires if min_expires is None else min(min_expires, expires) instances.append(instance_info) instances_data = Instances(instances=instances, links=links) - response.headers['cache-control'] = generate_max_age(min_expires) + if min_expires is not None: + response.headers['cache-control'] = generate_max_age(min_expires) return instances_data @@ -986,7 +989,7 @@ def covjson_from_resp(dats, vertical_name): else: values.append(value) - parameters: dict(str, CovJsonParameter) = {} + parameters: dict[str, CovJsonParameter] = {} ranges = {} unit = CovJsonUnit(symbol=dat["units"]) From ad96f73b3abf4c95c97f28187673ed43b308cd30 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Mon, 12 Feb 2024 08:50:04 +0100 Subject: [PATCH 11/44] Version bump --- Dockerfile | 2 +- adagucserverEC/Definitions.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index dd69aebc..88ebe2fa 100755 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ USER root LABEL maintainer="adaguc@knmi.nl" # Version should be same as in Definitions.h -LABEL version="2.16.0" +LABEL version="2.16.1" # Try to update image packages RUN apt-get -q -y update \ diff --git a/adagucserverEC/Definitions.h b/adagucserverEC/Definitions.h index a5c706f3..21f7c624 100755 --- a/adagucserverEC/Definitions.h +++ b/adagucserverEC/Definitions.h @@ -28,7 +28,7 @@ #ifndef Definitions_H #define Definitions_H -#define ADAGUCSERVER_VERSION "2.16.0" // Please also update in the Dockerfile to the same version +#define ADAGUCSERVER_VERSION "2.16.1" // Please also update in the Dockerfile to the same version // CConfigReaderLayerType #define CConfigReaderLayerTypeUnknown 0 From efef4d0dd76ffe07e9a4c654f096d3edfdbc7fe7 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Mon, 12 Feb 2024 08:58:02 +0100 Subject: [PATCH 12/44] Another EDR fix --- python/python_fastapi_server/routers/edr.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index e041f35d..ed82c4e9 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -620,7 +620,8 @@ async def rest_get_edr_collections(request: Request, response: Response): coll, expires = get_collectioninfo_for_id(edr_coll) if coll: collections.append(coll) - min_expires=expires if min_expires is None else min(min_expires, expires) + if expires is not None: + min_expires=expires if min_expires is None else min(min_expires, expires) else: logger.warning("Unable to fetch WMS GetCapabilities for %s", edr_coll) collections_data = Collections(links=links, collections=collections) @@ -755,7 +756,8 @@ async def rest_get_edr_inst_for_coll(collection_name: str, request: Request, res instance_link = Link(href=f"{instances_url}/{instance}", rel="collection") instance_links.append(instance_link) instance_info, expires = get_collectioninfo_for_id(collection_name, instance) - min_expires=expires if min_expires is None else min(min_expires, expires) + if expires is not None: + min_expires=expires if min_expires is None else min(min_expires, expires) instances.append(instance_info) instances_data = Instances(instances=instances, links=links) From 33bf3ad601d8e6b3edbe0e61458ebe187ea3ec74 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Tue, 13 Feb 2024 09:17:15 +0100 Subject: [PATCH 13/44] Re-enabled reading from redis --- .../edr_pydantic_classes/__init__.py | 0 .../edr_pydantic_classes/capabilities.py | 65 ------- .../edr_pydantic_classes/generic_models.py | 119 ------------- .../edr_pydantic_classes/instances.py | 164 ------------------ .../edr_pydantic_classes/my_base_model.py | 37 ---- .../edr_pydantic_classes/py.typed | 0 python/edr_package.MINE/setup.py | 31 ---- .../edr_pydantic_classes/__init__.py | 0 .../edr_pydantic_classes/capabilities.py | 65 ------- .../edr_pydantic_classes/generic_models.py | 114 ------------ .../edr_pydantic_classes/instances.py | 164 ------------------ .../edr_pydantic_classes/my_base_model.py | 37 ---- .../edr_pydantic_classes/py.typed | 0 python/edr_package.ORG/setup.py | 31 ---- .../routers/cachingmiddleware.py | 2 +- 15 files changed, 1 insertion(+), 828 deletions(-) delete mode 100644 python/edr_package.MINE/edr_pydantic_classes/__init__.py delete mode 100644 python/edr_package.MINE/edr_pydantic_classes/capabilities.py delete mode 100644 python/edr_package.MINE/edr_pydantic_classes/generic_models.py delete mode 100644 python/edr_package.MINE/edr_pydantic_classes/instances.py delete mode 100644 python/edr_package.MINE/edr_pydantic_classes/my_base_model.py delete mode 100644 python/edr_package.MINE/edr_pydantic_classes/py.typed delete mode 100755 python/edr_package.MINE/setup.py delete mode 100644 python/edr_package.ORG/edr_pydantic_classes/__init__.py delete mode 100644 python/edr_package.ORG/edr_pydantic_classes/capabilities.py delete mode 100644 python/edr_package.ORG/edr_pydantic_classes/generic_models.py delete mode 100644 python/edr_package.ORG/edr_pydantic_classes/instances.py delete mode 100644 python/edr_package.ORG/edr_pydantic_classes/my_base_model.py delete mode 100644 python/edr_package.ORG/edr_pydantic_classes/py.typed delete mode 100755 python/edr_package.ORG/setup.py diff --git a/python/edr_package.MINE/edr_pydantic_classes/__init__.py b/python/edr_package.MINE/edr_pydantic_classes/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/python/edr_package.MINE/edr_pydantic_classes/capabilities.py b/python/edr_package.MINE/edr_pydantic_classes/capabilities.py deleted file mode 100644 index 5a000490..00000000 --- a/python/edr_package.MINE/edr_pydantic_classes/capabilities.py +++ /dev/null @@ -1,65 +0,0 @@ -from __future__ import annotations - -from typing import List -from typing import Optional - -from pydantic import Field - -from .generic_models import Link -from .my_base_model import MyBaseModel - - -class Provider(MyBaseModel): - name: str - url: Optional[str] - - -class Contact(MyBaseModel): - email: Optional[str] - phone: Optional[str] - fax: Optional[str] - hours: Optional[str] - instructions: Optional[str] - address: Optional[str] - postalCode: Optional[str] # noqa: N815 - city: Optional[str] - stateorprovince: Optional[str] - country: Optional[str] - - -class LandingPageModel(MyBaseModel): - links: List[Link] = Field( - ..., - example=[ - { - "href": "http://www.example.org/edr", - "hreflang": "en", - "rel": "service-desc", - "type": "application/vnd.oai.openapi+json;version=3.0", - "title": "", - }, - { - "href": "http://www.example.org/edr/conformance", - "hreflang": "en", - "rel": "data", - "type": "application/json", - "title": "", - }, - { - "href": "http://www.example.org/edr/collections", - "hreflang": "en", - "rel": "data", - "type": "application/json", - "title": "", - }, - ], - ) - title: Optional[str] - description: Optional[str] - keywords: Optional[List[str]] - provider: Optional[Provider] - contact: Optional[Contact] - - -class ConformanceModel(MyBaseModel): - conformsTo: List[str] # noqa: N815 diff --git a/python/edr_package.MINE/edr_pydantic_classes/generic_models.py b/python/edr_package.MINE/edr_pydantic_classes/generic_models.py deleted file mode 100644 index b6917886..00000000 --- a/python/edr_package.MINE/edr_pydantic_classes/generic_models.py +++ /dev/null @@ -1,119 +0,0 @@ -from typing import List -from typing import Optional, Union -from enum import Enum - -from datetime import datetime - -from covjson_pydantic.domain import DomainType -from pydantic import Field - -from .my_base_model import MyBaseModel - - -class CRSOptions(str, Enum): - wgs84 = "WGS84" - - -class Spatial(MyBaseModel): - bbox: list[list[float]] - crs: CRSOptions - name: Optional[str] - - -class Temporal(MyBaseModel): - interval: list[list[datetime]] - values: list[str] - trs: str - name: Optional[str] - - -class Vertical(MyBaseModel): - interval: List[List[float]] - vrs: str - values: list[str] - name: Optional[str] - - -class Custom(MyBaseModel): - interval: List[str] - id: str - values: List[str] - reference: Optional[str] = None - - -class Extent(MyBaseModel): - spatial: Optional[Spatial] - temporal: Optional[Temporal] - vertical: Optional[Vertical] - custom: Optional[List[Custom]] - - -class CrsObject(MyBaseModel): - crs: str - wkt: str - - -class ObservedPropertyCollection(MyBaseModel): - id: Optional[str] = None - label: str - description: Optional[str] = None - # categories - - - -class Symbol(MyBaseModel, extra="allow"): - value: str - type: str - -class Unit(MyBaseModel): - label: Optional[str] - symbol: Optional[str] - id: Optional[Union[str, Symbol]] = None - - -class ParameterName(MyBaseModel): - id: Optional[str] = None - type: str = "Parameter" - label: Optional[str] = None - description: Optional[str] = None - data_type: Optional[str] = None - observedProperty: ObservedPropertyCollection - extent: Optional[Extent] = None - unit: Optional[Unit] = None - - -class Variables(MyBaseModel): - crs_details: list[CrsObject] - default_output_format: Optional[str] = None - output_formats: list[str] = [] - query_type: str = "" - title: str = "" - - -class Link(MyBaseModel): - href: str = Field( - ..., - example="http://data.example.com/collections/monitoringsites/locations/1234", - ) - rel: str = Field(..., example="alternate") - type: Optional[str] = Field(None, example="application/geo+json") - hreflang: Optional[str] = Field(None, example="en") - title: Optional[str] = Field(None, example="Monitoring site name") - length: Optional[int] = None - templated: Optional[bool] = Field( - False, - description="defines if the link href value is a template with values requiring replacement", - ) - variables: Optional[Variables] - - -class SupportedQueries(MyBaseModel): - domain_types: List[DomainType] = Field( - description="A list of domain types from which can be determined what endpoints are allowed.", - example="When [DomainType.point_series] is returned, " - "the /position endpoint is allowed but /cube is not allowed.", - ) - has_locations: bool = Field( - description="A boolean from which can be determined if the backend has the /locations endpoint.", - example="When True is returned, the backend has the /locations endpoint.", - ) diff --git a/python/edr_package.MINE/edr_pydantic_classes/instances.py b/python/edr_package.MINE/edr_pydantic_classes/instances.py deleted file mode 100644 index 40b532a9..00000000 --- a/python/edr_package.MINE/edr_pydantic_classes/instances.py +++ /dev/null @@ -1,164 +0,0 @@ -# generated by datamodel-codegen: -# filename: https://schemas.opengis.net/ogcapi/edr/1.0/openapi/schemas/instances.yaml -# timestamp: 2022-06-29T09:43:11+00:00 -from __future__ import annotations - - -from typing import Dict -from typing import List -from typing import Optional - -from pydantic import Field - -from .generic_models import Link, CrsObject, Variables, ParameterName, Extent -from .my_base_model import MyBaseModel - - -class InstancesVariables(Variables): - query_type: str = "instances" - title: str = "Instances query" - - -class InstancesLink(Link): - variables: InstancesVariables - - -class InstancesDataQueryLink(MyBaseModel): - link: InstancesLink - - -class PositionVariables(Variables): - query_type: str = "position" - title: str = "Position query" - coords: str = "Well Known Text POINT value i.e. POINT(24.9384 60.1699)" - - -class PositionLink(Link): - variables: PositionVariables - - -class PositionDataQueryLink(MyBaseModel): - link: PositionLink - - -class DataQueries(MyBaseModel): - position: Optional[PositionDataQueryLink] = None - # radius: Optional[DataQueryLink] = None - # area: Optional[DataQueryLink] = None - # cube: Optional[DataQueryLink] = None - # trajectory: Optional[DataQueryLink] = None - # corridor: Optional[DataQueryLink] = None - # locations: Optional[DataQueryLink] = None - # items: Optional[DataQueryLink] = None - instances: Optional[InstancesDataQueryLink] = None - - -class Instance(MyBaseModel): - links: List[Link] = Field( - ..., - example=[ - { - "href": "https://wwww.example.org/service/description.html", - "hreflang": "en", - "rel": "service-doc", - "type": "text/html", - "title": "", - }, - { - "href": "https://www.example.org/service/licence.html", - "hreflang": "en", - "rel": "licence", - "type": "text/html", - "title": "", - }, - { - "href": "https://www.example.org/service/terms-and-conditions.html", - "hreflang": "en", - "rel": "restrictions", - "type": "text/html", - "title": "", - }, - { - "href": "http://www.example.org/edr/collections/the_collection_id/", - "hreflang": "en", - "rel": "collection", - "type": "collection", - "title": "Collection", - }, - { - "href": "http://www.example.org/edr/collections/the_collection_id/position", - "hreflang": "en", - "rel": "data", - "type": "position", - "title": "Position", - }, - ], - ) - id: str - title: Optional[str] - description: Optional[str] - keywords: Optional[List[str]] - extent: Extent - data_queries: Optional[DataQueries] - crs: Optional[str | List[str]] - output_formats: Optional[List[str]] - parameter_names: Optional[Dict[str, ParameterName]] - - -class InstancesModel(MyBaseModel): - links: List[Link] = Field( - ..., - example=[ - { - "href": "http://www.example.org/edr/collections/the_collection_id/instances", - "hreflang": "en", - "rel": "self", - "type": "application/json", - }, - { - "href": "http://www.example.org/edr/collections/the_collection_id/instances?f=html", - "hreflang": "en", - "rel": "alternate", - "type": "text/html", - }, - { - "href": "http://www.example.org/edr/collections/the_collection_id/instances?f=xml", - "hreflang": "en", - "rel": "alternate", - "type": "application/xml", - }, - ], - ) - instances: List[Instance] - - -# For now, the collection metadata corresponds to the first instance metadata. So they have equal classes -class Collection(Instance): - pass - - -class CollectionsModel(MyBaseModel): - links: List[Link] = Field( - ..., - example=[ - { - "href": "http://www.example.org/edr/collections", - "hreflang": "en", - "rel": "self", - "type": "application/json", - }, - { - "href": "http://www.example.org/edr/collections?f=html", - "hreflang": "en", - "rel": "alternate", - "type": "text/html", - }, - { - "href": "http://www.example.org/edr/collections?f=xml", - "hreflang": "en", - "rel": "alternate", - "type": "application/xml", - }, - ], - ) - collections: List[Collection] diff --git a/python/edr_package.MINE/edr_pydantic_classes/my_base_model.py b/python/edr_package.MINE/edr_pydantic_classes/my_base_model.py deleted file mode 100644 index d6291d2d..00000000 --- a/python/edr_package.MINE/edr_pydantic_classes/my_base_model.py +++ /dev/null @@ -1,37 +0,0 @@ -import orjson -from pydantic import BaseModel -from pydantic import Extra - -from datetime import datetime -from zoneinfo import ZoneInfo - - -def orjson_dumps(v, *, default): - # orjson.dumps returns bytes, to match standard json.dumps we need to decode - return orjson.dumps( - v, - default=default, - option=orjson.OPT_NON_STR_KEYS | orjson.OPT_UTC_Z | orjson.OPT_NAIVE_UTC, - ).decode() - - -def convert_datetime_to_gmt(dt: datetime) -> str: - if not dt.tzinfo: - dt = dt.replace(tzinfo=ZoneInfo("UTC")) - - return dt.strftime("%Y-%m-%dT%H:%M:%SZ") - - -class MyBaseModel(BaseModel): - class Config: - allow_population_by_field_name = True - anystr_strip_whitespace = True - extra = Extra.forbid - min_anystr_length = 1 - smart_union = True - validate_all = True - validate_assignment = True - - json_loads = orjson.loads - json_dumps = orjson_dumps - json_encoders = {datetime: convert_datetime_to_gmt} diff --git a/python/edr_package.MINE/edr_pydantic_classes/py.typed b/python/edr_package.MINE/edr_pydantic_classes/py.typed deleted file mode 100644 index e69de29b..00000000 diff --git a/python/edr_package.MINE/setup.py b/python/edr_package.MINE/setup.py deleted file mode 100755 index bdb32bf4..00000000 --- a/python/edr_package.MINE/setup.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 -import logging -import os - -import setuptools - -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(os.environ.get("LOG_LEVEL", "INFO")) - -# Package meta-data. -NAME = "edr-pydantic-classes" - -env_suffix = os.environ.get("ENVIRONMENT_SUFFIX", "") -logger.debug(f"Environment suffix: {env_suffix}") - -if env_suffix: - NAME += f"-{env_suffix}" -logger.debug(f"Package name: {NAME}") - -setuptools.setup( - name=NAME, - version="0.0.9", - description="The Pydantic models for EDR datatypes", - package_data={"edr_pydantic_classes": ["py.typed"]}, - packages=["edr_pydantic_classes"], - include_package_data=False, - license="MIT", - install_requires=["pydantic", "orjson", "covjson-pydantic"], - python_requires=">=3.8.0", -) diff --git a/python/edr_package.ORG/edr_pydantic_classes/__init__.py b/python/edr_package.ORG/edr_pydantic_classes/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/python/edr_package.ORG/edr_pydantic_classes/capabilities.py b/python/edr_package.ORG/edr_pydantic_classes/capabilities.py deleted file mode 100644 index 5a000490..00000000 --- a/python/edr_package.ORG/edr_pydantic_classes/capabilities.py +++ /dev/null @@ -1,65 +0,0 @@ -from __future__ import annotations - -from typing import List -from typing import Optional - -from pydantic import Field - -from .generic_models import Link -from .my_base_model import MyBaseModel - - -class Provider(MyBaseModel): - name: str - url: Optional[str] - - -class Contact(MyBaseModel): - email: Optional[str] - phone: Optional[str] - fax: Optional[str] - hours: Optional[str] - instructions: Optional[str] - address: Optional[str] - postalCode: Optional[str] # noqa: N815 - city: Optional[str] - stateorprovince: Optional[str] - country: Optional[str] - - -class LandingPageModel(MyBaseModel): - links: List[Link] = Field( - ..., - example=[ - { - "href": "http://www.example.org/edr", - "hreflang": "en", - "rel": "service-desc", - "type": "application/vnd.oai.openapi+json;version=3.0", - "title": "", - }, - { - "href": "http://www.example.org/edr/conformance", - "hreflang": "en", - "rel": "data", - "type": "application/json", - "title": "", - }, - { - "href": "http://www.example.org/edr/collections", - "hreflang": "en", - "rel": "data", - "type": "application/json", - "title": "", - }, - ], - ) - title: Optional[str] - description: Optional[str] - keywords: Optional[List[str]] - provider: Optional[Provider] - contact: Optional[Contact] - - -class ConformanceModel(MyBaseModel): - conformsTo: List[str] # noqa: N815 diff --git a/python/edr_package.ORG/edr_pydantic_classes/generic_models.py b/python/edr_package.ORG/edr_pydantic_classes/generic_models.py deleted file mode 100644 index ccbe666d..00000000 --- a/python/edr_package.ORG/edr_pydantic_classes/generic_models.py +++ /dev/null @@ -1,114 +0,0 @@ -from typing import List -from typing import Optional -from enum import Enum - -from datetime import datetime - -from covjson_pydantic.domain import DomainType -from pydantic import Field - -from .my_base_model import MyBaseModel - - -class CRSOptions(str, Enum): - wgs84 = "WGS84" - - -class Spatial(MyBaseModel): - bbox: list[list[float]] - crs: CRSOptions - name: Optional[str] - - -class Temporal(MyBaseModel): - interval: list[list[datetime]] - values: list[str] - trs: str - name: Optional[str] - - -class Vertical(MyBaseModel): - interval: List[List[float]] - vrs: str - values: list[str] - name: Optional[str] - - -class Custom(MyBaseModel): - interval: List[str] - id: str - values: List[str] - reference: Optional[str] = None - - -class Extent(MyBaseModel): - spatial: Optional[Spatial] - temporal: Optional[Temporal] - vertical: Optional[Vertical] - custom: Optional[List[Custom]] - - -class CrsObject(MyBaseModel): - crs: str - wkt: str - - -class ObservedPropertyCollection(MyBaseModel): - id: Optional[str] = None - label: str - description: Optional[str] = None - # categories - - -class Units(MyBaseModel): - label: Optional[str] - symbol: Optional[str] - id: Optional[str] = None - - -class ParameterName(MyBaseModel): - id: Optional[str] = None - type: str = "Parameter" - label: Optional[str] = None - description: Optional[str] = None - data_type: Optional[str] = None - observedProperty: ObservedPropertyCollection - extent: Optional[Extent] = None - unit: Optional[Units] = None - - -class Variables(MyBaseModel): - crs_details: list[CrsObject] - default_output_format: Optional[str] = None - output_formats: list[str] = [] - query_type: str = "" - title: str = "" - - -class Link(MyBaseModel): - href: str = Field( - ..., - example="http://data.example.com/collections/monitoringsites/locations/1234", - ) - rel: str = Field(..., example="alternate") - type: Optional[str] = Field(None, example="application/geo+json") - hreflang: Optional[str] = Field(None, example="en") - title: Optional[str] = Field(None, example="Monitoring site name") - length: Optional[int] = None - templated: Optional[bool] = Field( - False, - description="defines if the link href value is a template with values requiring replacement", - ) - variables: Optional[Variables] - - -class SupportedQueries(MyBaseModel): - domain_types: List[DomainType] = Field( - description="A list of domain types from which can be determined what endpoints are allowed.", - example="When [DomainType.point_series] is returned, " - "the /position endpoint is allowed but /cube is not allowed.", - ) - has_locations: bool = Field( - description="A boolean from which can be determined if the backend has the /locations endpoint.", - example="When True is returned, the backend has the /locations endpoint.", - ) diff --git a/python/edr_package.ORG/edr_pydantic_classes/instances.py b/python/edr_package.ORG/edr_pydantic_classes/instances.py deleted file mode 100644 index 40b532a9..00000000 --- a/python/edr_package.ORG/edr_pydantic_classes/instances.py +++ /dev/null @@ -1,164 +0,0 @@ -# generated by datamodel-codegen: -# filename: https://schemas.opengis.net/ogcapi/edr/1.0/openapi/schemas/instances.yaml -# timestamp: 2022-06-29T09:43:11+00:00 -from __future__ import annotations - - -from typing import Dict -from typing import List -from typing import Optional - -from pydantic import Field - -from .generic_models import Link, CrsObject, Variables, ParameterName, Extent -from .my_base_model import MyBaseModel - - -class InstancesVariables(Variables): - query_type: str = "instances" - title: str = "Instances query" - - -class InstancesLink(Link): - variables: InstancesVariables - - -class InstancesDataQueryLink(MyBaseModel): - link: InstancesLink - - -class PositionVariables(Variables): - query_type: str = "position" - title: str = "Position query" - coords: str = "Well Known Text POINT value i.e. POINT(24.9384 60.1699)" - - -class PositionLink(Link): - variables: PositionVariables - - -class PositionDataQueryLink(MyBaseModel): - link: PositionLink - - -class DataQueries(MyBaseModel): - position: Optional[PositionDataQueryLink] = None - # radius: Optional[DataQueryLink] = None - # area: Optional[DataQueryLink] = None - # cube: Optional[DataQueryLink] = None - # trajectory: Optional[DataQueryLink] = None - # corridor: Optional[DataQueryLink] = None - # locations: Optional[DataQueryLink] = None - # items: Optional[DataQueryLink] = None - instances: Optional[InstancesDataQueryLink] = None - - -class Instance(MyBaseModel): - links: List[Link] = Field( - ..., - example=[ - { - "href": "https://wwww.example.org/service/description.html", - "hreflang": "en", - "rel": "service-doc", - "type": "text/html", - "title": "", - }, - { - "href": "https://www.example.org/service/licence.html", - "hreflang": "en", - "rel": "licence", - "type": "text/html", - "title": "", - }, - { - "href": "https://www.example.org/service/terms-and-conditions.html", - "hreflang": "en", - "rel": "restrictions", - "type": "text/html", - "title": "", - }, - { - "href": "http://www.example.org/edr/collections/the_collection_id/", - "hreflang": "en", - "rel": "collection", - "type": "collection", - "title": "Collection", - }, - { - "href": "http://www.example.org/edr/collections/the_collection_id/position", - "hreflang": "en", - "rel": "data", - "type": "position", - "title": "Position", - }, - ], - ) - id: str - title: Optional[str] - description: Optional[str] - keywords: Optional[List[str]] - extent: Extent - data_queries: Optional[DataQueries] - crs: Optional[str | List[str]] - output_formats: Optional[List[str]] - parameter_names: Optional[Dict[str, ParameterName]] - - -class InstancesModel(MyBaseModel): - links: List[Link] = Field( - ..., - example=[ - { - "href": "http://www.example.org/edr/collections/the_collection_id/instances", - "hreflang": "en", - "rel": "self", - "type": "application/json", - }, - { - "href": "http://www.example.org/edr/collections/the_collection_id/instances?f=html", - "hreflang": "en", - "rel": "alternate", - "type": "text/html", - }, - { - "href": "http://www.example.org/edr/collections/the_collection_id/instances?f=xml", - "hreflang": "en", - "rel": "alternate", - "type": "application/xml", - }, - ], - ) - instances: List[Instance] - - -# For now, the collection metadata corresponds to the first instance metadata. So they have equal classes -class Collection(Instance): - pass - - -class CollectionsModel(MyBaseModel): - links: List[Link] = Field( - ..., - example=[ - { - "href": "http://www.example.org/edr/collections", - "hreflang": "en", - "rel": "self", - "type": "application/json", - }, - { - "href": "http://www.example.org/edr/collections?f=html", - "hreflang": "en", - "rel": "alternate", - "type": "text/html", - }, - { - "href": "http://www.example.org/edr/collections?f=xml", - "hreflang": "en", - "rel": "alternate", - "type": "application/xml", - }, - ], - ) - collections: List[Collection] diff --git a/python/edr_package.ORG/edr_pydantic_classes/my_base_model.py b/python/edr_package.ORG/edr_pydantic_classes/my_base_model.py deleted file mode 100644 index d6291d2d..00000000 --- a/python/edr_package.ORG/edr_pydantic_classes/my_base_model.py +++ /dev/null @@ -1,37 +0,0 @@ -import orjson -from pydantic import BaseModel -from pydantic import Extra - -from datetime import datetime -from zoneinfo import ZoneInfo - - -def orjson_dumps(v, *, default): - # orjson.dumps returns bytes, to match standard json.dumps we need to decode - return orjson.dumps( - v, - default=default, - option=orjson.OPT_NON_STR_KEYS | orjson.OPT_UTC_Z | orjson.OPT_NAIVE_UTC, - ).decode() - - -def convert_datetime_to_gmt(dt: datetime) -> str: - if not dt.tzinfo: - dt = dt.replace(tzinfo=ZoneInfo("UTC")) - - return dt.strftime("%Y-%m-%dT%H:%M:%SZ") - - -class MyBaseModel(BaseModel): - class Config: - allow_population_by_field_name = True - anystr_strip_whitespace = True - extra = Extra.forbid - min_anystr_length = 1 - smart_union = True - validate_all = True - validate_assignment = True - - json_loads = orjson.loads - json_dumps = orjson_dumps - json_encoders = {datetime: convert_datetime_to_gmt} diff --git a/python/edr_package.ORG/edr_pydantic_classes/py.typed b/python/edr_package.ORG/edr_pydantic_classes/py.typed deleted file mode 100644 index e69de29b..00000000 diff --git a/python/edr_package.ORG/setup.py b/python/edr_package.ORG/setup.py deleted file mode 100755 index bdb32bf4..00000000 --- a/python/edr_package.ORG/setup.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 -import logging -import os - -import setuptools - -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(os.environ.get("LOG_LEVEL", "INFO")) - -# Package meta-data. -NAME = "edr-pydantic-classes" - -env_suffix = os.environ.get("ENVIRONMENT_SUFFIX", "") -logger.debug(f"Environment suffix: {env_suffix}") - -if env_suffix: - NAME += f"-{env_suffix}" -logger.debug(f"Package name: {NAME}") - -setuptools.setup( - name=NAME, - version="0.0.9", - description="The Pydantic models for EDR datatypes", - package_data={"edr_pydantic_classes": ["py.typed"]}, - packages=["edr_pydantic_classes"], - include_package_data=False, - license="MIT", - install_requires=["pydantic", "orjson", "covjson-pydantic"], - python_requires=">=3.8.0", -) diff --git a/python/python_fastapi_server/routers/cachingmiddleware.py b/python/python_fastapi_server/routers/cachingmiddleware.py index 58e02574..d198c82f 100644 --- a/python/python_fastapi_server/routers/cachingmiddleware.py +++ b/python/python_fastapi_server/routers/cachingmiddleware.py @@ -68,7 +68,7 @@ async def dispatch(self, request, call_next): return await call_next(request) #Check if request is in cache, if so return that - expire, headers, data = None, None, None #await get_cached_response(self.redis, request) + expire, headers, data = await get_cached_response(self.redis, request) if data: #Fix Age header From 86ac73747a282b9bae958afc55c550157193d845 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Fri, 23 Feb 2024 13:35:00 +0100 Subject: [PATCH 14/44] Extra caching in runadaguc.py --- python/lib/adaguc/runAdaguc.py | 72 +++++++++++++++++++ .../routers/cachingmiddleware.py | 1 + 2 files changed, 73 insertions(+) diff --git a/python/lib/adaguc/runAdaguc.py b/python/lib/adaguc/runAdaguc.py index 527934a2..2e9c434b 100644 --- a/python/lib/adaguc/runAdaguc.py +++ b/python/lib/adaguc/runAdaguc.py @@ -9,6 +9,12 @@ import shutil import random import string +from redis import from_url + +import re +import calendar +import json +from datetime import datetime, timedelta from adaguc.CGIRunner import CGIRunner @@ -34,6 +40,13 @@ def __init__(self): self.ADAGUC_TMP = os.getenv("ADAGUC_TMP", "/tmp") self.ADAGUC_FONT = os.getenv( "ADAGUC_FONT", self.ADAGUC_PATH + "/data/fonts/Roboto-Medium.ttf") + self.ADAGUC_REDIS = os.getenv("ADAGUC_REDIS", "") + if self.ADAGUC_REDIS.startswith("redis://") or self.ADAGUC_REDIS.startswith("rediss://"): + self.redis_url = self.ADAGUC_REDIS + self.use_cache = True + print("USE CACHE") + else: + self.use_cache = False def setAdagucPath(self, newAdagucPath): self.ADAGUC_PATH = newAdagucPath @@ -142,6 +155,7 @@ def runADAGUCServer( # adagucenv=os.environ.copy() # adagucenv.update(env) + url = re.sub(r"^(.*)(&[0-9\.]*)$", r"\g<1>&1", url) adagucenv = env adagucenv["ADAGUC_ENABLELOGBUFFER"] = os.getenv( @@ -180,6 +194,16 @@ def runADAGUCServer( if args is not None: adagucargs = adagucargs + args + # Check cache for entry with keys of (url,adagucargs) if configured + if self.use_cache: + cache_key = str((url, adagucargs)) + print(f"Checking cache for {cache_key}") + redis = from_url(self.ADAGUC_REDIS) + age, headers, data = get_cached_response(redis, cache_key) + if age is not None: + return [0, data, headers] + + print(f"Generating {cache_key}") filetogenerate = BytesIO() status, headers, processErr = CGIRunner().run( adagucargs, @@ -213,6 +237,10 @@ def runADAGUCServer( return [status, filetogenerate, headers] else: + if self.use_cache: + print(f"CACHING {cache_key}") + cache_response(redis, cache_key, headers, filetogenerate, 60) + print("HEADERS:", headers) return [status, filetogenerate, headers] def writetofile(self, filename, data): @@ -236,3 +264,47 @@ def cleanTempDir(self): def mkdir_p(self, directory): if not os.path.exists(directory): os.makedirs(directory) + +skip_headers = ["x-process-time", "age"] +def cache_response(redis_client, key, headers:str, data, ex: int=60): + allheaders=[] + expire = ex + for header in headers: + k,v = header.split(":") + if k not in skip_headers: + allheaders.append(header) + if k.lower().startswith("cache-control"): + for term in v.split(";"): + if term.startswith("max-age"): + try: + expire = int(term.split('=')[1]) + if expire==0: + expire=ex + except: + expire=ex + + allheaders_json=json.dumps(allheaders) + + entrytime="%10d"%calendar.timegm(datetime.utcnow().utctimetuple()) + redis_client.set(key, bytes(entrytime, 'utf-8')+bytes("%06d"%len(allheaders_json), 'utf-8')+bytes(allheaders_json, 'utf-8')+data.getvalue(), ex=expire) + redis_client.close() + +def get_cached_response(redis_client, key): + cached = redis_client.get(key) + redis_client.close() + if not cached: + print("Cache miss") + return None, None, None + print("Cache hit", len(cached)) + + entrytime=int(cached[:10]) + currenttime=calendar.timegm(datetime.utcnow().utctimetuple()) + age=currenttime-entrytime + + headers_len=int(cached[10:16]) + headers=json.loads(cached[16:16+headers_len]) + headers.append(f"Age: {age}") + + data = cached[16+headers_len:] + print("HIT: ", type(data)) + return age, headers, BytesIO(data) diff --git a/python/python_fastapi_server/routers/cachingmiddleware.py b/python/python_fastapi_server/routers/cachingmiddleware.py index d198c82f..6d1641a2 100644 --- a/python/python_fastapi_server/routers/cachingmiddleware.py +++ b/python/python_fastapi_server/routers/cachingmiddleware.py @@ -60,6 +60,7 @@ def __init__(self, app): if "ADAGUC_REDIS" in os.environ: self.shortcut=False self.redis = None + self.shortcut = True #TODO async def dispatch(self, request, call_next): if self.redis is None: From f7a85aae497bd4d002e8877639cd3eca8804cba8 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Mon, 26 Feb 2024 11:47:47 +0100 Subject: [PATCH 15/44] run_adaguc caches only getcapabilities --- python/lib/adaguc/runAdaguc.py | 14 +++++++++++--- python/python_fastapi_server/main.py | 1 - python/python_fastapi_server/routers/edr.py | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/python/lib/adaguc/runAdaguc.py b/python/lib/adaguc/runAdaguc.py index 2e9c434b..1f4d700a 100644 --- a/python/lib/adaguc/runAdaguc.py +++ b/python/lib/adaguc/runAdaguc.py @@ -142,6 +142,13 @@ def printLogFile(self): print(self.getLogFile()) print("=== END ADAGUC LOGS ===") + def cache_wanted(self, url: str): + if not self.use_cache: + return False + if "getcapabilities" in url.lower(): + return True + return False + def runADAGUCServer( self, url=None, @@ -195,7 +202,7 @@ def runADAGUCServer( adagucargs = adagucargs + args # Check cache for entry with keys of (url,adagucargs) if configured - if self.use_cache: + if self.cache_wanted(url): cache_key = str((url, adagucargs)) print(f"Checking cache for {cache_key}") redis = from_url(self.ADAGUC_REDIS) @@ -203,7 +210,8 @@ def runADAGUCServer( if age is not None: return [0, data, headers] - print(f"Generating {cache_key}") + print(f"Generating {cache_key}") + filetogenerate = BytesIO() status, headers, processErr = CGIRunner().run( adagucargs, @@ -237,7 +245,7 @@ def runADAGUCServer( return [status, filetogenerate, headers] else: - if self.use_cache: + if self.cache_wanted(url): print(f"CACHING {cache_key}") cache_response(redis, cache_key, headers, filetogenerate, 60) print("HEADERS:", headers) diff --git a/python/python_fastapi_server/main.py b/python/python_fastapi_server/main.py index 5c266bec..c8a736d4 100644 --- a/python/python_fastapi_server/main.py +++ b/python/python_fastapi_server/main.py @@ -22,7 +22,6 @@ from routers.opendap import opendapRouter from routers.wmswcs import testadaguc, wmsWcsRouter from routers.cachingmiddleware import CachingMiddleware -# from routers.cachingmiddleware2 import CachingMiddleware2 logger = logging.getLogger(__name__) diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index ed82c4e9..8b299f2f 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -657,7 +657,7 @@ def get_ttl_from_adaguc_call(headers): break return ttl -@cached(cache=edr_cache, key=partial(hashkey, "get_capabilities")) +# @cached(cache=edr_cache, key=partial(hashkey, "get_capabilities")) def get_capabilities(collname): """ Get the collectioninfo from the WMS GetCapabilities From 31dc3edc23c8f9d40711138e97369f7a2708ac96 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Wed, 28 Feb 2024 17:57:30 +0100 Subject: [PATCH 16/44] worked on review comments --- .../routers/cachingmiddleware.py | 44 ++++++++----------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/python/python_fastapi_server/routers/cachingmiddleware.py b/python/python_fastapi_server/routers/cachingmiddleware.py index 6d1641a2..e957193d 100644 --- a/python/python_fastapi_server/routers/cachingmiddleware.py +++ b/python/python_fastapi_server/routers/cachingmiddleware.py @@ -1,14 +1,12 @@ import os from urllib.parse import urlsplit -# from starlette.middleware.base import BaseHTTPMiddleware from starlette.middleware.base import BaseHTTPMiddleware from starlette.responses import Response from fastapi import BackgroundTasks import calendar -from datetime import datetime, timedelta -import time +from datetime import datetime import redis.asyncio as redis import json @@ -17,36 +15,35 @@ async def get_cached_response(redis_client, request): key = generate_key(request) cached = await redis_client.get(key) - await redis_client.aclose() + # await redis_client.close() if not cached: # print("Cache miss") return None, None, None # print("Cache hit", len(cached)) - entrytime=int(cached[:10]) - currenttime=calendar.timegm(datetime.utcnow().utctimetuple()) - age=currenttime-entrytime + entrytime = int(cached[:10]) + currenttime = calendar.timegm(datetime.utcnow().utctimetuple()) + age = currenttime-entrytime - headers_len=int(cached[10:16]) - headers=json.loads(cached[16:16+headers_len]) + headers_len = int(cached[10:16]) + headers = json.loads(cached[16:16+headers_len]) data = cached[16+headers_len:] return age, headers, data -skip_headers=["x-process-time", "age"] +skip_headers = ["x-process-time", "age"] async def cache_response(redis_client, request, headers, data, ex: int=60): key=generate_key(request) - allheaders={} + headers_to_keep={} for k in headers.keys(): if k not in skip_headers: - allheaders[k]=headers[k] - allheaders_json=json.dumps(allheaders) + headers_to_keep[k] = headers[k] + headers_to_keep_json = json.dumps(headers_to_keep) entrytime="%10d"%calendar.timegm(datetime.utcnow().utctimetuple()) - await redis_client.set(key, bytes(entrytime, 'utf-8')+bytes("%06d"%len(allheaders_json), 'utf-8')+bytes(allheaders_json, 'utf-8')+data, ex=ex) - await redis_client.aclose() + await redis_client.set(key, bytes(entrytime, 'utf-8')+bytes("%06d"%len(headers_to_keep_json), 'utf-8')+bytes(headers_to_keep_json, 'utf-8')+data, ex=ex) def generate_key(request): key = f"{request.url.path}?{request['query_string']}" @@ -59,21 +56,18 @@ def __init__(self, app): super().__init__(app) if "ADAGUC_REDIS" in os.environ: self.shortcut=False - self.redis = None - self.shortcut = True #TODO + self.redis = redis.from_url(ADAGUC_REDIS) async def dispatch(self, request, call_next): - if self.redis is None: - self.redis = redis.from_url(ADAGUC_REDIS) if self.shortcut: return await call_next(request) #Check if request is in cache, if so return that - expire, headers, data = await get_cached_response(self.redis, request) + age, headers, data = await get_cached_response(self.redis, request) if data: #Fix Age header - headers["Age"]="%1d"%(expire) + headers["Age"] = "%1d"%(age) return Response(content=data, status_code=200, headers=headers, media_type=headers['content-type']) response: Response = await call_next(request) @@ -85,15 +79,15 @@ async def dispatch(self, request, call_next): for term in cache_control_terms: age_terms = term.split("=") if age_terms[0].lower()=="max-age": - ttl=int(age_terms[1]) + ttl = int(age_terms[1]) break if ttl is None: - return + return response response_body = b"" async for chunk in response.body_iterator: - response_body+=chunk + response_body += chunk tasks = BackgroundTasks() tasks.add_task(cache_response, redis_client=self.redis, request=request, headers=response.headers, data=response_body, ex=ttl) - response.headers['age']="0" + response.headers['age'] = "0" return Response(content=response_body, status_code=200, headers=response.headers, media_type=response.media_type, background=tasks) return response From f391f7fec161a1106f078e58c0d19bce3ac3fc15 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Thu, 29 Feb 2024 11:57:44 +0100 Subject: [PATCH 17/44] More review comments --- python/lib/adaguc/runAdaguc.py | 31 +++++++++---------- .../routers/cachingmiddleware.py | 7 ++--- python/python_fastapi_server/routers/edr.py | 9 ------ 3 files changed, 18 insertions(+), 29 deletions(-) diff --git a/python/lib/adaguc/runAdaguc.py b/python/lib/adaguc/runAdaguc.py index 1f4d700a..55225914 100644 --- a/python/lib/adaguc/runAdaguc.py +++ b/python/lib/adaguc/runAdaguc.py @@ -44,6 +44,7 @@ def __init__(self): if self.ADAGUC_REDIS.startswith("redis://") or self.ADAGUC_REDIS.startswith("rediss://"): self.redis_url = self.ADAGUC_REDIS self.use_cache = True + self.redis_client = from_url(self.redis_url) print("USE CACHE") else: self.use_cache = False @@ -145,7 +146,7 @@ def printLogFile(self): def cache_wanted(self, url: str): if not self.use_cache: return False - if "getcapabilities" in url.lower(): + if "request=getcapabilities" in url.lower(): return True return False @@ -205,8 +206,8 @@ def runADAGUCServer( if self.cache_wanted(url): cache_key = str((url, adagucargs)) print(f"Checking cache for {cache_key}") - redis = from_url(self.ADAGUC_REDIS) - age, headers, data = get_cached_response(redis, cache_key) + + age, headers, data = get_cached_response(self.redis_client, cache_key) if age is not None: return [0, data, headers] @@ -247,7 +248,7 @@ def runADAGUCServer( else: if self.cache_wanted(url): print(f"CACHING {cache_key}") - cache_response(redis, cache_key, headers, filetogenerate, 60) + cache_response(self.redis_client, cache_key, headers, filetogenerate) print("HEADERS:", headers) return [status, filetogenerate, headers] @@ -274,28 +275,26 @@ def mkdir_p(self, directory): os.makedirs(directory) skip_headers = ["x-process-time", "age"] -def cache_response(redis_client, key, headers:str, data, ex: int=60): - allheaders=[] - expire = ex +def cache_response(redis_client, key, headers:str, data): + useable_headers=[] + ttl = 0 for header in headers: k,v = header.split(":") if k not in skip_headers: - allheaders.append(header) + useable_headers.append(header) if k.lower().startswith("cache-control"): for term in v.split(";"): if term.startswith("max-age"): try: - expire = int(term.split('=')[1]) - if expire==0: - expire=ex + ttl = int(term.split('=')[1]) except: - expire=ex + pass - allheaders_json=json.dumps(allheaders) + if ttl>0: + useable_headers_json=json.dumps(useable_headers) - entrytime="%10d"%calendar.timegm(datetime.utcnow().utctimetuple()) - redis_client.set(key, bytes(entrytime, 'utf-8')+bytes("%06d"%len(allheaders_json), 'utf-8')+bytes(allheaders_json, 'utf-8')+data.getvalue(), ex=expire) - redis_client.close() + entrytime="%10d"%calendar.timegm(datetime.utcnow().utctimetuple()) + redis_client.set(key, bytes(entrytime, 'utf-8')+bytes("%06d"%len(useable_headers_json), 'utf-8')+bytes(useable_headers_json, 'utf-8')+data.getvalue(), ex=ttl) def get_cached_response(redis_client, key): cached = redis_client.get(key) diff --git a/python/python_fastapi_server/routers/cachingmiddleware.py b/python/python_fastapi_server/routers/cachingmiddleware.py index e957193d..4463201f 100644 --- a/python/python_fastapi_server/routers/cachingmiddleware.py +++ b/python/python_fastapi_server/routers/cachingmiddleware.py @@ -33,7 +33,7 @@ async def get_cached_response(redis_client, request): skip_headers = ["x-process-time", "age"] -async def cache_response(redis_client, request, headers, data, ex: int=60): +async def response_to_cache(redis_client, request, headers, data, ex: int=60): key=generate_key(request) headers_to_keep={} @@ -46,8 +46,7 @@ async def cache_response(redis_client, request, headers, data, ex: int=60): await redis_client.set(key, bytes(entrytime, 'utf-8')+bytes("%06d"%len(headers_to_keep_json), 'utf-8')+bytes(headers_to_keep_json, 'utf-8')+data, ex=ex) def generate_key(request): - key = f"{request.url.path}?{request['query_string']}" - # print(f"generate_key({key})") + key = f"{request.url.path}?bytes({request['query_string']}, 'utf-8')" return key class CachingMiddleware(BaseHTTPMiddleware): @@ -87,7 +86,7 @@ async def dispatch(self, request, call_next): async for chunk in response.body_iterator: response_body += chunk tasks = BackgroundTasks() - tasks.add_task(cache_response, redis_client=self.redis, request=request, headers=response.headers, data=response_body, ex=ttl) + tasks.add_task(response_to_cache, redis_client=self.redis, request=request, headers=response.headers, data=response_body, ex=ttl) response.headers['age'] = "0" return Response(content=response_body, status_code=200, headers=response.headers, media_type=response.media_type, background=tasks) return response diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index 8b299f2f..ce9718b7 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -15,9 +15,6 @@ from datetime import datetime, timezone, timedelta from typing import Union -from cachetools import TTLCache, cached -from cachetools.keys import hashkey -from CacheToolsUtils import PrefixedRedisCache from covjson_pydantic.coverage import Coverage from covjson_pydantic.domain import Domain, ValuesAxis from covjson_pydantic.observed_property import ( @@ -66,12 +63,6 @@ logger = logging.getLogger(__name__) logger.debug("Starting EDR") -redis_url = os.environ.get("ADAGUC_REDIS", None) -if redis_url: - edr_cache = PrefixedRedisCache(from_url(os.environ.get("ADAGUC_REDIS")), "edr", SHORT_CACHE_TIME) -else: - edr_cache = TTLCache(maxsize=1024, ttl=SHORT_CACHE_TIME) - edrApiApp = FastAPI(debug=True) OWSLIB_DUMMY_URL = "http://localhost:8000" From 7bb0cd69917950951731de16f61b76b31150a5e0 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Mon, 4 Mar 2024 13:10:22 +0100 Subject: [PATCH 18/44] Fixed review comments; cache compression --- python/lib/adaguc/runAdaguc.py | 25 +++++------ python/python_fastapi_server/main.py | 2 +- ...ingmiddleware.py => caching_middleware.py} | 43 +++++++++++-------- requirements.in | 3 +- requirements.txt | 2 - 5 files changed, 41 insertions(+), 34 deletions(-) rename python/python_fastapi_server/routers/{cachingmiddleware.py => caching_middleware.py} (66%) diff --git a/python/lib/adaguc/runAdaguc.py b/python/lib/adaguc/runAdaguc.py index f0c0b5de..6ea8399d 100644 --- a/python/lib/adaguc/runAdaguc.py +++ b/python/lib/adaguc/runAdaguc.py @@ -163,7 +163,8 @@ def runADAGUCServer( # adagucenv=os.environ.copy() # adagucenv.update(env) - url = re.sub(r"^(.*)(&[0-9\.]*)$", r"\g<1>&1", url) + # url = re.sub(r"^(.*)(&[0-9\.]*)$", r"\g<1>&1", url) # This removes adaguc-viewer's unique-URL feature + adagucenv = env adagucenv["ADAGUC_ENABLELOGBUFFER"] = os.getenv( @@ -248,7 +249,7 @@ def runADAGUCServer( else: if self.cache_wanted(url): print(f"CACHING {cache_key}") - cache_response(self.redis_client, cache_key, headers, filetogenerate) + response_to_cache(self.redis_client, cache_key, headers, filetogenerate) print("HEADERS:", headers) return [status, filetogenerate, headers] @@ -275,13 +276,13 @@ def mkdir_p(self, directory): os.makedirs(directory) skip_headers = ["x-process-time", "age"] -def cache_response(redis_client, key, headers:str, data): - useable_headers=[] +def response_to_cache(redis_client, key, headers:str, data): + cacheable_headers=[] ttl = 0 for header in headers: k,v = header.split(":") if k not in skip_headers: - useable_headers.append(header) + cacheable_headers.append(header) if k.lower().startswith("cache-control"): for term in v.split(";"): if term.startswith("max-age"): @@ -291,10 +292,10 @@ def cache_response(redis_client, key, headers:str, data): pass if ttl>0: - useable_headers_json=json.dumps(useable_headers) + cacheable_headers_json=json.dumps(cacheable_headers, ensure_ascii=True).encode('utf-8') - entrytime="%10d"%calendar.timegm(datetime.utcnow().utctimetuple()) - redis_client.set(key, bytes(entrytime, 'utf-8')+bytes("%06d"%len(useable_headers_json), 'utf-8')+bytes(useable_headers_json, 'utf-8')+data.getvalue(), ex=ttl) + entrytime=f"{calendar.timegm(datetime.utcnow().utctimetuple()):10d}".encode('utf-8') + redis_client.set(key, entrytime+f"{len(cacheable_headers_json):06d}".encode('utf-8')+cacheable_headers_json+data.getvalue(), ex=ttl) def get_cached_response(redis_client, key): cached = redis_client.get(key) @@ -304,14 +305,14 @@ def get_cached_response(redis_client, key): return None, None, None print("Cache hit", len(cached)) - entrytime=int(cached[:10]) + entrytime=int(cached[:10].decode('utf-8')) currenttime=calendar.timegm(datetime.utcnow().utctimetuple()) age=currenttime-entrytime - headers_len=int(cached[10:16]) + headers_len=int(cached[10:16].decode('utf-8')) headers=json.loads(cached[16:16+headers_len]) - headers.append(f"Age: {age}") + headers.append(f"age: {age}") data = cached[16+headers_len:] - print("HIT: ", type(data)) + print(f"HIT: {len(data)} bytes cached") return age, headers, BytesIO(data) diff --git a/python/python_fastapi_server/main.py b/python/python_fastapi_server/main.py index c8a736d4..3f205c9a 100644 --- a/python/python_fastapi_server/main.py +++ b/python/python_fastapi_server/main.py @@ -21,7 +21,7 @@ from routers.ogcapi import ogcApiApp from routers.opendap import opendapRouter from routers.wmswcs import testadaguc, wmsWcsRouter -from routers.cachingmiddleware import CachingMiddleware +from routers.caching_middleware import CachingMiddleware logger = logging.getLogger(__name__) diff --git a/python/python_fastapi_server/routers/cachingmiddleware.py b/python/python_fastapi_server/routers/caching_middleware.py similarity index 66% rename from python/python_fastapi_server/routers/cachingmiddleware.py rename to python/python_fastapi_server/routers/caching_middleware.py index 4463201f..ae76243c 100644 --- a/python/python_fastapi_server/routers/cachingmiddleware.py +++ b/python/python_fastapi_server/routers/caching_middleware.py @@ -1,3 +1,4 @@ +from io import BytesIO import os from urllib.parse import urlsplit @@ -9,6 +10,7 @@ from datetime import datetime import redis.asyncio as redis import json +import brotli ADAGUC_REDIS=os.environ.get('ADAGUC_REDIS') @@ -21,32 +23,35 @@ async def get_cached_response(redis_client, request): return None, None, None # print("Cache hit", len(cached)) - entrytime = int(cached[:10]) + entrytime = int(cached[:10].decode('utf-8')) currenttime = calendar.timegm(datetime.utcnow().utctimetuple()) age = currenttime-entrytime - headers_len = int(cached[10:16]) - headers = json.loads(cached[16:16+headers_len]) + headers_len = int(cached[10:16].decode('utf-8')) + headers = json.loads(cached[16:16+headers_len].decode('utf-8')) - data = cached[16+headers_len:] + data = brotli.decompress(cached[16+headers_len:]) return age, headers, data skip_headers = ["x-process-time", "age"] -async def response_to_cache(redis_client, request, headers, data, ex: int=60): +async def response_to_cache(redis_client, request, headers, data, ex: int): key=generate_key(request) - headers_to_keep={} - for k in headers.keys(): - if k not in skip_headers: - headers_to_keep[k] = headers[k] - headers_to_keep_json = json.dumps(headers_to_keep) + fixed_headers={} + for k in headers: + if not k in skip_headers: + fixed_headers[k] = headers[k] + headers_json = json.dumps(fixed_headers, ensure_ascii=False).encode('utf-8') - entrytime="%10d"%calendar.timegm(datetime.utcnow().utctimetuple()) - await redis_client.set(key, bytes(entrytime, 'utf-8')+bytes("%06d"%len(headers_to_keep_json), 'utf-8')+bytes(headers_to_keep_json, 'utf-8')+data, ex=ex) + entrytime=f"{calendar.timegm(datetime.utcnow().utctimetuple()):10d}".encode('utf-8') + await redis_client.set(key, entrytime+f"{len(headers_json):06d}".encode('utf-8')+headers_json+brotli.compress(data), ex=ex) def generate_key(request): - key = f"{request.url.path}?bytes({request['query_string']}, 'utf-8')" + if len(request['query_string'])==0: + key = f"{request.url.path}" + else: + key = f"{request.url.path}?bytes({request['query_string']}, 'utf-8')" return key class CachingMiddleware(BaseHTTPMiddleware): @@ -67,6 +72,7 @@ async def dispatch(self, request, call_next): if data: #Fix Age header headers["Age"] = "%1d"%(age) + headers["adaguc-cache"] = "hit" return Response(content=data, status_code=200, headers=headers, media_type=headers['content-type']) response: Response = await call_next(request) @@ -82,11 +88,14 @@ async def dispatch(self, request, call_next): break if ttl is None: return response - response_body = b"" + response_body_file = BytesIO() async for chunk in response.body_iterator: - response_body += chunk + response_body_file.write(chunk) tasks = BackgroundTasks() - tasks.add_task(response_to_cache, redis_client=self.redis, request=request, headers=response.headers, data=response_body, ex=ttl) + tasks.add_task(response_to_cache, redis_client=self.redis, request=request, headers=response.headers, data=response_body_file.getvalue(), ex=ttl) response.headers['age'] = "0" - return Response(content=response_body, status_code=200, headers=response.headers, media_type=response.media_type, background=tasks) + response.headers['adaguc-cache'] = "miss" + return Response(content=response_body_file.getvalue(), status_code=200, headers=response.headers, media_type=response.media_type, background=tasks) + + response.headers["adaguc-cache"] = "err" return response diff --git a/requirements.in b/requirements.in index b0da62d3..964f11ea 100644 --- a/requirements.in +++ b/requirements.in @@ -25,5 +25,4 @@ uvicorn~=0.23.2 covjson-pydantic~=0.2.0 geomet~=1.0 edr-pydantic~=0.2.0 -redis~=5.0 -CacheToolsUtils~=8.5 \ No newline at end of file +redis~=5.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e6642cfd..9dcdfa3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,8 +26,6 @@ cachetools==5.3.2 # via # -r requirements.in # cachetoolsutils -cachetoolsutils==8.5 - # via -r requirements.in certifi==2024.2.2 # via # httpcore From 67af8340b8dda305647bfd77e41db1749ca01ee3 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Tue, 5 Mar 2024 10:13:58 +0100 Subject: [PATCH 19/44] Added redis port --- Docker/docker-compose-generate-env.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Docker/docker-compose-generate-env.sh b/Docker/docker-compose-generate-env.sh index f4f99d17..007c5e0d 100644 --- a/Docker/docker-compose-generate-env.sh +++ b/Docker/docker-compose-generate-env.sh @@ -2,14 +2,15 @@ ADAGUC_PORT=443 +ADAGUC_REDIS_PORT=6379 ADAGUC_DATA_DIR=${HOME}/adaguc-docker/adaguc-data ADAGUC_AUTOWMS_DIR=${HOME}/adaguc-docker/adaguc-autowms ADAGUC_DATASET_DIR=${HOME}/adaguc-docker/adaguc-datasets -usage() { echo "Usage: $0 -p -e -a -d -f " 1>&2; exit 1; } +usage() { echo "Usage: $0 -p -e -a -d -f -a " 1>&2; exit 1; } -while getopts ":e:p:h:a:d:f:" o; do +while getopts ":e:p:h:a:d:f:r" o; do case "${o}" in e) EXTERNALADDRESS=${OPTARG} @@ -26,6 +27,9 @@ while getopts ":e:p:h:a:d:f:" o; do f) ADAGUC_DATA_DIR=${OPTARG} ;; + r) + REDIS_PORT=${OPTARG} + ;; h) usage ;; @@ -55,6 +59,7 @@ echo "ADAGUC_AUTOWMS_DIR=${ADAGUC_AUTOWMS_DIR=}" >> .env echo "ADAGUC_DATASET_DIR=${ADAGUC_DATASET_DIR}" >> .env echo "ADAGUC_PORT=${ADAGUC_PORT}" >> .env echo "EXTERNALADDRESS=${EXTERNALADDRESS}" >> .env +echo "REDIS_PORT=${REDIS_PORT} >>.env echo "############### env file ###############" cat .env echo "############### env file ###############" From 2a1d35579e5dba3b5fe6df4aa9687b9d271e46f3 Mon Sep 17 00:00:00 2001 From: Lukas Phaf Date: Tue, 5 Mar 2024 14:10:20 +0100 Subject: [PATCH 20/44] Redis client cleanup. --- Docker/docker-compose.yml | 9 ++++----- python/lib/adaguc/runAdaguc.py | 8 ++++---- .../routers/caching_middleware.py | 14 +++++++------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Docker/docker-compose.yml b/Docker/docker-compose.yml index 5f252e7a..2dd8ddf4 100755 --- a/Docker/docker-compose.yml +++ b/Docker/docker-compose.yml @@ -21,6 +21,7 @@ services: max-file: "10" adaguc-server: image: openearth/adaguc-server +# build: .. container_name: my-adaguc-server hostname: my-adaguc-server volumes: @@ -36,7 +37,7 @@ services: - "ADAGUC_AUTOWMS_DIR=/data/adaguc-autowms" - "ADAGUC_DATA_DIR=/data/adaguc-data" - "ADAGUC_DATASET_DIR=/data/adaguc-datasets" - - "ADAGUC_REDIS=redis://adaguc-redis:${REDIS_PORT}" + - "ADAGUC_REDIS=redis://adaguc-redis:6379" env_file: - .env restart: unless-stopped @@ -80,7 +81,7 @@ services: networks: - adaguc-network volumes: - - adaguc-server-compose-adagucdb:/adaguc/adagucdb + - adaguc-server-compose-adagucdb:/var/lib/postgresql/data environment: - "POSTGRES_USER=adaguc" - "POSTGRES_PASSWORD=adaguc" @@ -92,13 +93,11 @@ services: max-size: "200k" max-file: "10" adaguc-redis: - image: redis:alpine + image: redis:7 container_name: adaguc-redis hostname: adaguc-redis networks: - adaguc-network - expose: - - "${REDIS_PORT}" volumes: adaguc-server-compose-adagucdb: networks: diff --git a/python/lib/adaguc/runAdaguc.py b/python/lib/adaguc/runAdaguc.py index 6ea8399d..dc717884 100644 --- a/python/lib/adaguc/runAdaguc.py +++ b/python/lib/adaguc/runAdaguc.py @@ -9,7 +9,8 @@ import shutil import random import string -from redis import from_url +from redis import Redis # This can also be used to connect to a Redis cluster +# from redis.cluster import RedisCluster as Redis # Cluster client, for testing import re import calendar @@ -44,7 +45,7 @@ def __init__(self): if self.ADAGUC_REDIS.startswith("redis://") or self.ADAGUC_REDIS.startswith("rediss://"): self.redis_url = self.ADAGUC_REDIS self.use_cache = True - self.redis_client = from_url(self.redis_url) + self.redis_client = Redis.from_url(self.redis_url) print("USE CACHE") else: self.use_cache = False @@ -205,7 +206,7 @@ def runADAGUCServer( # Check cache for entry with keys of (url,adagucargs) if configured if self.cache_wanted(url): - cache_key = str((url, adagucargs)) + cache_key = str((url, adagucargs)).encode('utf-8') print(f"Checking cache for {cache_key}") age, headers, data = get_cached_response(self.redis_client, cache_key) @@ -299,7 +300,6 @@ def response_to_cache(redis_client, key, headers:str, data): def get_cached_response(redis_client, key): cached = redis_client.get(key) - redis_client.close() if not cached: print("Cache miss") return None, None, None diff --git a/python/python_fastapi_server/routers/caching_middleware.py b/python/python_fastapi_server/routers/caching_middleware.py index ae76243c..3fcda841 100644 --- a/python/python_fastapi_server/routers/caching_middleware.py +++ b/python/python_fastapi_server/routers/caching_middleware.py @@ -8,7 +8,9 @@ import calendar from datetime import datetime -import redis.asyncio as redis +from redis.asyncio import Redis # This can also be used to connect to a Redis cluster +# from redis.asyncio.cluster import RedisCluster as Redis # Cluster client, for testing + import json import brotli @@ -17,7 +19,6 @@ async def get_cached_response(redis_client, request): key = generate_key(request) cached = await redis_client.get(key) - # await redis_client.close() if not cached: # print("Cache miss") return None, None, None @@ -48,10 +49,9 @@ async def response_to_cache(redis_client, request, headers, data, ex: int): await redis_client.set(key, entrytime+f"{len(headers_json):06d}".encode('utf-8')+headers_json+brotli.compress(data), ex=ex) def generate_key(request): - if len(request['query_string'])==0: - key = f"{request.url.path}" - else: - key = f"{request.url.path}?bytes({request['query_string']}, 'utf-8')" + key = request.url.path.encode('utf-8') + if len(request['query_string']) > 0: + key += b"?" + request['query_string'] return key class CachingMiddleware(BaseHTTPMiddleware): @@ -60,7 +60,7 @@ def __init__(self, app): super().__init__(app) if "ADAGUC_REDIS" in os.environ: self.shortcut=False - self.redis = redis.from_url(ADAGUC_REDIS) + self.redis = Redis.from_url(ADAGUC_REDIS) async def dispatch(self, request, call_next): if self.shortcut: From d5bfd356cc579d16d8994a55a7ddade8f38e18c2 Mon Sep 17 00:00:00 2001 From: Lukas Phaf Date: Tue, 5 Mar 2024 16:15:32 +0100 Subject: [PATCH 21/44] Use redis connection pool. --- python/lib/adaguc/runAdaguc.py | 16 ++++++++++------ .../routers/caching_middleware.py | 19 ++++++++++++------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/python/lib/adaguc/runAdaguc.py b/python/lib/adaguc/runAdaguc.py index dc717884..c7d2f4ba 100644 --- a/python/lib/adaguc/runAdaguc.py +++ b/python/lib/adaguc/runAdaguc.py @@ -9,7 +9,7 @@ import shutil import random import string -from redis import Redis # This can also be used to connect to a Redis cluster +import redis # This can also be used to connect to a Redis cluster # from redis.cluster import RedisCluster as Redis # Cluster client, for testing import re @@ -45,7 +45,7 @@ def __init__(self): if self.ADAGUC_REDIS.startswith("redis://") or self.ADAGUC_REDIS.startswith("rediss://"): self.redis_url = self.ADAGUC_REDIS self.use_cache = True - self.redis_client = Redis.from_url(self.redis_url) + self.redis_pool = redis.ConnectionPool.from_url(self.ADAGUC_REDIS) print("USE CACHE") else: self.use_cache = False @@ -209,7 +209,7 @@ def runADAGUCServer( cache_key = str((url, adagucargs)).encode('utf-8') print(f"Checking cache for {cache_key}") - age, headers, data = get_cached_response(self.redis_client, cache_key) + age, headers, data = get_cached_response(self.redis_pool, cache_key) if age is not None: return [0, data, headers] @@ -250,7 +250,7 @@ def runADAGUCServer( else: if self.cache_wanted(url): print(f"CACHING {cache_key}") - response_to_cache(self.redis_client, cache_key, headers, filetogenerate) + response_to_cache(self.redis_pool, cache_key, headers, filetogenerate) print("HEADERS:", headers) return [status, filetogenerate, headers] @@ -277,7 +277,7 @@ def mkdir_p(self, directory): os.makedirs(directory) skip_headers = ["x-process-time", "age"] -def response_to_cache(redis_client, key, headers:str, data): +def response_to_cache(redis_pool, key, headers:str, data): cacheable_headers=[] ttl = 0 for header in headers: @@ -296,10 +296,14 @@ def response_to_cache(redis_client, key, headers:str, data): cacheable_headers_json=json.dumps(cacheable_headers, ensure_ascii=True).encode('utf-8') entrytime=f"{calendar.timegm(datetime.utcnow().utctimetuple()):10d}".encode('utf-8') + redis_client = redis.Redis(connection_pool=redis_pool) redis_client.set(key, entrytime+f"{len(cacheable_headers_json):06d}".encode('utf-8')+cacheable_headers_json+data.getvalue(), ex=ttl) + redis_client.close() -def get_cached_response(redis_client, key): +def get_cached_response(redis_pool, key): + redis_client = redis.Redis(connection_pool=redis_pool) cached = redis_client.get(key) + redis_client.close() if not cached: print("Cache miss") return None, None, None diff --git a/python/python_fastapi_server/routers/caching_middleware.py b/python/python_fastapi_server/routers/caching_middleware.py index 3fcda841..9e1d2e18 100644 --- a/python/python_fastapi_server/routers/caching_middleware.py +++ b/python/python_fastapi_server/routers/caching_middleware.py @@ -8,7 +8,7 @@ import calendar from datetime import datetime -from redis.asyncio import Redis # This can also be used to connect to a Redis cluster +import redis.asyncio as redis # This can also be used to connect to a Redis cluster # from redis.asyncio.cluster import RedisCluster as Redis # Cluster client, for testing import json @@ -16,9 +16,11 @@ ADAGUC_REDIS=os.environ.get('ADAGUC_REDIS') -async def get_cached_response(redis_client, request): +async def get_cached_response(redis_pool, request): key = generate_key(request) + redis_client = redis.Redis(connection_pool=redis_pool) cached = await redis_client.get(key) + await redis_client.aclose() if not cached: # print("Cache miss") return None, None, None @@ -36,7 +38,7 @@ async def get_cached_response(redis_client, request): skip_headers = ["x-process-time", "age"] -async def response_to_cache(redis_client, request, headers, data, ex: int): +async def response_to_cache(redis_pool, request, headers, data, ex: int): key=generate_key(request) fixed_headers={} @@ -46,7 +48,10 @@ async def response_to_cache(redis_client, request, headers, data, ex: int): headers_json = json.dumps(fixed_headers, ensure_ascii=False).encode('utf-8') entrytime=f"{calendar.timegm(datetime.utcnow().utctimetuple()):10d}".encode('utf-8') + redis_client = redis.Redis(connection_pool=redis_pool) await redis_client.set(key, entrytime+f"{len(headers_json):06d}".encode('utf-8')+headers_json+brotli.compress(data), ex=ex) + await redis_client.aclose() + def generate_key(request): key = request.url.path.encode('utf-8') @@ -59,15 +64,15 @@ class CachingMiddleware(BaseHTTPMiddleware): def __init__(self, app): super().__init__(app) if "ADAGUC_REDIS" in os.environ: - self.shortcut=False - self.redis = Redis.from_url(ADAGUC_REDIS) + self.shortcut = False + self.redis_pool = redis.ConnectionPool.from_url(ADAGUC_REDIS) async def dispatch(self, request, call_next): if self.shortcut: return await call_next(request) #Check if request is in cache, if so return that - age, headers, data = await get_cached_response(self.redis, request) + age, headers, data = await get_cached_response(self.redis_pool, request) if data: #Fix Age header @@ -92,7 +97,7 @@ async def dispatch(self, request, call_next): async for chunk in response.body_iterator: response_body_file.write(chunk) tasks = BackgroundTasks() - tasks.add_task(response_to_cache, redis_client=self.redis, request=request, headers=response.headers, data=response_body_file.getvalue(), ex=ttl) + tasks.add_task(response_to_cache, redis_pool=self.redis_pool, request=request, headers=response.headers, data=response_body_file.getvalue(), ex=ttl) response.headers['age'] = "0" response.headers['adaguc-cache'] = "miss" return Response(content=response_body_file.getvalue(), status_code=200, headers=response.headers, media_type=response.media_type, background=tasks) From 793ff6ce6d771878b456c75eda5c2c2176dc391d Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Thu, 7 Mar 2024 10:58:12 +0100 Subject: [PATCH 22/44] Removed old caching from OGCAPI-Features --- .../python_fastapi_server/routers/ogcapi.py | 2 - .../routers/ogcapi_tools.py | 38 +++++++------------ 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/python/python_fastapi_server/routers/ogcapi.py b/python/python_fastapi_server/routers/ogcapi.py index 102071fc..3f782349 100644 --- a/python/python_fastapi_server/routers/ogcapi.py +++ b/python/python_fastapi_server/routers/ogcapi.py @@ -8,7 +8,6 @@ from typing import Dict, List, Type, Union import yaml -from cachetools import TTLCache from defusedxml.ElementTree import fromstring from fastapi import Depends, FastAPI, HTTPException, Query, Request, Response from fastapi import status as fastapi_status @@ -30,7 +29,6 @@ logger = logging.getLogger(__name__) -cache = TTLCache(maxsize=1000, ttl=30) DEFAULT_CRS = "http://www.opengis.net/def/crs/OGC/1.3/CRS84" SUPPORTED_CRS_LIST = [DEFAULT_CRS] diff --git a/python/python_fastapi_server/routers/ogcapi_tools.py b/python/python_fastapi_server/routers/ogcapi_tools.py index 03abf359..1bd0545a 100644 --- a/python/python_fastapi_server/routers/ogcapi_tools.py +++ b/python/python_fastapi_server/routers/ogcapi_tools.py @@ -4,7 +4,6 @@ import time from typing import List -from cachetools import TTLCache, cached from defusedxml.ElementTree import ParseError, parse from owslib.wms import WebMapService @@ -15,8 +14,6 @@ logger = logging.getLogger(__name__) -cache = TTLCache(maxsize=1000, ttl=60) - def make_bbox(extent): s_extent = [] for i in extent: @@ -119,35 +116,29 @@ def call_adaguc(url): return status, data, headers -@cached(cache=cache) def get_capabilities(collname): """ Get the collectioninfo from the WMS GetCapabilities """ coll = generate_collections().get(collname) - if "dataset" in coll: - logger.info("callADAGUC by dataset") - dataset = coll["dataset"] - urlrequest = ( - f"dataset={dataset}&service=wms&version=1.3.0&request=getcapabilities" - ) - status, response, headers = call_adaguc(url=urlrequest.encode("UTF-8")) - for hdr in headers: - if hdr.lower().startswith("cache-control"): - logger.info("%s", hdr) - if status == 0: - xml = response.getvalue() - wms = WebMapService(coll["service"], xml=xml, version="1.3.0") - else: - logger.error("status: %d", status) - return {} + logger.info("callADAGUC by dataset") + dataset = coll["dataset"] + urlrequest = ( + f"dataset={dataset}&service=wms&version=1.3.0&request=getcapabilities" + ) + status, response, headers = call_adaguc(url=urlrequest.encode("UTF-8")) + for hdr in headers: + if hdr.lower().startswith("cache-control"): + logger.info("%s", hdr) + if status == 0: + xml = response.getvalue() + wms = WebMapService(coll["service"], xml=xml, version="1.3.0") else: - logger.info("callADAGUC by service %s", coll) - wms = WebMapService(coll["service"], version="1.3.0") + logger.error("status: %d", status) + return {} return wms.contents -@cached(cache=cache) def generate_collections(): """ Generate OGC API Feature collections @@ -174,7 +165,6 @@ def get_dimensions(layer, skip_dims=None): return dims -@cached(cache=cache) def get_parameters(collname): """ get_parameters From d6c4e79fceaa27ee3cbcd4bcbead36f34a1781e0 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Thu, 7 Mar 2024 12:36:45 +0100 Subject: [PATCH 23/44] Cleanup of get_capabilties call --- python/python_fastapi_server/routers/edr.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index 12d21a4c..9e9a8075 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -403,7 +403,7 @@ async def get_collectioninfo_for_id( output_formats=output_formats, ) - get_cap = get_capabilities(edr_collection) + get_cap = await get_capabilities(edr_collection) return collection, get_cap["expires"] @@ -504,7 +504,8 @@ async def get_times_for_collection( It does this for given parameter. When the parameter is not given it will do it for the first Layer in the GetCapabilities document. """ logger.info("get_times_for_dataset(%s,%s)", edr_collectioninfo["name"], parameter) - wms = await get_capabilities(edr_collectioninfo["name"])["layers"] + getcap = await get_capabilities(edr_collectioninfo["name"]) + wms = getcap["layers"] if parameter and parameter in wms: layer = wms[parameter] else: @@ -543,7 +544,8 @@ async def get_custom_dims_for_collection(edr_collectioninfo: dict, parameter: st """ Return the dimensions other then elevation or time from the WMS GetCapabilities document. """ - wms = await get_capabilities(edr_collectioninfo["name"])["layers"] + getcap = await get_capabilities(edr_collectioninfo["name"]) + wms = getcap["layers"] custom = [] if parameter and parameter in list(wms): layer = wms[parameter] @@ -572,7 +574,8 @@ async def get_vertical_dim_for_collection(edr_collectioninfo: dict, parameter: s """ Return the verticel dimension the WMS GetCapabilities document. """ - wms = await get_capabilities(edr_collectioninfo["name"])["layers"] + getcap = await get_capabilities(edr_collectioninfo["name"]) + wms = getcap["layers"] if parameter and parameter in list(wms): layer = wms[parameter] else: @@ -705,7 +708,8 @@ async def get_extent(edr_collectioninfo: dict): """ Get the boundingbox extent from the WMS GetCapabilities """ - contents = await get_capabilities(edr_collectioninfo["name"])["layers"] + getcap = await get_capabilities(edr_collectioninfo["name"]) + contents = getcap["layers"] first_layer = edr_collectioninfo["parameters"][0]["name"] if len(contents): if first_layer in contents: From 46ddd69c1c4b7a1e65c6957afa089a0c26176113 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Thu, 7 Mar 2024 14:25:08 +0100 Subject: [PATCH 24/44] Fixed tests after merge --- .../routers/ogcapi_tools.py | 2 +- .../python_fastapi_server/routers/wmswcs.py | 4 +- .../python_fastapi_server/test_ogc_api_edr.py | 24 ++++++------ .../test_ogc_api_features.py | 39 ++++++++++--------- 4 files changed, 36 insertions(+), 33 deletions(-) diff --git a/python/python_fastapi_server/routers/ogcapi_tools.py b/python/python_fastapi_server/routers/ogcapi_tools.py index c5602580..eb5cfd7b 100644 --- a/python/python_fastapi_server/routers/ogcapi_tools.py +++ b/python/python_fastapi_server/routers/ogcapi_tools.py @@ -128,7 +128,7 @@ async def get_capabilities(collname): urlrequest = ( f"dataset={dataset}&service=wms&version=1.3.0&request=getcapabilities" ) - status, response = await call_adaguc(url=urlrequest.encode("UTF-8")) + status, response, _headers = await call_adaguc(url=urlrequest.encode("UTF-8")) if status == 0: xml = response.getvalue() wms = WebMapService(coll["service"], xml=xml, version="1.3.0") diff --git a/python/python_fastapi_server/routers/wmswcs.py b/python/python_fastapi_server/routers/wmswcs.py index a4c3cfc4..120c3d6f 100644 --- a/python/python_fastapi_server/routers/wmswcs.py +++ b/python/python_fastapi_server/routers/wmswcs.py @@ -85,7 +85,7 @@ async def handle_wms( return response -async def testadaguc(): +def testadaguc(): """Test adaguc is setup correctly""" logger.info("Checking adaguc-server.") adaguc_instance = setup_adaguc() @@ -103,7 +103,7 @@ async def testadaguc(): # Run adaguc-server # pylint: disable=unused-variable - status, _data, headers = await asyncio.run( + status, _data, headers = asyncio.run( adaguc_instance.runADAGUCServer(url, env=adagucenv, showLogOnError=False) ) assert status == 0 diff --git a/python/python_fastapi_server/test_ogc_api_edr.py b/python/python_fastapi_server/test_ogc_api_edr.py index 6370d627..3db112a2 100644 --- a/python/python_fastapi_server/test_ogc_api_edr.py +++ b/python/python_fastapi_server/test_ogc_api_edr.py @@ -3,11 +3,8 @@ import os import pytest -import pytest_asyncio from adaguc.AdagucTestTools import AdagucTestTools from fastapi.testclient import TestClient -from httpx import AsyncClient -import asyncio from main import app @@ -21,6 +18,7 @@ def set_environ(): def setup_test_data(): + print("About to ingest data") AdagucTestTools().cleanTempDir() for service in ["netcdf_5d.xml", "dataset_a.xml"]: _status, _data, _headers = AdagucTestTools().runADAGUCServer( @@ -34,23 +32,25 @@ def setup_test_data(): showLog=True, ) + @pytest.fixture(name="client") -def fixture_client() -> TestClient: +def fixture_client(): # Initialize adaguc-server set_environ() setup_test_data() yield TestClient(app) -@pytest.mark.asyncio -async def test_root(client): + +def test_root(client: TestClient): resp = client.get("/edr/") root_info = resp.json() print("resp:", resp, json.dumps(root_info, indent=2)) + print() assert root_info["description"] == "EDR service for ADAGUC datasets" assert len(root_info["links"]) >= 4 -@pytest.mark.asyncio -async def test_collections(client): + +def test_collections(client: TestClient): resp = client.get("/edr/collections") colls = resp.json() assert len(colls["collections"]) == 1 @@ -74,7 +74,9 @@ async def test_collections(client): assert "position" in coll_5d["data_queries"] -@pytest.mark.asyncio -async def test_coll_5d_position(client): - resp = client.get("/edr/collections/data_5d/position?coords=POINT(5.2 50.0)¶meter-name=data") + +def test_coll_5d_position(client: TestClient): + resp = client.get( + "/edr/collections/data_5d/position?coords=POINT(5.2 50.0)¶meter-name=data" + ) print(resp.json()) \ No newline at end of file diff --git a/python/python_fastapi_server/test_ogc_api_features.py b/python/python_fastapi_server/test_ogc_api_features.py index 49f0d9ce..84b81611 100644 --- a/python/python_fastapi_server/test_ogc_api_features.py +++ b/python/python_fastapi_server/test_ogc_api_features.py @@ -1,13 +1,11 @@ import json import logging import os -from httpx import AsyncClient import pytest from adaguc.AdagucTestTools import AdagucTestTools from fastapi.testclient import TestClient -import pytest_asyncio from main import app logger = logging.getLogger(__name__) @@ -20,6 +18,7 @@ def set_environ(): def setup_test_data(): + print("About to ingest data") AdagucTestTools().cleanTempDir() for service in ["netcdf_5d.xml", "dataset_a.xml"]: _status, _data, _headers = AdagucTestTools().runADAGUCServer( @@ -34,25 +33,27 @@ def setup_test_data(): ) -@pytest_asyncio.fixture(name="clientdata") -async def clientdata(): +@pytest.fixture(name="client") +def fixture_client(): + # Initialize adaguc-server set_environ() setup_test_data() + yield TestClient(app) -@pytest.mark.asyncio() -async def test_root(clientdata): - async with AsyncClient(app=app, base_url="http://test") as client: - resp = await client.get( - "/adaguc-server?dataset=netcdf_5d&request=getcapabilities&service=wms&version=1.3.0" - ) - resp = await client.get("/ogcapi/") - assert resp.json()["description"] == "ADAGUC OGCAPI-Features server" +def test_root(client: TestClient): + resp = client.get( + "/adaguc-server?dataset=netcdf_5d&request=getcapabilities&service=wms&version=1.3.0" + ) + print("getcap:", resp.text) + + resp = client.get("/ogcapi/") + print("resp:", resp, resp.json()) + assert resp.json()["description"] == "ADAGUC OGCAPI-Features server" + -@pytest.mark.asyncio() -async def test_collections(clientdata): - async with AsyncClient(app=app, base_url="http://test") as client: - resp = await client.get("/ogcapi/collections") - colls = resp.json() - print(json.dumps(colls["collections"][1], indent=2)) - assert len(colls["collections"]) == 2 +def test_collections(client: TestClient): + resp = client.get("/ogcapi/collections") + colls = resp.json() + print(json.dumps(colls["collections"][1], indent=2)) + assert len(colls["collections"]) == 2 \ No newline at end of file From c36a00af54fc536b3b1a6aee9d6dbd6352a233f6 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Thu, 7 Mar 2024 16:14:32 +0100 Subject: [PATCH 25/44] Fixes from review; less GetCapabiltites calls --- python/lib/adaguc/runAdaguc.py | 114 ++++--- .../routers/caching_middleware.py | 101 ++++-- python/python_fastapi_server/routers/edr.py | 156 +++++---- .../python_fastapi_server/routers/ogcapi.py | 301 +++++++++--------- 4 files changed, 374 insertions(+), 298 deletions(-) diff --git a/python/lib/adaguc/runAdaguc.py b/python/lib/adaguc/runAdaguc.py index 2e45bff0..f46780b3 100644 --- a/python/lib/adaguc/runAdaguc.py +++ b/python/lib/adaguc/runAdaguc.py @@ -11,6 +11,7 @@ import random import string import redis # This can also be used to connect to a Redis cluster + # from redis.cluster import RedisCluster as Redis # Cluster client, for testing import re @@ -22,6 +23,13 @@ class runAdaguc: + ADAGUC_REDIS = os.getenv("ADAGUC_REDIS", "") + use_cache = False + redis_pool = None + + if ADAGUC_REDIS.startswith("redis://") or ADAGUC_REDIS.startswith("rediss://"): + redis_pool = redis.ConnectionPool.from_url(ADAGUC_REDIS) + use_cache = True def __init__(self): """ADAGUC_LOGFILE is the location where logfiles are stored. @@ -29,27 +37,22 @@ def __init__(self): Please note regenerating the DB each time for each request can cause performance problems. You can safely configure a permanent location for the database which is permanent in adaguc.autoresource.xml (or your own config) """ - self.ADAGUC_LOGFILE = ("/tmp/adaguc-server-" + - self.get_random_string(10) + ".log") + self.ADAGUC_LOGFILE = ( + "/tmp/adaguc-server-" + self.get_random_string(10) + ".log" + ) self.ADAGUC_PATH = os.getenv("ADAGUC_PATH", "./") self.ADAGUC_CONFIG = self.ADAGUC_PATH + "/data/config/adaguc.autoresource.xml" - self.ADAGUC_DATA_DIR = os.getenv("ADAGUC_DATA_DIR", - "/data/adaguc-data") - self.ADAGUC_AUTOWMS_DIR = os.getenv("ADAGUC_AUTOWMS_DIR", - "/data/adaguc-autowms") - self.ADAGUC_DATASET_DIR = os.getenv("ADAGUC_DATASET_DIR", - "/data/adaguc-datasets") + self.ADAGUC_DATA_DIR = os.getenv("ADAGUC_DATA_DIR", "/data/adaguc-data") + self.ADAGUC_AUTOWMS_DIR = os.getenv( + "ADAGUC_AUTOWMS_DIR", "/data/adaguc-autowms" + ) + self.ADAGUC_DATASET_DIR = os.getenv( + "ADAGUC_DATASET_DIR", "/data/adaguc-datasets" + ) self.ADAGUC_TMP = os.getenv("ADAGUC_TMP", "/tmp") self.ADAGUC_FONT = os.getenv( - "ADAGUC_FONT", self.ADAGUC_PATH + "/data/fonts/Roboto-Medium.ttf") - self.ADAGUC_REDIS = os.getenv("ADAGUC_REDIS", "") - if self.ADAGUC_REDIS.startswith("redis://") or self.ADAGUC_REDIS.startswith("rediss://"): - self.redis_url = self.ADAGUC_REDIS - self.use_cache = True - self.redis_pool = redis.ConnectionPool.from_url(self.ADAGUC_REDIS) - print("USE CACHE") - else: - self.use_cache = False + "ADAGUC_FONT", self.ADAGUC_PATH + "/data/fonts/Roboto-Medium.ttf" + ) def setAdagucPath(self, newAdagucPath): self.ADAGUC_PATH = newAdagucPath @@ -90,10 +93,11 @@ def scanDataset(self, datasetName): adagucenv["ADAGUC_TMP"] = self.ADAGUC_TMP adagucenv["ADAGUC_FONT"] = self.ADAGUC_FONT - status, data, headers = asyncio.run(self.runADAGUCServer( - args=["--updatedb", "--config", config], - env=adagucenv, - isCGI=False)) + status, data, headers = asyncio.run( + self.runADAGUCServer( + args=["--updatedb", "--config", config], env=adagucenv, isCGI=False + ) + ) return data.getvalue().decode() @@ -108,9 +112,9 @@ def runGetMapUrl(self, url): adagucenv["ADAGUC_DATASET_DIR"] = self.ADAGUC_DATASET_DIR adagucenv["ADAGUC_TMP"] = self.ADAGUC_TMP adagucenv["ADAGUC_FONT"] = self.ADAGUC_FONT - status, data, headers = asyncio.run(self.runADAGUCServer(url, - env=adagucenv, - showLogOnError=False)) + status, data, headers = asyncio.run( + self.runADAGUCServer(url, env=adagucenv, showLogOnError=False) + ) logfile = self.getLogFile() self.removeLogFile() if data is not None: @@ -146,7 +150,7 @@ def printLogFile(self): print("=== END ADAGUC LOGS ===") def cache_wanted(self, url: str): - if not self.use_cache: + if not runAdaguc.use_cache: return False if "request=getcapabilities" in url.lower(): return True @@ -170,7 +174,8 @@ async def runADAGUCServer( adagucenv = env adagucenv["ADAGUC_ENABLELOGBUFFER"] = os.getenv( - "ADAGUC_ENABLELOGBUFFER", "TRUE") + "ADAGUC_ENABLELOGBUFFER", "TRUE" + ) adagucenv["ADAGUC_CONFIG"] = self.ADAGUC_CONFIG adagucenv["ADAGUC_LOGFILE"] = self.ADAGUC_LOGFILE adagucenv["ADAGUC_PATH"] = self.ADAGUC_PATH @@ -187,7 +192,7 @@ async def runADAGUCServer( # Forward all environment variables starting with ADAGUCENV_ prefix: str = "ADAGUCENV_" for key, value in os.environ.items(): - if key[:len(prefix)] == prefix: + if key[: len(prefix)] == prefix: adagucenv[key] = value ADAGUC_PATH = adagucenv["ADAGUC_PATH"] @@ -207,14 +212,11 @@ async def runADAGUCServer( # Check cache for entry with keys of (url,adagucargs) if configured if self.cache_wanted(url): - cache_key = str((url, adagucargs)).encode('utf-8') - print(f"Checking cache for {cache_key}") + cache_key = str((url, adagucargs)).encode("utf-8") age, headers, data = get_cached_response(self.redis_pool, cache_key) if age is not None: - return [0, data, headers] - - print(f"Generating {cache_key}") + return [0, data, headers] filetogenerate = BytesIO() status, headers, processErr = await CGIRunner().run( @@ -250,9 +252,7 @@ async def runADAGUCServer( else: if self.cache_wanted(url): - print(f"CACHING {cache_key}") response_to_cache(self.redis_pool, cache_key, headers, filetogenerate) - print("HEADERS:", headers) return [status, filetogenerate, headers] def writetofile(self, filename, data): @@ -277,47 +277,59 @@ def mkdir_p(self, directory): if not os.path.exists(directory): os.makedirs(directory) + skip_headers = ["x-process-time", "age"] -def response_to_cache(redis_pool, key, headers:str, data): - cacheable_headers=[] + + +def response_to_cache(redis_pool, key, headers: str, data): + cacheable_headers = [] ttl = 0 for header in headers: - k,v = header.split(":") + k, v = header.split(":") if k not in skip_headers: cacheable_headers.append(header) if k.lower().startswith("cache-control"): for term in v.split(";"): if term.startswith("max-age"): try: - ttl = int(term.split('=')[1]) + ttl = int(term.split("=")[1]) except: pass - if ttl>0: - cacheable_headers_json=json.dumps(cacheable_headers, ensure_ascii=True).encode('utf-8') + if ttl > 0: + cacheable_headers_json = json.dumps( + cacheable_headers, ensure_ascii=True + ).encode("utf-8") - entrytime=f"{calendar.timegm(datetime.utcnow().utctimetuple()):10d}".encode('utf-8') + entrytime = f"{calendar.timegm(datetime.utcnow().utctimetuple()):10d}".encode( + "utf-8" + ) redis_client = redis.Redis(connection_pool=redis_pool) - redis_client.set(key, entrytime+f"{len(cacheable_headers_json):06d}".encode('utf-8')+cacheable_headers_json+data.getvalue(), ex=ttl) + redis_client.set( + key, + entrytime + + f"{len(cacheable_headers_json):06d}".encode("utf-8") + + cacheable_headers_json + + data.getvalue(), + ex=ttl, + ) redis_client.close() + def get_cached_response(redis_pool, key): redis_client = redis.Redis(connection_pool=redis_pool) cached = redis_client.get(key) redis_client.close() if not cached: - print("Cache miss") return None, None, None - print("Cache hit", len(cached)) - entrytime=int(cached[:10].decode('utf-8')) - currenttime=calendar.timegm(datetime.utcnow().utctimetuple()) - age=currenttime-entrytime + entrytime = int(cached[:10].decode("utf-8")) + currenttime = calendar.timegm(datetime.utcnow().utctimetuple()) + age = currenttime - entrytime - headers_len=int(cached[10:16].decode('utf-8')) - headers=json.loads(cached[16:16+headers_len]) + headers_len = int(cached[10:16].decode("utf-8")) + headers = json.loads(cached[16 : 16 + headers_len]) headers.append(f"age: {age}") - data = cached[16+headers_len:] - print(f"HIT: {len(data)} bytes cached") + data = cached[16 + headers_len :] return age, headers, BytesIO(data) diff --git a/python/python_fastapi_server/routers/caching_middleware.py b/python/python_fastapi_server/routers/caching_middleware.py index 9e1d2e18..14af67f8 100644 --- a/python/python_fastapi_server/routers/caching_middleware.py +++ b/python/python_fastapi_server/routers/caching_middleware.py @@ -9,12 +9,14 @@ import calendar from datetime import datetime import redis.asyncio as redis # This can also be used to connect to a Redis cluster + # from redis.asyncio.cluster import RedisCluster as Redis # Cluster client, for testing import json import brotli -ADAGUC_REDIS=os.environ.get('ADAGUC_REDIS') +ADAGUC_REDIS = os.environ.get("ADAGUC_REDIS") + async def get_cached_response(redis_pool, request): key = generate_key(request) @@ -22,45 +24,56 @@ async def get_cached_response(redis_pool, request): cached = await redis_client.get(key) await redis_client.aclose() if not cached: - # print("Cache miss") return None, None, None - # print("Cache hit", len(cached)) - entrytime = int(cached[:10].decode('utf-8')) + entrytime = int(cached[:10].decode("utf-8")) currenttime = calendar.timegm(datetime.utcnow().utctimetuple()) - age = currenttime-entrytime + age = currenttime - entrytime - headers_len = int(cached[10:16].decode('utf-8')) - headers = json.loads(cached[16:16+headers_len].decode('utf-8')) + headers_len = int(cached[10:16].decode("utf-8")) + headers = json.loads(cached[16 : 16 + headers_len].decode("utf-8")) - data = brotli.decompress(cached[16+headers_len:]) + data = brotli.decompress(cached[16 + headers_len :]) return age, headers, data + skip_headers = ["x-process-time", "age"] + async def response_to_cache(redis_pool, request, headers, data, ex: int): - key=generate_key(request) + key = generate_key(request) - fixed_headers={} + fixed_headers = {} for k in headers: - if not k in skip_headers: - fixed_headers[k] = headers[k] - headers_json = json.dumps(fixed_headers, ensure_ascii=False).encode('utf-8') + if not k in skip_headers: + fixed_headers[k] = headers[k] + headers_json = json.dumps(fixed_headers, ensure_ascii=False).encode("utf-8") - entrytime=f"{calendar.timegm(datetime.utcnow().utctimetuple()):10d}".encode('utf-8') + entrytime = f"{calendar.timegm(datetime.utcnow().utctimetuple()):10d}".encode( + "utf-8" + ) redis_client = redis.Redis(connection_pool=redis_pool) - await redis_client.set(key, entrytime+f"{len(headers_json):06d}".encode('utf-8')+headers_json+brotli.compress(data), ex=ex) + await redis_client.set( + key, + entrytime + + f"{len(headers_json):06d}".encode("utf-8") + + headers_json + + brotli.compress(data), + ex=ex, + ) await redis_client.aclose() def generate_key(request): - key = request.url.path.encode('utf-8') - if len(request['query_string']) > 0: - key += b"?" + request['query_string'] + key = request.url.path.encode("utf-8") + if len(request["query_string"]) > 0: + key += b"?" + request["query_string"] return key + class CachingMiddleware(BaseHTTPMiddleware): shortcut = True + def __init__(self, app): super().__init__(app) if "ADAGUC_REDIS" in os.environ: @@ -71,36 +84,60 @@ async def dispatch(self, request, call_next): if self.shortcut: return await call_next(request) - #Check if request is in cache, if so return that + # Check if request is in cache, if so return that age, headers, data = await get_cached_response(self.redis_pool, request) if data: - #Fix Age header - headers["Age"] = "%1d"%(age) + # Fix Age header + headers["Age"] = "%1d" % (age) headers["adaguc-cache"] = "hit" - return Response(content=data, status_code=200, headers=headers, media_type=headers['content-type']) + return Response( + content=data, + status_code=200, + headers=headers, + media_type=headers["content-type"], + ) response: Response = await call_next(request) if response.status_code == 200: - if "cache-control" in response.headers and response.headers['cache-control']!="no-store": - cache_control_terms = response.headers['cache-control'].split(",") + if ( + "cache-control" in response.headers + and response.headers["cache-control"] != "no-store" + ): + cache_control_terms = response.headers["cache-control"].split(",") ttl = None for term in cache_control_terms: age_terms = term.split("=") - if age_terms[0].lower()=="max-age": + if age_terms[0].lower() == "max-age": ttl = int(age_terms[1]) break if ttl is None: return response - response_body_file = BytesIO() - async for chunk in response.body_iterator: - response_body_file.write(chunk) + + with BytesIO() as response_body_file: + async for chunk in response.body_iterator: + response_body_file.write(chunk) + response_body = response_body_file.getvalue() + tasks = BackgroundTasks() - tasks.add_task(response_to_cache, redis_pool=self.redis_pool, request=request, headers=response.headers, data=response_body_file.getvalue(), ex=ttl) - response.headers['age'] = "0" - response.headers['adaguc-cache'] = "miss" - return Response(content=response_body_file.getvalue(), status_code=200, headers=response.headers, media_type=response.media_type, background=tasks) + tasks.add_task( + response_to_cache, + redis_pool=self.redis_pool, + request=request, + headers=response.headers, + data=response_body, + ex=ttl, + ) + response.headers["age"] = "0" + response.headers["adaguc-cache"] = "miss" + return Response( + content=response_body, + status_code=200, + headers=response.headers, + media_type=response.media_type, + background=tasks, + ) response.headers["adaguc-cache"] = "err" return response diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index 9e9a8075..2c7ab9df 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -7,6 +7,7 @@ KNMI """ + import itertools import json import logging @@ -58,8 +59,6 @@ from .covjsonresponse import CovJSONResponse from .ogcapi_tools import call_adaguc -SHORT_CACHE_TIME=60 - logger = logging.getLogger(__name__) logger.debug("Starting EDR") @@ -101,6 +100,7 @@ async def edr_exception_handler(_, exc: EdrException): content={"code": str(exc.code), "description": exc.description}, ) + def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DIR"]): """ Return all possible OGCAPI EDR datasets, based on the dataset directory @@ -160,7 +160,10 @@ def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DI pass return edr_collections + edr_collections = None + + def get_edr_collections(): """Returns all EDR collections""" global edr_collections @@ -229,7 +232,6 @@ async def get_collection_position( datetime_par: str = Query(default=None, alias="datetime"), parameter_name: str = Query(alias="parameter-name"), z_par: str = Query(alias="z", default=None), - ) -> Coverage: """ returns data for the EDR /position endpoint @@ -259,8 +261,8 @@ async def get_collection_position( dat = json.loads(resp) ttl = get_ttl_from_adaguc_call(headers) if ttl is not None: - expires = (datetime.utcnow()+timedelta(seconds=ttl)).timestamp() - response.headers["cache-control"]=generate_max_age(expires) + expires = (datetime.utcnow() + timedelta(seconds=ttl)).timestamp() + response.headers["cache-control"] = generate_max_age(expires) return covjson_from_resp(dat, edr_collections[collection_name]["vertical_name"]) raise EdrException(code=400, description="No data") @@ -306,17 +308,20 @@ async def get_collectioninfo_for_id( ) links.append(instances_link) - bbox = await get_extent(edr_collectioninfo) + wmslayers, expires = await get_capabilities(edr_collectioninfo["name"]) + + bbox = await get_extent(edr_collectioninfo, wmslayers) if bbox is None: return None, None crs = 'GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]' spatial = Spatial(bbox=bbox, crs=crs) + (interval, time_values) = await get_times_for_collection( - edr_collectioninfo, edr_collectioninfo["parameters"][0]["name"] + edr_collectioninfo, wmslayers, edr_collectioninfo["parameters"][0]["name"] ) customlist: list = await get_custom_dims_for_collection( - edr_collectioninfo, edr_collectioninfo["parameters"][0]["name"] + edr_collectioninfo, wmslayers, edr_collectioninfo["parameters"][0]["name"] ) # Custom can be a list of custom dimensions, like ensembles, thresholds @@ -327,7 +332,7 @@ async def get_collectioninfo_for_id( vertical = None vertical_dim = await get_vertical_dim_for_collection( - edr_collectioninfo, edr_collectioninfo["parameters"][0]["name"] + edr_collectioninfo, wmslayers, edr_collectioninfo["parameters"][0]["name"] ) if vertical_dim: vertical = Vertical(**vertical_dim) @@ -403,8 +408,7 @@ async def get_collectioninfo_for_id( output_formats=output_formats, ) - get_cap = await get_capabilities(edr_collection) - return collection, get_cap["expires"] + return collection, expires def get_params_for_collection(edr_collection: str) -> dict[str, Parameter]: @@ -496,7 +500,7 @@ def get_time_values_for_range(rng: str) -> list[str]: async def get_times_for_collection( - edr_collectioninfo: dict, parameter: str = None + edr_collectioninfo: dict, wmslayers, parameter: str = None ) -> tuple[list[list[str]], list[str]]: """ Returns a list of times based on the time dimensions, it does a WMS GetCapabilities to the given dataset (cached) @@ -504,12 +508,10 @@ async def get_times_for_collection( It does this for given parameter. When the parameter is not given it will do it for the first Layer in the GetCapabilities document. """ logger.info("get_times_for_dataset(%s,%s)", edr_collectioninfo["name"], parameter) - getcap = await get_capabilities(edr_collectioninfo["name"]) - wms = getcap["layers"] - if parameter and parameter in wms: - layer = wms[parameter] + if parameter and parameter in wmslayers: + layer = wmslayers[parameter] else: - layer = wms[list(wms)[0]] + layer = wmslayers[list(wmslayers)[0]] if "time" in layer["dimensions"]: time_dim = layer["dimensions"]["time"] @@ -540,18 +542,18 @@ async def get_times_for_collection( return (None, None) -async def get_custom_dims_for_collection(edr_collectioninfo: dict, parameter: str = None): +async def get_custom_dims_for_collection( + edr_collectioninfo: dict, wmslayers, parameter: str = None +): """ Return the dimensions other then elevation or time from the WMS GetCapabilities document. """ - getcap = await get_capabilities(edr_collectioninfo["name"]) - wms = getcap["layers"] custom = [] - if parameter and parameter in list(wms): - layer = wms[parameter] + if parameter and parameter in list(wmslayers): + layer = wmslayers[parameter] else: # default to first layer - layer = wms[list(wms)[0]] + layer = wmslayers[list(wmslayers)[0]] for dim_name in layer["dimensions"]: # Not needed for non custom dims: if dim_name not in [ @@ -570,16 +572,16 @@ async def get_custom_dims_for_collection(edr_collectioninfo: dict, parameter: st return custom if len(custom) > 0 else None -async def get_vertical_dim_for_collection(edr_collectioninfo: dict, parameter: str = None): +async def get_vertical_dim_for_collection( + edr_collectioninfo: dict, wmslayers, parameter: str = None +): """ Return the verticel dimension the WMS GetCapabilities document. """ - getcap = await get_capabilities(edr_collectioninfo["name"]) - wms = getcap["layers"] - if parameter and parameter in list(wms): - layer = wms[parameter] + if parameter and parameter in list(wmslayers): + layer = wmslayers[parameter] else: - layer = wms[list(wms)[0]] + layer = wmslayers[list(wmslayers)[0]] for dim_name in layer["dimensions"]: if dim_name in ["elevation"] or ( @@ -608,14 +610,16 @@ async def rest_get_edr_collections(request: Request, response: Response): links.append(self_link) collections: list[Collection] = [] - min_expires=None + min_expires = None edr_collections = get_edr_collections() for edr_coll in edr_collections: coll, expires = await get_collectioninfo_for_id(edr_coll) if coll: collections.append(coll) if expires is not None: - min_expires=expires if min_expires is None else min(min_expires, expires) + min_expires = ( + expires if min_expires is None else min(min_expires, expires) + ) else: logger.warning("Unable to fetch WMS GetCapabilities for %s", edr_coll) collections_data = Collections(links=links, collections=collections) @@ -633,23 +637,26 @@ async def rest_get_edr_collection_by_id(collection_name: str, response: Response """ GET Returns collection information for given collection id """ - collection,expires = await get_collectioninfo_for_id(collection_name) - response.headers['cache-control']=generate_max_age(expires) + collection, expires = await get_collectioninfo_for_id(collection_name) + response.headers["cache-control"] = generate_max_age(expires) return collection + def get_ttl_from_adaguc_call(headers): - ttl = None - for hdr in headers: - hdr_terms=hdr.split(":") - if hdr_terms[0].lower() == "cache-control": - for cache_control_terms in hdr_terms[1].split(","): - terms = cache_control_terms.split("=") - if terms[0]=='max-age': - ttl = int(terms[1]) - break - if ttl is not None: - break - return ttl + try: + for hdr in headers: + hdr_terms = hdr.split(":") + if hdr_terms[0].lower() == "cache-control": + for cache_control_terms in hdr_terms[1].split(","): + terms = cache_control_terms.split("=") + if terms[0].lower() == "max-age": + ttl = int(terms[1]) + return ttl + return None + except: + pass + return None + async def get_capabilities(collname): """ @@ -664,7 +671,7 @@ async def get_capabilities(collname): ) status, response, headers = await call_adaguc(url=urlrequest.encode("UTF-8")) ttl = get_ttl_from_adaguc_call(headers) - now=datetime.utcnow() + now = datetime.utcnow() logger.info("status: %d", status) if status == 0: xml = response.getvalue() @@ -675,18 +682,22 @@ async def get_capabilities(collname): else: logger.info("callADAGUC by service %s", dataset) wms = WebMapService(dataset["service"], version="1.3.0") - now=datetime.utcnow() - ttl=None - - layers={} - for layername,layerinfo in wms.contents.items(): - layers[layername]={ "name": layername, "dimensions":{**layerinfo.dimensions}, "boundingBoxWGS84": layerinfo.boundingBoxWGS84} + now = datetime.utcnow() + ttl = None + + layers = {} + for layername, layerinfo in wms.contents.items(): + layers[layername] = { + "name": layername, + "dimensions": {**layerinfo.dimensions}, + "boundingBoxWGS84": layerinfo.boundingBoxWGS84, + } if ttl is not None: expires = (now + timedelta(seconds=ttl)).timestamp() else: - expires=None + expires = None - return {"layers": layers, "expires": expires} + return (layers, expires) async def get_ref_times_for_coll(edr_collectioninfo: dict, layer: str) -> list[str]: @@ -704,19 +715,17 @@ async def get_ref_times_for_coll(edr_collectioninfo: dict, layer: str) -> list[s return [] -async def get_extent(edr_collectioninfo: dict): +async def get_extent(edr_collectioninfo: dict, wmslayers): """ Get the boundingbox extent from the WMS GetCapabilities """ - getcap = await get_capabilities(edr_collectioninfo["name"]) - contents = getcap["layers"] first_layer = edr_collectioninfo["parameters"][0]["name"] - if len(contents): - if first_layer in contents: - bbox = contents[first_layer]["boundingBoxWGS84"] + if len(wmslayers): + if first_layer in wmslayers: + bbox = wmslayers[first_layer]["boundingBoxWGS84"] else: - #Fallback to first layer in getcapabilities - bbox = contents[next(iter(contents))]["boundingBoxWGS84"] + # Fallback to first layer in getcapabilities + bbox = wmslayers[next(iter(wmslayers))]["boundingBoxWGS84"] return [[bbox[0], bbox[1]], [bbox[2], bbox[3]]] return None @@ -727,7 +736,9 @@ async def get_extent(edr_collectioninfo: dict): response_model=Instances, response_model_exclude_none=True, ) -async def rest_get_edr_inst_for_coll(collection_name: str, request: Request, response: Response): +async def rest_get_edr_inst_for_coll( + collection_name: str, request: Request, response: Response +): """ GET: Returns all available instances for the collection """ @@ -744,19 +755,21 @@ async def rest_get_edr_inst_for_coll(collection_name: str, request: Request, res ) links: list[Link] = [] links.append(Link(href=instances_url, rel="collection")) - min_expires=None + min_expires = None for instance in list(ref_times): instance_links: list[Link] = [] instance_link = Link(href=f"{instances_url}/{instance}", rel="collection") instance_links.append(instance_link) - instance_info, expires = await get_collectioninfo_for_id(collection_name, instance) + instance_info, expires = await get_collectioninfo_for_id( + collection_name, instance + ) if expires is not None: - min_expires=expires if min_expires is None else min(min_expires, expires) + min_expires = expires if min_expires is None else min(min_expires, expires) instances.append(instance_info) instances_data = Instances(instances=instances, links=links) if min_expires is not None: - response.headers['cache-control'] = generate_max_age(min_expires) + response.headers["cache-control"] = generate_max_age(min_expires) return instances_data @@ -770,12 +783,15 @@ async def rest_get_collection_info(collection_name: str, instance, response: Res GET "/collections/{collection_name}/instances/{instance}" """ coll, expires = await get_collectioninfo_for_id(collection_name, instance) - response.headers['cache-control'] = generate_max_age(expires) + response.headers["cache-control"] = generate_max_age(expires) return coll + def generate_max_age(expires): - rest_age = int((datetime.fromtimestamp(expires)-datetime.utcnow()).total_seconds()) - if rest_age>0: + rest_age = int( + (datetime.fromtimestamp(expires) - datetime.utcnow()).total_seconds() + ) + if rest_age > 0: return f"max-age={rest_age}" return f"max-age=0" diff --git a/python/python_fastapi_server/routers/ogcapi.py b/python/python_fastapi_server/routers/ogcapi.py index d08cfd26..4fed661d 100644 --- a/python/python_fastapi_server/routers/ogcapi.py +++ b/python/python_fastapi_server/routers/ogcapi.py @@ -1,4 +1,5 @@ """ogcApiApp""" + import json import logging import os @@ -18,14 +19,28 @@ from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates -from .models.ogcapifeatures_1_model import (Collection, Collections, - ConfClasses, Extent, - FeatureCollectionGeoJSON, - FeatureGeoJSON, LandingPage, Link, - Spatial, Type) -from .ogcapi_tools import (calculate_coords, call_adaguc, feature_from_dat, - generate_collections, get_extent, get_items_links, - get_parameters, make_bbox) +from .models.ogcapifeatures_1_model import ( + Collection, + Collections, + ConfClasses, + Extent, + FeatureCollectionGeoJSON, + FeatureGeoJSON, + LandingPage, + Link, + Spatial, + Type, +) +from .ogcapi_tools import ( + calculate_coords, + call_adaguc, + feature_from_dat, + generate_collections, + get_extent, + get_items_links, + get_parameters, + make_bbox, +) logger = logging.getLogger(__name__) @@ -37,7 +52,7 @@ def custom_openapi(): - '''Returns fixed openapi schema''' + """Returns fixed openapi schema""" if ogcApiApp.openapi_schema: return ogcApiApp.openapi_schema openapi_schema = make_open_api() @@ -49,9 +64,7 @@ def custom_openapi(): script_dir = os.path.dirname(__file__) static_abs_file_path = os.path.join(script_dir, "static") -ogcApiApp.mount("/static", - StaticFiles(directory=static_abs_file_path), - name="static") +ogcApiApp.mount("/static", StaticFiles(directory=static_abs_file_path), name="static") templates_abs_file_path = os.path.join(script_dir, "templates/ogcapi") templates = Jinja2Templates(directory=templates_abs_file_path) @@ -60,19 +73,12 @@ def custom_openapi(): async def validation_exception_handler(request: Request, exc: RequestValidationError): return JSONResponse( status_code=fastapi_status.HTTP_400_BAD_REQUEST, - content=jsonable_encoder({ - "detail": exc.errors(), - "body": exc.body - }), + content=jsonable_encoder({"detail": exc.errors(), "body": exc.body}), ) -@ogcApiApp.get("/", - response_model=LandingPage, - response_model_exclude_none=True) -@ogcApiApp.get("", - response_model=LandingPage, - response_model_exclude_none=True) +@ogcApiApp.get("/", response_model=LandingPage, response_model_exclude_none=True) +@ogcApiApp.get("", response_model=LandingPage, response_model_exclude_none=True) async def handle_ogc_api_root(req: Request, response: Response, f: str = "json"): links: List[Link] = [] links.append( @@ -81,53 +87,59 @@ async def handle_ogc_api_root(req: Request, response: Response, f: str = "json") rel="self", title="This document in JSON", type="application/json", - )) + ) + ) links.append( Link( href=str(req.url_for("handle_ogc_api_root")) + "?f=html", rel="alternate", title="This document in HTML", type="text/html", - )) + ) + ) links.append( Link( href=str(req.url_for("get_conformance")), rel="conformance", title="Conformance document", type="application/json", - )) + ) + ) links.append( Link( href=str(req.url_for("get_collections")), rel="data", title="Collections", type="application/json", - )) + ) + ) links.append( Link( href=str(req.url_for("openapi")), rel="service-desc", title="The OpenAPI definition as JSON", type="application/vnd.oai.openapi+json;version=3.0", - )) + ) + ) links.append( Link( href=str(req.url_for("get_open_api_yaml")), rel="service-desc", title="The OpenAPI definition as YAML", type="application/vnd.oai.openapi;version=3.0", - )) - landing_page = LandingPage(title="ogcapi", - description="ADAGUC OGCAPI-Features server", - links=links) + ) + ) + landing_page = LandingPage( + title="ogcapi", description="ADAGUC OGCAPI-Features server", links=links + ) - response.headers['cache-control']="max-age=18" + response.headers["cache-control"] = "max-age=60" # TODO find better value if request_type(f) == "HTML": - return templates.TemplateResponse("landingpage.html", { - "request": req, - "landingpage": landing_page.dict() - }) + return templates.TemplateResponse( + "landingpage.html", + {"request": req, "landingpage": landing_page.model_dump()}, + ) return landing_page @@ -140,28 +152,32 @@ def get_collection_links(url): rel="self", title="This document in JSON", type="application/json", - )) + ) + ) links.append( Link( href=url + "?f=html", rel="alternate", title="This document in HTML", type="text/html", - )) + ) + ) links.append( Link( href=url + "/items?f=json", rel="items", title="Items of this collection", type="application/geo+json", - )) + ) + ) links.append( Link( href=url + "/items?f=html", rel="items", title="Items of this collection in HTML", type="text/html", - )) + ) + ) return links @@ -174,14 +190,16 @@ def get_collections_links(url): rel="self", title="This document in JSON", type="application/json", - )) + ) + ) links.append( Link( href=url + "?f=html", rel="alternate", title="This document in HTML", type="text/html", - )) + ) + ) return links @@ -195,20 +213,19 @@ def request_type(wanted_format: str) -> str: return json -@ogcApiApp.get("/collections/", - response_model=Collections, - response_model_exclude_none=True) -@ogcApiApp.get("/collections", - response_model=Collections, - response_model_exclude_none=True) +@ogcApiApp.get( + "/collections/", response_model=Collections, response_model_exclude_none=True +) +@ogcApiApp.get( + "/collections", response_model=Collections, response_model_exclude_none=True +) async def get_collections(req: Request, response: Response, f: str = "json"): collections: List[Collection] = [] parsed_collections = generate_collections() for parsed_collection in parsed_collections.values(): parsed_extent = await get_extent(parsed_collection["dataset"]) if parsed_extent: - spatial = Spatial( - bbox=[list(parsed_extent)]) + spatial = Spatial(bbox=[list(parsed_extent)]) extent = Extent(spatial=spatial) collections.append( Collection( @@ -220,23 +237,25 @@ async def get_collections(req: Request, response: Response, f: str = "json"): req.url_for( "get_collection", coll=parsed_collection["dataset"], - ))), + ) + ) + ), extent=extent, itemType="feature", crs=[DEFAULT_CRS], storageCrs=DEFAULT_CRS, - )) + ) + ) links = get_collections_links(req.url_for("get_collections")) - response.headers["cache-control"]="max-age=18" + response.headers["cache-control"] = "max-age=60" # TODO find better value if request_type(f) == "HTML": - collections_list = [c.dict() for c in collections] - return templates.TemplateResponse("collections.html", { - "request": req, - "collections": collections_list - }) + collections_list = [c.model_dump() for c in collections] + return templates.TemplateResponse( + "collections.html", {"request": req, "collections": collections_list} + ) return Collections( links=links, @@ -244,24 +263,22 @@ async def get_collections(req: Request, response: Response, f: str = "json"): ) -@ogcApiApp.get("/collections/{coll}", - response_model=Collection, - response_model_exclude_none=True) +@ogcApiApp.get( + "/collections/{coll}", response_model=Collection, response_model_exclude_none=True +) async def get_collection(coll: str, req: Request, f: str = "json"): extent = Extent(spatial=Spatial(bbox=[await get_extent(coll)])) - coll = Collection( + collection = Collection( id=coll, title="title1", description="descr1", extent=extent, - links=get_collection_links( - str(req.url_for("get_collection", coll=coll))), + links=get_collection_links(str(req.url_for("get_collection", coll=coll))), ) if request_type(f) == "HTML": - return templates.TemplateResponse("collection.html", { - "request": req, - "collection": coll.dict() - }) + return templates.TemplateResponse( + "collection.html", {"request": req, "collection": collection.model_dump()} + ) return coll @@ -276,9 +293,9 @@ async def get_collection(coll: str, req: Request, f: str = "json"): ] -@ogcApiApp.get("/conformance", - response_model=ConfClasses, - response_model_exclude_none=True) +@ogcApiApp.get( + "/conformance", response_model=ConfClasses, response_model_exclude_none=True +) async def get_conformance(req: Request, f: str = "json"): conf_classes = ConfClasses(conformsTo=conformanceClasses) if request_type(f) == "HTML": @@ -295,14 +312,16 @@ async def get_conformance(req: Request, f: str = "json"): def make_open_api(): - """ Adjusts openapi object """ - openapi = get_openapi(title=ogcApiApp.title, - version=ogcApiApp.version, - routes=ogcApiApp.routes, - servers=ogcApiApp.servers) + """Adjusts openapi object""" + openapi = get_openapi( + title=ogcApiApp.title, + version=ogcApiApp.version, + routes=ogcApiApp.routes, + servers=ogcApiApp.servers, + ) paths = openapi.get("paths", {}) for pth in paths.keys(): - params = (paths.get(pth).get("get", {}).get("parameters", [])) + params = paths.get(pth).get("get", {}).get("parameters", []) for param in params: if param["in"] == "query" and param["name"] == "bbox": param["style"] = "form" @@ -311,9 +330,7 @@ def make_open_api(): "type": "array", "minItems": 4, "maxItems": 6, - "items": { - "type": "number" - }, + "items": {"type": "number"}, } if param["in"] == "query" and param["name"] == "limit": param["style"] = "form" @@ -344,8 +361,7 @@ async def get_open_api_yaml(): async def get_single_item(item_id: str, url: str) -> FeatureGeoJSON: - collection, observed_property_name, point, dims, datetime_ = item_id.split( - ";") + collection, observed_property_name, point, dims, datetime_ = item_id.split(";") coord = list(map(float, point.split(","))) dimspec = "" if len(dims): @@ -356,26 +372,23 @@ async def get_single_item(item_id: str, url: str) -> FeatureGeoJSON: datetime_ = datetime_.replace("$", "/") request_url = ( f"http://localhost:8080/wms?dataset={collection}&query_layers={observed_property_name}" - + "&service=WMS&version=1.3.0&request=getPointValue" + - "&FORMAT=application/json&INFO_FORMAT=application/json" + - f"&X={coord[0]}&Y={coord[1]}&CRS=EPSG:4326") + + "&service=WMS&version=1.3.0&request=getPointValue" + + "&FORMAT=application/json&INFO_FORMAT=application/json" + + f"&X={coord[0]}&Y={coord[1]}&CRS=EPSG:4326" + ) if datetime_: request_url += f"&TIME={datetime_}" request_url += dimspec status, data = await call_adaguc(request_url.encode("UTF-8")) if status == 0: try: - response_data = json.loads(data.getvalue(), - object_pairs_hook=OrderedDict) + response_data = json.loads(data.getvalue(), object_pairs_hook=OrderedDict) except ValueError: root = fromstring(data) - retval = json.dumps({ - "Error": { - "code": root[0].attrib["code"], - "message": root[0].text - } - }) + retval = json.dumps( + {"Error": {"code": root[0].attrib["code"], "message": root[0].text}} + ) return 400, retval features = [] for data in response_data: @@ -409,9 +422,9 @@ async def get_features_for_items( coords = [point] if not observed_property_name: collinfo = await get_parameters(coll) - first_param=next(iter(collinfo)) + first_param = next(iter(collinfo)) print(first_param, flush=True) - observed_property_name =[first_param] + observed_property_name = [first_param] # Default observedPropertyName = first layername param_list = ",".join(observed_property_name) @@ -425,9 +438,10 @@ async def get_features_for_items( for coord in coords: request_url = ( f"http://localhost:8080/wms?dataset={coll}&query_layers={param_list}" - + "&service=WMS&version=1.3.0&request=getPointValue&" + - "FORMAT=application/json&INFO_FORMAT=application/json" + - f"&X={coord[0]}&Y={coord[1]}&CRS=EPSG:4326") + + "&service=WMS&version=1.3.0&request=getPointValue&" + + "FORMAT=application/json&INFO_FORMAT=application/json" + + f"&X={coord[0]}&Y={coord[1]}&CRS=EPSG:4326" + ) if datetime_: request_url += f"&TIME={datetime_}" if result_time: @@ -436,17 +450,15 @@ async def get_features_for_items( status, data = await call_adaguc(request_url.encode("UTF-8")) if status == 0: try: - response_data = json.loads(data.getvalue(), - object_pairs_hook=OrderedDict) + response_data = json.loads( + data.getvalue(), object_pairs_hook=OrderedDict + ) except ValueError: root = fromstring(data) - retval = json.dumps({ - "Error": { - "code": root[0].attrib["code"], - "message": root[0].text - } - }) + retval = json.dumps( + {"Error": {"code": root[0].attrib["code"], "message": root[0].text}} + ) return 400, retval # TODO for data in response_data: data_features = feature_from_dat(data, coll, base_url, False) @@ -461,8 +473,7 @@ def check_point(point: Union[str, None] = Query(default=None)) -> List[float]: coords = point.split(",") if len(coords) != 2: # Error - raise HTTPException(status_code=404, - detail="point should contain 2 floats") + raise HTTPException(status_code=404, detail="point should contain 2 floats") return list(map(float, coords)) @@ -472,21 +483,21 @@ def check_bbox(bbox: Union[str, None] = Query(default=None)) -> List[float]: coords = bbox.split(",") if len(coords) != 4 and len(coords) != 6: # Error - raise HTTPException(status_code=404, - detail="bbox should contain 4 or 6 floats") + raise HTTPException(status_code=404, detail="bbox should contain 4 or 6 floats") return list(map(float, coords)) -def check_observed_property_name(observed_property_name: Union[ - str, None] = Query(default=None)) -> List[str]: +def check_observed_property_name( + observed_property_name: Union[str, None] = Query(default=None) +) -> List[str]: if observed_property_name is None: return None names = observed_property_name.split(",") if len(names) < 1: # Error raise HTTPException( - status_code=404, - detail="observedPropertyName should contain > 0 names") + status_code=404, detail="observedPropertyName should contain > 0 names" + ) return names @@ -496,8 +507,7 @@ def check_dims(dims: Union[str, None] = Query(default=None)) -> Dict[str, str]: dim_terms = dims.split(";") if len(dim_terms) < 1: # Error - raise HTTPException(status_code=404, - detail="dims should contain > 0 names") + raise HTTPException(status_code=404, detail="dims should contain > 0 names") dimensions = {} for dim in dim_terms: dimname, dimval = dim.split(":") @@ -509,17 +519,17 @@ def check_dims(dims: Union[str, None] = Query(default=None)) -> Dict[str, str]: return dimensions -def check_bbox_crs(bbox_crs: Union[str, - None] = Query(default=None, - alias="bbox-crs")) -> str: +def check_bbox_crs( + bbox_crs: Union[str, None] = Query(default=None, alias="bbox-crs") +) -> str: if bbox_crs is None: return None if bbox_crs not in SUPPORTED_CRS_LIST: # Error raise HTTPException( - status_code=400, - detail=f"bbox-crs {bbox_crs} not in supported list") + status_code=400, detail=f"bbox-crs {bbox_crs} not in supported list" + ) return bbox_crs @@ -529,8 +539,7 @@ def check_crs(crs: Union[str, None] = Query(default=None)) -> str: if crs not in SUPPORTED_CRS_LIST: # Error - raise HTTPException(status_code=400, - detail=f"crs {crs} not in supported list") + raise HTTPException(status_code=400, detail=f"crs {crs} not in supported list") return crs @@ -540,20 +549,19 @@ def check_crs(crs: Union[str, None] = Query(default=None)) -> str: response_model_exclude_none=True, ) async def get_items_for_collection( - coll: str, - req: Request, - response: Response, - f: str = "json", - limit: Union[int, None] = Query(default=10), - start: Union[int, None] = Query(default=0), - bbox: Union[str, None] = Depends(check_bbox), - point: Union[str, None] = Depends(check_point), - result_time: Union[str, None] = Query(default=None), - datetime_: Union[str, None] = Query(default=None, alias="datetime"), - observed_property_name: Union[str, None] = Depends( - check_observed_property_name), - dims: Union[str, None] = Depends(check_dims), - npoints: Union[int, None] = Query(default=4), + coll: str, + req: Request, + response: Response, + f: str = "json", + limit: Union[int, None] = Query(default=10), + start: Union[int, None] = Query(default=0), + bbox: Union[str, None] = Depends(check_bbox), + point: Union[str, None] = Depends(check_point), + result_time: Union[str, None] = Query(default=None), + datetime_: Union[str, None] = Query(default=None, alias="datetime"), + observed_property_name: Union[str, None] = Depends(check_observed_property_name), + dims: Union[str, None] = Depends(check_dims), + npoints: Union[int, None] = Query(default=4), ): allowed_params = [ "f", @@ -574,10 +582,12 @@ async def get_items_for_collection( if len(extra_params): return JSONResponse( status_code=fastapi_status.HTTP_400_BAD_REQUEST, - content=jsonable_encoder({ - "detail": "extra parameters", - "body": [extra_params], - }), + content=jsonable_encoder( + { + "detail": "extra parameters", + "body": [extra_params], + } + ), ) base_url = str(req.url_for("get_items_for_collection", coll=coll)) try: @@ -592,7 +602,7 @@ async def get_items_for_collection( npoints=npoints, dims=dims, ) - features_to_return = features[start:start + limit] + features_to_return = features[start : start + limit] number_matched = len(features) number_returned = len(features_to_return) prev_start = None @@ -618,7 +628,7 @@ async def get_items_for_collection( ) response.headers["Content-Crs"] = f"<{DEFAULT_CRS}>" if request_type(f) == "HTML": - features = [f.dict() for f in feature_collection.features] + features = [f.model_dump() for f in feature_collection.features] return templates.TemplateResponse( "items.html", { @@ -631,8 +641,9 @@ async def get_items_for_collection( ) return feature_collection except Exception as exc: - logger.error("ERR: %s", - traceback.format_exception(None, exc, exc.__traceback__)) + logger.error( + "ERR: %s", traceback.format_exception(None, exc, exc.__traceback__) + ) return None From 8faf0f13ff9b1d2dad78f8f4675b234b6b7acf98 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Thu, 7 Mar 2024 16:31:58 +0100 Subject: [PATCH 26/44] Less FastAPI debugging --- python/python_fastapi_server/routers/edr.py | 2 +- python/python_fastapi_server/routers/ogcapi.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index 8ddb1e5a..ca89b8c1 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -64,7 +64,7 @@ logger = logging.getLogger(__name__) logger.debug("Starting EDR") -edrApiApp = FastAPI(debug=True) +edrApiApp = FastAPI(debug=False) OWSLIB_DUMMY_URL = "http://localhost:8000" diff --git a/python/python_fastapi_server/routers/ogcapi.py b/python/python_fastapi_server/routers/ogcapi.py index 4fed661d..b01b2006 100644 --- a/python/python_fastapi_server/routers/ogcapi.py +++ b/python/python_fastapi_server/routers/ogcapi.py @@ -48,7 +48,7 @@ DEFAULT_CRS = "http://www.opengis.net/def/crs/OGC/1.3/CRS84" SUPPORTED_CRS_LIST = [DEFAULT_CRS] -ogcApiApp = FastAPI(debug=True, openapi_url="/api") +ogcApiApp = FastAPI(debug=False, openapi_url="/api") def custom_openapi(): From ebed1e4ae9980ad88649027f8f7318e1348f2f8a Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Thu, 7 Mar 2024 16:53:43 +0100 Subject: [PATCH 27/44] Fix for class based redis pool --- python/lib/adaguc/runAdaguc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/lib/adaguc/runAdaguc.py b/python/lib/adaguc/runAdaguc.py index f46780b3..b250e3c2 100644 --- a/python/lib/adaguc/runAdaguc.py +++ b/python/lib/adaguc/runAdaguc.py @@ -214,7 +214,7 @@ async def runADAGUCServer( if self.cache_wanted(url): cache_key = str((url, adagucargs)).encode("utf-8") - age, headers, data = get_cached_response(self.redis_pool, cache_key) + age, headers, data = get_cached_response(runAdaguc.redis_pool, cache_key) if age is not None: return [0, data, headers] @@ -252,7 +252,9 @@ async def runADAGUCServer( else: if self.cache_wanted(url): - response_to_cache(self.redis_pool, cache_key, headers, filetogenerate) + response_to_cache( + runAdaguc.redis_pool, cache_key, headers, filetogenerate + ) return [status, filetogenerate, headers] def writetofile(self, filename, data): From 2abd2b6a81403d9267064d01fec76904446ff173 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Fri, 8 Mar 2024 12:18:21 +0100 Subject: [PATCH 28/44] Fixed mag-age/age caclulations; removed aiocache --- python/python_fastapi_server/routers/edr.py | 79 +++++++++------------ 1 file changed, 35 insertions(+), 44 deletions(-) diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index ca89b8c1..fd274549 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -8,8 +8,6 @@ KNMI """ -import aiocache -from aiocache.serializers import PickleSerializer import itertools import json import logging @@ -261,10 +259,9 @@ async def get_collection_position( ) if resp: dat = json.loads(resp) - ttl = get_ttl_from_adaguc_call(headers) + ttl = get_ttl_from_adaguc_headers(headers) if ttl is not None: - expires = (datetime.utcnow() + timedelta(seconds=ttl)).timestamp() - response.headers["cache-control"] = generate_max_age(expires) + response.headers["cache-control"] = generate_max_age(ttl) return covjson_from_resp(dat, edr_collections[collection_name]["vertical_name"]) raise EdrException(code=400, description="No data") @@ -276,7 +273,6 @@ async def get_collection_position( } -@aiocache.cached(ttl=60, serializer=PickleSerializer()) async def get_collectioninfo_for_id( edr_collection: str, instance: str = None, @@ -311,7 +307,7 @@ async def get_collectioninfo_for_id( ) links.append(instances_link) - wmslayers, expires = await get_capabilities(edr_collectioninfo["name"]) + wmslayers, ttl = await get_capabilities(edr_collectioninfo["name"]) bbox = await get_extent(edr_collectioninfo, wmslayers) if bbox is None: @@ -411,7 +407,7 @@ async def get_collectioninfo_for_id( output_formats=output_formats, ) - return collection, expires + return collection, ttl def get_params_for_collection(edr_collection: str) -> dict[str, Parameter]: @@ -613,21 +609,19 @@ async def rest_get_edr_collections(request: Request, response: Response): links.append(self_link) collections: list[Collection] = [] - min_expires = None + min_ttl = None edr_collections = get_edr_collections() for edr_coll in edr_collections: - coll, expires = await get_collectioninfo_for_id(edr_coll) + coll, ttl = await get_collectioninfo_for_id(edr_coll) if coll: collections.append(coll) - if expires is not None: - min_expires = ( - expires if min_expires is None else min(min_expires, expires) - ) + if ttl is not None: + min_ttl = ttl if min_ttl is None else min(min_ttl, ttl) else: logger.warning("Unable to fetch WMS GetCapabilities for %s", edr_coll) collections_data = Collections(links=links, collections=collections) - if min_expires is not None: - response.headers["cache-control"] = generate_max_age(min_expires) + if min_ttl is not None: + response.headers["cache-control"] = generate_max_age(min_ttl) return collections_data @@ -640,28 +634,34 @@ async def rest_get_edr_collection_by_id(collection_name: str, response: Response """ GET Returns collection information for given collection id """ - collection, expires = await get_collectioninfo_for_id(collection_name) - response.headers["cache-control"] = generate_max_age(expires) + collection, ttl = await get_collectioninfo_for_id(collection_name) + response.headers["cache-control"] = generate_max_age(ttl) return collection -def get_ttl_from_adaguc_call(headers): +def get_ttl_from_adaguc_headers(headers): try: + max_age = None + age = None for hdr in headers: hdr_terms = hdr.split(":") if hdr_terms[0].lower() == "cache-control": for cache_control_terms in hdr_terms[1].split(","): terms = cache_control_terms.split("=") if terms[0].lower() == "max-age": - ttl = int(terms[1]) - return ttl + max_age = int(terms[1]) + elif hdr_terms[0].lower() == "age": + age = int(hdr_terms[1]) + if max_age and age: + return max_age - age + elif max_age: + return max_age return None except: pass return None -@aiocache.cached(ttl=60, serializer=PickleSerializer()) async def get_capabilities(collname): """ Get the collectioninfo from the WMS GetCapabilities @@ -674,7 +674,7 @@ async def get_capabilities(collname): f"dataset={dataset}&service=wms&version=1.3.0&request=getcapabilities" ) status, response, headers = await call_adaguc(url=urlrequest.encode("UTF-8")) - ttl = get_ttl_from_adaguc_call(headers) + ttl = get_ttl_from_adaguc_headers(headers) now = datetime.utcnow() logger.info("status: %d", status) if status == 0: @@ -696,12 +696,8 @@ async def get_capabilities(collname): "dimensions": {**layerinfo.dimensions}, "boundingBoxWGS84": layerinfo.boundingBoxWGS84, } - if ttl is not None: - expires = (now + timedelta(seconds=ttl)).timestamp() - else: - expires = None - return (layers, expires) + return layers, ttl async def get_ref_times_for_coll(edr_collectioninfo: dict, layer: str) -> list[str]: @@ -759,21 +755,19 @@ async def rest_get_edr_inst_for_coll( ) links: list[Link] = [] links.append(Link(href=instances_url, rel="collection")) - min_expires = None + min_ttl = None for instance in list(ref_times): instance_links: list[Link] = [] instance_link = Link(href=f"{instances_url}/{instance}", rel="collection") instance_links.append(instance_link) - instance_info, expires = await get_collectioninfo_for_id( - collection_name, instance - ) - if expires is not None: - min_expires = expires if min_expires is None else min(min_expires, expires) + instance_info, ttl = await get_collectioninfo_for_id(collection_name, instance) + if ttl is not None: + min_ttl = ttl if min_ttl is None else min(min_ttl, ttl) instances.append(instance_info) instances_data = Instances(instances=instances, links=links) - if min_expires is not None: - response.headers["cache-control"] = generate_max_age(min_expires) + if min_ttl is not None: + response.headers["cache-control"] = generate_max_age(min_ttl) return instances_data @@ -786,17 +780,14 @@ async def rest_get_collection_info(collection_name: str, instance, response: Res """ GET "/collections/{collection_name}/instances/{instance}" """ - coll, expires = await get_collectioninfo_for_id(collection_name, instance) - response.headers["cache-control"] = generate_max_age(expires) + coll, ttl = await get_collectioninfo_for_id(collection_name, instance) + response.headers["cache-control"] = generate_max_age(ttl) return coll -def generate_max_age(expires): - rest_age = int( - (datetime.fromtimestamp(expires) - datetime.utcnow()).total_seconds() - ) - if rest_age > 0: - return f"max-age={rest_age}" +def generate_max_age(ttl): + if ttl >= 0: + return f"max-age={ttl}" return f"max-age=0" From 549046eda0db3f0e659793cecf8aebd23e3223b6 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Fri, 8 Mar 2024 12:20:52 +0100 Subject: [PATCH 29/44] Removed aiocache from OGCAPI features --- python/python_fastapi_server/routers/ogcapi_tools.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/python/python_fastapi_server/routers/ogcapi_tools.py b/python/python_fastapi_server/routers/ogcapi_tools.py index 4bff5bbc..af2d8d38 100644 --- a/python/python_fastapi_server/routers/ogcapi_tools.py +++ b/python/python_fastapi_server/routers/ogcapi_tools.py @@ -1,5 +1,3 @@ -import aiocache -from aiocache.serializers import PickleSerializer import itertools import logging import os @@ -21,6 +19,7 @@ logger = logging.getLogger(__name__) + def make_bbox(extent): s_extent = [] for i in extent: @@ -120,7 +119,6 @@ async def call_adaguc(url): return status, data, headers -@aiocache.cached(ttl=60, serializer=PickleSerializer()) async def get_capabilities(collname): """ Get the collectioninfo from the WMS GetCapabilities @@ -128,9 +126,7 @@ async def get_capabilities(collname): coll = generate_collections().get(collname) logger.info("callADAGUC by dataset") dataset = coll["dataset"] - urlrequest = ( - f"dataset={dataset}&service=wms&version=1.3.0&request=getcapabilities" - ) + urlrequest = f"dataset={dataset}&service=wms&version=1.3.0&request=getcapabilities" status, response, _headers = await call_adaguc(url=urlrequest.encode("UTF-8")) if status == 0: xml = response.getvalue() @@ -167,7 +163,6 @@ def get_dimensions(layer, skip_dims=None): return dims -@aiocache.cached(ttl=60, serializer=PickleSerializer()) async def get_parameters(collname): """ get_parameters From 411ca59987926e91b7865d7b337a57bf1d505e0f Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Fri, 8 Mar 2024 12:42:36 +0100 Subject: [PATCH 30/44] Fixed broken OGCAPI features --- python/python_fastapi_server/routers/ogcapi.py | 10 +++++----- python/python_fastapi_server/routers/ogcapi_tools.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/python_fastapi_server/routers/ogcapi.py b/python/python_fastapi_server/routers/ogcapi.py index b01b2006..9ac6465e 100644 --- a/python/python_fastapi_server/routers/ogcapi.py +++ b/python/python_fastapi_server/routers/ogcapi.py @@ -379,7 +379,7 @@ async def get_single_item(item_id: str, url: str) -> FeatureGeoJSON: if datetime_: request_url += f"&TIME={datetime_}" request_url += dimspec - status, data = await call_adaguc(request_url.encode("UTF-8")) + status, data, _ = await call_adaguc(request_url.encode("UTF-8")) if status == 0: try: response_data = json.loads(data.getvalue(), object_pairs_hook=OrderedDict) @@ -423,10 +423,9 @@ async def get_features_for_items( if not observed_property_name: collinfo = await get_parameters(coll) first_param = next(iter(collinfo)) - print(first_param, flush=True) - observed_property_name = [first_param] + print("FIRST", first_param, flush=True) + observed_property_name = [first_param["name"]] # Default observedPropertyName = first layername - param_list = ",".join(observed_property_name) param_list = ",".join(observed_property_name) @@ -447,7 +446,7 @@ async def get_features_for_items( if result_time: request_url += f"&DIM_REFERENCE_TIME={result_time}" request_url += dimspec - status, data = await call_adaguc(request_url.encode("UTF-8")) + status, data, _ = await call_adaguc(request_url.encode("UTF-8")) if status == 0: try: response_data = json.loads( @@ -626,6 +625,7 @@ async def get_items_for_collection( numberMatched=number_matched, numberReturned=number_returned, ) + print("FC:", feature_collection) response.headers["Content-Crs"] = f"<{DEFAULT_CRS}>" if request_type(f) == "HTML": features = [f.model_dump() for f in feature_collection.features] diff --git a/python/python_fastapi_server/routers/ogcapi_tools.py b/python/python_fastapi_server/routers/ogcapi_tools.py index af2d8d38..22d44f77 100644 --- a/python/python_fastapi_server/routers/ogcapi_tools.py +++ b/python/python_fastapi_server/routers/ogcapi_tools.py @@ -127,7 +127,7 @@ async def get_capabilities(collname): logger.info("callADAGUC by dataset") dataset = coll["dataset"] urlrequest = f"dataset={dataset}&service=wms&version=1.3.0&request=getcapabilities" - status, response, _headers = await call_adaguc(url=urlrequest.encode("UTF-8")) + status, response, _ = await call_adaguc(url=urlrequest.encode("UTF-8")) if status == 0: xml = response.getvalue() wms = WebMapService(coll["service"], xml=xml, version="1.3.0") @@ -178,7 +178,7 @@ async def get_parameters(collname): layers.append(layer) layers.sort(key=lambda l: l["name"]) - return {"layers": layers} + return layers def make_dims(dims, data): From ec1466481007486ccc516edb6369b30c45c9ab4c Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Fri, 8 Mar 2024 13:08:19 +0100 Subject: [PATCH 31/44] Caching brotli compressed if <10M --- python/lib/adaguc/runAdaguc.py | 8 +++---- .../routers/caching_middleware.py | 22 ++++++++++--------- .../python_fastapi_server/routers/ogcapi.py | 1 - 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/python/lib/adaguc/runAdaguc.py b/python/lib/adaguc/runAdaguc.py index b250e3c2..1247d598 100644 --- a/python/lib/adaguc/runAdaguc.py +++ b/python/lib/adaguc/runAdaguc.py @@ -1,12 +1,10 @@ import asyncio from ast import Dict -import subprocess import os from os.path import expanduser from PIL import Image from io import BytesIO -import io -import tempfile +import brotli import shutil import random import string @@ -312,7 +310,7 @@ def response_to_cache(redis_pool, key, headers: str, data): entrytime + f"{len(cacheable_headers_json):06d}".encode("utf-8") + cacheable_headers_json - + data.getvalue(), + + brotli.compress(data.getvalue()), ex=ttl, ) redis_client.close() @@ -333,5 +331,5 @@ def get_cached_response(redis_pool, key): headers = json.loads(cached[16 : 16 + headers_len]) headers.append(f"age: {age}") - data = cached[16 + headers_len :] + data = brotli.decompress(cached[16 + headers_len :]) return age, headers, BytesIO(data) diff --git a/python/python_fastapi_server/routers/caching_middleware.py b/python/python_fastapi_server/routers/caching_middleware.py index 14af67f8..3e2b470a 100644 --- a/python/python_fastapi_server/routers/caching_middleware.py +++ b/python/python_fastapi_server/routers/caching_middleware.py @@ -52,16 +52,18 @@ async def response_to_cache(redis_pool, request, headers, data, ex: int): entrytime = f"{calendar.timegm(datetime.utcnow().utctimetuple()):10d}".encode( "utf-8" ) - redis_client = redis.Redis(connection_pool=redis_pool) - await redis_client.set( - key, - entrytime - + f"{len(headers_json):06d}".encode("utf-8") - + headers_json - + brotli.compress(data), - ex=ex, - ) - await redis_client.aclose() + compressed_data = brotli.compress(data) + if len(compressed_data) < 10000000: + redis_client = redis.Redis(connection_pool=redis_pool) + await redis_client.set( + key, + entrytime + + f"{len(headers_json):06d}".encode("utf-8") + + headers_json + + brotli.compress(data), + ex=ex, + ) + await redis_client.aclose() def generate_key(request): diff --git a/python/python_fastapi_server/routers/ogcapi.py b/python/python_fastapi_server/routers/ogcapi.py index 9ac6465e..91607186 100644 --- a/python/python_fastapi_server/routers/ogcapi.py +++ b/python/python_fastapi_server/routers/ogcapi.py @@ -625,7 +625,6 @@ async def get_items_for_collection( numberMatched=number_matched, numberReturned=number_returned, ) - print("FC:", feature_collection) response.headers["Content-Crs"] = f"<{DEFAULT_CRS}>" if request_type(f) == "HTML": features = [f.model_dump() for f in feature_collection.features] From 8e7c3c50dee63269d0fff24002d510285bb2d213 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Mon, 11 Mar 2024 15:23:21 +0100 Subject: [PATCH 32/44] Fixes from review --- adagucserverEC/CCreateHistogram.cpp | 10 ++++++---- adagucserverEC/CImageDataWriter.cpp | 8 +++++--- python/lib/adaguc/runAdaguc.py | 11 +++++++---- .../routers/caching_middleware.py | 4 ++-- python/python_fastapi_server/routers/edr.py | 11 +++++------ python/python_fastapi_server/routers/ogcapi_tools.py | 1 - 6 files changed, 25 insertions(+), 20 deletions(-) diff --git a/adagucserverEC/CCreateHistogram.cpp b/adagucserverEC/CCreateHistogram.cpp index dcc17006..0b9280fc 100644 --- a/adagucserverEC/CCreateHistogram.cpp +++ b/adagucserverEC/CCreateHistogram.cpp @@ -11,10 +11,11 @@ int CCreateHistogram::createHistogram(CDataSource *dataSource, CDrawImage *) { CT::string resultJSON; if (dataSource->srvParams->JSONP.length() == 0) { CDBDebug("CREATING JSON"); - printf("%s%c%c\n", "Content-Type: application/json", 13, 10); + printf("%s%s%c%c\n", "Content-Type: application/json", dataSource->srvParams->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); } else { CDBDebug("CREATING JSONP %s", dataSource->srvParams->JSONP.c_str()); - printf("%s%c%c\n%s(", "Content-Type: application/javascript", 13, 10, dataSource->srvParams->JSONP.c_str()); + printf("%s%s%c%c", "Content-Type: application/javascript", dataSource->srvParams->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); + printf("\n%s()", dataSource->srvParams->JSONP.c_str()); } // puts("{\"a\": 1}"); @@ -257,10 +258,11 @@ int CCreateHistogram::end() { CT::string resultJSON; if (baseDataSource->srvParams->JSONP.length() == 0) { CDBDebug("CREATING JSON"); - printf("%s%c%c\n", "Content-Type: application/json", 13, 10); + printf("%s%s%c%c\n", "Content-Type: application/json", 13, 10); } else { CDBDebug("CREATING JSONP %s", baseDataSource->srvParams->JSONP.c_str()); - printf("%s%c%c\n%s(", "Content-Type: application/javascript", 13, 10, baseDataSource->srvParams->JSONP.c_str()); + printf("%s%s%c%c", "Content-Type: application/javascript", 13, 10); + printf("\n%s", baseDataSource->srvParams->JSONP.c_str()); } puts(JSONdata.c_str()); diff --git a/adagucserverEC/CImageDataWriter.cpp b/adagucserverEC/CImageDataWriter.cpp index 6fe91b02..e334ec0b 100644 --- a/adagucserverEC/CImageDataWriter.cpp +++ b/adagucserverEC/CImageDataWriter.cpp @@ -2417,7 +2417,8 @@ int CImageDataWriter::end() { printf("%s%s%c%c\n", "Content-Type: application/json", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); } else { CDBDebug("CREATING JSONP %s", srvParam->JSONP.c_str()); - printf("%s%s%c%c\n", "Content-Type: application/javascript", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); + printf("%s%s%c%c", "Content-Type: application/javascript", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); + printf("\n%s(", srvParam->JSONP.c_str()); } puts(data.c_str()); @@ -2734,10 +2735,11 @@ int CImageDataWriter::end() { CT::string resultJSON; if (srvParam->JSONP.length() == 0) { CDBDebug("CREATING JSON"); - resultJSON.print("%s%c%c\n", "Content-Type: application/json", 13, 10); + resultJSON.print("%s%s%c%c\n", "Content-Type: application/json", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); } else { CDBDebug("CREATING JSONP %s", srvParam->JSONP.c_str()); - resultJSON.print("%s%c%c\n", "Content-Type: application/javascript", 13, 10); + resultJSON.print("%s%s%c%c", "Content-Type: application/javascript", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); + resultJSON.print("\n%s(", srvParam->JSONP.c_str()); } CXMLParser::XMLElement rootElement; diff --git a/python/lib/adaguc/runAdaguc.py b/python/lib/adaguc/runAdaguc.py index 1247d598..409c321d 100644 --- a/python/lib/adaguc/runAdaguc.py +++ b/python/lib/adaguc/runAdaguc.py @@ -148,6 +148,11 @@ def printLogFile(self): print("=== END ADAGUC LOGS ===") def cache_wanted(self, url: str): + """determine if the results of this url request should be stored in + the Redis cache + + Returns: boolean + """ if not runAdaguc.use_cache: return False if "request=getcapabilities" in url.lower(): @@ -167,8 +172,6 @@ async def runADAGUCServer( # adagucenv=os.environ.copy() # adagucenv.update(env) - # url = re.sub(r"^(.*)(&[0-9\.]*)$", r"\g<1>&1", url) # This removes adaguc-viewer's unique-URL feature - adagucenv = env adagucenv["ADAGUC_ENABLELOGBUFFER"] = os.getenv( @@ -278,7 +281,7 @@ def mkdir_p(self, directory): os.makedirs(directory) -skip_headers = ["x-process-time", "age"] +headers_to_skip = ["x-process-time", "age"] def response_to_cache(redis_pool, key, headers: str, data): @@ -286,7 +289,7 @@ def response_to_cache(redis_pool, key, headers: str, data): ttl = 0 for header in headers: k, v = header.split(":") - if k not in skip_headers: + if k not in headers_to_skip: cacheable_headers.append(header) if k.lower().startswith("cache-control"): for term in v.split(";"): diff --git a/python/python_fastapi_server/routers/caching_middleware.py b/python/python_fastapi_server/routers/caching_middleware.py index 3e2b470a..5c9f45ce 100644 --- a/python/python_fastapi_server/routers/caching_middleware.py +++ b/python/python_fastapi_server/routers/caching_middleware.py @@ -37,7 +37,7 @@ async def get_cached_response(redis_pool, request): return age, headers, data -skip_headers = ["x-process-time", "age"] +headers_to_skip = ["x-process-time", "age"] async def response_to_cache(redis_pool, request, headers, data, ex: int): @@ -45,7 +45,7 @@ async def response_to_cache(redis_pool, request, headers, data, ex: int): fixed_headers = {} for k in headers: - if not k in skip_headers: + if not k in headers_to_skip: fixed_headers[k] = headers[k] headers_json = json.dumps(fixed_headers, ensure_ascii=False).encode("utf-8") diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index fd274549..57a8cd47 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -259,7 +259,7 @@ async def get_collection_position( ) if resp: dat = json.loads(resp) - ttl = get_ttl_from_adaguc_headers(headers) + ttl = get_age_from_adaguc_headers(headers) if ttl is not None: response.headers["cache-control"] = generate_max_age(ttl) return covjson_from_resp(dat, edr_collections[collection_name]["vertical_name"]) @@ -639,7 +639,8 @@ async def rest_get_edr_collection_by_id(collection_name: str, response: Response return collection -def get_ttl_from_adaguc_headers(headers): +def get_age_from_adaguc_headers(headers): + """Derives an age value from the max-age/age cache headers that ADAGUC executable returns""" try: max_age = None age = None @@ -674,8 +675,7 @@ async def get_capabilities(collname): f"dataset={dataset}&service=wms&version=1.3.0&request=getcapabilities" ) status, response, headers = await call_adaguc(url=urlrequest.encode("UTF-8")) - ttl = get_ttl_from_adaguc_headers(headers) - now = datetime.utcnow() + ttl = get_age_from_adaguc_headers(headers) logger.info("status: %d", status) if status == 0: xml = response.getvalue() @@ -686,7 +686,6 @@ async def get_capabilities(collname): else: logger.info("callADAGUC by service %s", dataset) wms = WebMapService(dataset["service"], version="1.3.0") - now = datetime.utcnow() ttl = None layers = {} @@ -707,7 +706,7 @@ async def get_ref_times_for_coll(edr_collectioninfo: dict, layer: str) -> list[s dataset = edr_collectioninfo["dataset"] url = f"?DATASET={dataset}&SERVICE=WMS&VERSION=1.3.0&request=getreferencetimes&LAYER={layer}" logger.info("getreftime_url(%s,%s): %s", dataset, layer, url) - status, response, headers = await call_adaguc(url=url.encode("UTF-8")) + status, response, _ = await call_adaguc(url=url.encode("UTF-8")) if status == 0: ref_times = json.loads(response.getvalue()) instance_ids = [parse_iso(reft).strftime("%Y%m%d%H%M") for reft in ref_times] diff --git a/python/python_fastapi_server/routers/ogcapi_tools.py b/python/python_fastapi_server/routers/ogcapi_tools.py index 22d44f77..c0ede8ff 100644 --- a/python/python_fastapi_server/routers/ogcapi_tools.py +++ b/python/python_fastapi_server/routers/ogcapi_tools.py @@ -124,7 +124,6 @@ async def get_capabilities(collname): Get the collectioninfo from the WMS GetCapabilities """ coll = generate_collections().get(collname) - logger.info("callADAGUC by dataset") dataset = coll["dataset"] urlrequest = f"dataset={dataset}&service=wms&version=1.3.0&request=getcapabilities" status, response, _ = await call_adaguc(url=urlrequest.encode("UTF-8")) From 2a1cd6fe81e13261ebc87a893479de5feecaf676 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Mon, 11 Mar 2024 17:21:20 +0100 Subject: [PATCH 33/44] Changed x-process-time header generation to include cache access time --- python/python_fastapi_server/main.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/python/python_fastapi_server/main.py b/python/python_fastapi_server/main.py index d0966bde..d91b4e71 100644 --- a/python/python_fastapi_server/main.py +++ b/python/python_fastapi_server/main.py @@ -36,15 +36,6 @@ logging.getLogger("access").propagate = False -@app.middleware("http") -async def add_process_time_header(request: Request, call_next): - start_time = time.time() - response = await call_next(request) - process_time = time.time() - start_time - response.headers["X-Process-Time"] = str(process_time) - return response - - @app.middleware("http") async def add_hsts_header(request: Request, call_next): response = await call_next(request) @@ -74,6 +65,16 @@ async def add_hsts_header(request: Request, call_next): if "EXTERNALADDRESS" in os.environ: app.add_middleware(FixSchemeMiddleware) + +@app.middleware("http") +async def add_process_time_header(request: Request, call_next): + start_time = time.time() + response = await call_next(request) + process_time = time.time() - start_time + response.headers["X-Process-Time"] = str(process_time) + return response + + app.add_middleware(BrotliMiddleware, gzip_fallback=True) From 42094cd26ec9d5b9e3a70212c597ae3f46045e4a Mon Sep 17 00:00:00 2001 From: maartenplieger Date: Wed, 13 Mar 2024 11:07:39 +0100 Subject: [PATCH 34/44] Less logging and force CORS header --- adagucserverEC/CCreateHistogram.cpp | 4 +-- adagucserverEC/CDataReader.cpp | 34 +++++++++---------- adagucserverEC/CDataReader.h | 2 +- adagucserverEC/CImageDataWriter.cpp | 3 +- adagucserverEC/CRequest.cpp | 2 +- python/python_fastapi_server/main.py | 2 ++ .../python_fastapi_server/routers/wmswcs.py | 4 +-- 7 files changed, 26 insertions(+), 25 deletions(-) diff --git a/adagucserverEC/CCreateHistogram.cpp b/adagucserverEC/CCreateHistogram.cpp index 0b9280fc..edbe6739 100644 --- a/adagucserverEC/CCreateHistogram.cpp +++ b/adagucserverEC/CCreateHistogram.cpp @@ -258,10 +258,10 @@ int CCreateHistogram::end() { CT::string resultJSON; if (baseDataSource->srvParams->JSONP.length() == 0) { CDBDebug("CREATING JSON"); - printf("%s%s%c%c\n", "Content-Type: application/json", 13, 10); + printf("%s%s%c%c\n", "Content-Type: application/json", baseDataSource->srvParams->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); } else { CDBDebug("CREATING JSONP %s", baseDataSource->srvParams->JSONP.c_str()); - printf("%s%s%c%c", "Content-Type: application/javascript", 13, 10); + printf("%s%s%c%c", "Content-Type: application/javascript", baseDataSource->srvParams->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); printf("\n%s", baseDataSource->srvParams->JSONP.c_str()); } diff --git a/adagucserverEC/CDataReader.cpp b/adagucserverEC/CDataReader.cpp index a07419e4..7189287c 100755 --- a/adagucserverEC/CDataReader.cpp +++ b/adagucserverEC/CDataReader.cpp @@ -292,10 +292,10 @@ bool CDataReader::copyCRSFromADAGUCProjectionVariable(CDataSource *dataSource, c return false; } - // if (this->_enableReporting) { - // CREPORT_INFO_NODOC(CT::string("Retrieving the projection according to the ADAGUC standards from the proj4_params or proj4 attribute: ") + proj4Attr->toString(), - // CReportMessage::Categories::GENERAL); - // } + if (this->_enableReporting) { + CREPORT_INFO_NODOC(CT::string("Retrieving the projection according to the ADAGUC standards from the proj4_params or proj4 attribute: ") + proj4Attr->toString(), + CReportMessage::Categories::GENERAL); + } dataSource->nativeProj4.copy(proj4Attr->toString().c_str()); // Fixes issue https://github.com/KNMI/adaguc-server/issues/279 @@ -340,13 +340,15 @@ void CDataReader::copyEPSGCodeFromProjectionVariable(CDataSource *dataSource, co // Get EPSG_code CDF::Attribute *epsgAttr = projVar->getAttributeNE("EPSG_code"); if (epsgAttr != NULL) { - CREPORT_INFO_NODOC(CT::string("Using EPSG_code defined in projection variable ") + projVar->name, CReportMessage::Categories::GENERAL); + if (this->_enableReporting) { + CREPORT_INFO_NODOC(CT::string("Using EPSG_code defined in projection variable ") + projVar->name, CReportMessage::Categories::GENERAL); + } dataSource->nativeEPSG.copy((char *)epsgAttr->data); } else { // Make a projection code based on PROJ4: namespace - // if (this->_enableReporting) { - // CREPORT_INFO_NODOC(CT::string("Using projection string to create EPSG code.") + dataSource->nativeProj4, CReportMessage::Categories::GENERAL); - // } + if (this->_enableReporting) { + CREPORT_INFO_NODOC(CT::string("Using projection string to create EPSG code.") + dataSource->nativeProj4, CReportMessage::Categories::GENERAL); + } dataSource->nativeEPSG.print("PROJ4:%s", dataSource->nativeProj4.c_str()); dataSource->nativeEPSG.replaceSelf("\"", ""); dataSource->nativeEPSG.replaceSelf("\n", ""); @@ -717,12 +719,10 @@ bool CDataReader::determineXandYVars(CDataSource *dataSource, const CDF::Variabl CReportMessage::Categories::GENERAL); return false; } - // if (this->_enableReporting) { - // CREPORT_INFO_NODOC( - // CT::string("Using variable ") + dataSource->varX->name + - // CT::string(" as X variable and variable ") + dataSource->varY->name + - // CT::string(" as Y variable."), CReportMessage::Categories::GENERAL); - // } + if (this->_enableReporting) { + CREPORT_INFO_NODOC(CT::string("Using variable ") + dataSource->varX->name + CT::string(" as X variable and variable ") + dataSource->varY->name + CT::string(" as Y variable."), + CReportMessage::Categories::GENERAL); + } return true; } @@ -746,9 +746,9 @@ void CDataReader::determineStride2DMap(CDataSource *dataSource) const { } dataSource->stride2DMap = 1; - // if (this->_enableReporting) { - // CREPORT_INFO_NODOC(CT::string("No stride defined in the RenderSettings, using a default stride of 1."), CReportMessage::Categories::GENERAL); - // } + if (this->_enableReporting) { + CREPORT_INFO_NODOC(CT::string("No stride defined in the RenderSettings, using a default stride of 1."), CReportMessage::Categories::GENERAL); + } return; } diff --git a/adagucserverEC/CDataReader.h b/adagucserverEC/CDataReader.h index f1fb49d8..d1b7bc3e 100644 --- a/adagucserverEC/CDataReader.h +++ b/adagucserverEC/CDataReader.h @@ -121,7 +121,7 @@ class CDataReader { bool calculateCellSizeAndBBox(CDataSource *dataSource, const CDF::Variable *dataSourceVar) const; public: - CDataReader() { _enableReporting = true; } + CDataReader() { _enableReporting = false; } ~CDataReader() {} bool enablePostProcessors = true; // Allows disabling of other postprocessors. For example when a datasource is output of another postprocessor. bool enableObjectCache = true; diff --git a/adagucserverEC/CImageDataWriter.cpp b/adagucserverEC/CImageDataWriter.cpp index e334ec0b..b1e67590 100644 --- a/adagucserverEC/CImageDataWriter.cpp +++ b/adagucserverEC/CImageDataWriter.cpp @@ -3451,7 +3451,6 @@ int CImageDataWriter::end() { printf("%s%s%c%c\n", "Content-Type:image/png", cacheControl.c_str(), 13, 10); status = drawImage.printImagePng32(); } else if (srvParam->imageFormat == IMAGEFORMAT_IMAGEWEBP) { - CDBDebug("Creating 32 bit webp"); printf("%s%s%c%c\n", "Content-Type:image/webp", cacheControl.c_str(), 13, 10); int webPQuality = srvParam->imageQuality; if (!srvParam->Format.empty()) { @@ -3464,7 +3463,7 @@ int CImageDataWriter::end() { } } } - CDBDebug("webPQuality = %d", webPQuality); + CDBDebug("Creating 32 bit webp quality = %d", webPQuality); status = drawImage.printImageWebP32(webPQuality); } else if (srvParam->imageFormat == IMAGEFORMAT_IMAGEGIF) { // CDBDebug("LegendGraphic GIF"); diff --git a/adagucserverEC/CRequest.cpp b/adagucserverEC/CRequest.cpp index bf50b2d1..796f5445 100644 --- a/adagucserverEC/CRequest.cpp +++ b/adagucserverEC/CRequest.cpp @@ -1496,7 +1496,7 @@ int CRequest::queryDimValuesForDataSource(CDataSource *dataSource, CServerParams maxQueryResultLimit = dataSource->cfgLayer->FilePath[0]->attr.maxquerylimit.toInt(); } } - CDBDebug("Using maxquerylimit %d", maxQueryResultLimit); + // CDBDebug("Using maxquerylimit %d", maxQueryResultLimit); store = CDBFactory::getDBAdapter(srvParam->cfg)->getFilesAndIndicesForDimensions(dataSource, maxQueryResultLimit); } diff --git a/python/python_fastapi_server/main.py b/python/python_fastapi_server/main.py index d91b4e71..e826d42d 100644 --- a/python/python_fastapi_server/main.py +++ b/python/python_fastapi_server/main.py @@ -72,6 +72,8 @@ async def add_process_time_header(request: Request, call_next): response = await call_next(request) process_time = time.time() - start_time response.headers["X-Process-Time"] = str(process_time) + # TODO: Temporary fix to enable CORS, it seems that CORSMiddleware does not always send this information + response.headers["Access-Control-Allow-Origin"] = "*" return response diff --git a/python/python_fastapi_server/routers/wmswcs.py b/python/python_fastapi_server/routers/wmswcs.py index 8361873d..a87a1b67 100644 --- a/python/python_fastapi_server/routers/wmswcs.py +++ b/python/python_fastapi_server/routers/wmswcs.py @@ -27,7 +27,7 @@ async def handle_wms( # logger.info("instance: %s", str(adaguc_instance)) url = req.url - logger.info(req.url) + # logger.info(req.url) adagucenv = {} @@ -61,7 +61,7 @@ async def handle_wms( adaguc_instance.removeLogFile() if len(logfile) > 0: - logger.info(logfile) + logger.info(f":\n{logfile}") response_code = 200 # Note: The Adaguc implementation requires non-zero status codes to correspond to the From 29720bca4e73b54f1cb02bc1a48d3127443c6ae9 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Wed, 13 Mar 2024 14:54:16 +0100 Subject: [PATCH 35/44] Changed order of middlewares, so CORS is last in code --- python/python_fastapi_server/main.py | 14 +++++++------- python/python_fastapi_server/routers/middleware.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/python/python_fastapi_server/main.py b/python/python_fastapi_server/main.py index d91b4e71..6e6407a4 100644 --- a/python/python_fastapi_server/main.py +++ b/python/python_fastapi_server/main.py @@ -52,13 +52,6 @@ async def add_hsts_header(request: Request, call_next): return response -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_methods=["*"], - allow_headers=["*"], -) - if "ADAGUC_REDIS" in os.environ: app.add_middleware(CachingMiddleware) @@ -77,6 +70,13 @@ async def add_process_time_header(request: Request, call_next): app.add_middleware(BrotliMiddleware, gzip_fallback=True) +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + @app.get("/") async def root(): diff --git a/python/python_fastapi_server/routers/middleware.py b/python/python_fastapi_server/routers/middleware.py index 9a7194fb..42399178 100644 --- a/python/python_fastapi_server/routers/middleware.py +++ b/python/python_fastapi_server/routers/middleware.py @@ -3,6 +3,7 @@ For use behind proxies """ + import os from urllib.parse import urlsplit @@ -20,8 +21,7 @@ def __init__(self, app: ASGIApp): external_url = urlsplit(external_address) self.scheme = external_url.scheme - async def __call__(self, scope: Scope, receive: Receive, - send: Send) -> None: + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: if scope["type"] == "http": if self.scheme: scope["scheme"] = self.scheme From c919b881e915b8487f6c9c8cbf41e5acf4b4c862 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Wed, 13 Mar 2024 16:43:15 +0100 Subject: [PATCH 36/44] More review comments --- python/lib/adaguc/runAdaguc.py | 14 ++-- .../routers/caching_middleware.py | 4 +- python/python_fastapi_server/routers/edr.py | 67 +++++++++---------- 3 files changed, 44 insertions(+), 41 deletions(-) diff --git a/python/lib/adaguc/runAdaguc.py b/python/lib/adaguc/runAdaguc.py index 409c321d..76b589f2 100644 --- a/python/lib/adaguc/runAdaguc.py +++ b/python/lib/adaguc/runAdaguc.py @@ -8,7 +8,7 @@ import shutil import random import string -import redis # This can also be used to connect to a Redis cluster +import redis.asyncio as redis # This can also be used to connect to a Redis cluster # from redis.cluster import RedisCluster as Redis # Cluster client, for testing @@ -215,7 +215,9 @@ async def runADAGUCServer( if self.cache_wanted(url): cache_key = str((url, adagucargs)).encode("utf-8") - age, headers, data = get_cached_response(runAdaguc.redis_pool, cache_key) + age, headers, data = await get_cached_response( + runAdaguc.redis_pool, cache_key + ) if age is not None: return [0, data, headers] @@ -319,10 +321,10 @@ def response_to_cache(redis_pool, key, headers: str, data): redis_client.close() -def get_cached_response(redis_pool, key): +async def get_cached_response(redis_pool, key): redis_client = redis.Redis(connection_pool=redis_pool) - cached = redis_client.get(key) - redis_client.close() + cached = await redis_client.get(key) + redis_client.aclose() if not cached: return None, None, None @@ -331,7 +333,7 @@ def get_cached_response(redis_pool, key): age = currenttime - entrytime headers_len = int(cached[10:16].decode("utf-8")) - headers = json.loads(cached[16 : 16 + headers_len]) + headers = json.loads(cached[16 : 16 + headers_len].decode["utf-8"]) headers.append(f"age: {age}") data = brotli.decompress(cached[16 + headers_len :]) diff --git a/python/python_fastapi_server/routers/caching_middleware.py b/python/python_fastapi_server/routers/caching_middleware.py index 5c9f45ce..0108d7f3 100644 --- a/python/python_fastapi_server/routers/caching_middleware.py +++ b/python/python_fastapi_server/routers/caching_middleware.py @@ -17,6 +17,8 @@ ADAGUC_REDIS = os.environ.get("ADAGUC_REDIS") +MAX_SIZE_FOR_CACHING = 10000000 + async def get_cached_response(redis_pool, request): key = generate_key(request) @@ -53,7 +55,7 @@ async def response_to_cache(redis_pool, request, headers, data, ex: int): "utf-8" ) compressed_data = brotli.compress(data) - if len(compressed_data) < 10000000: + if len(compressed_data) < MAX_SIZE_FOR_CACHING: redis_client = redis.Redis(connection_pool=redis_pool) await redis_client.set( key, diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index 57a8cd47..698aab11 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -13,7 +13,7 @@ import logging import os import re -from datetime import datetime, timezone, timedelta +from datetime import datetime, timezone from typing import Union from covjson_pydantic.coverage import Coverage @@ -47,14 +47,12 @@ from fastapi import FastAPI, Query, Request, Response from fastapi.responses import JSONResponse -from fastapi.encoders import jsonable_encoder from fastapi.openapi.utils import get_openapi from geomet import wkt from owslib.wms import WebMapService from pydantic import AwareDatetime -from redis import from_url -from functools import partial +from cachetools import cached, TTLCache from .covjsonresponse import CovJSONResponse from .ogcapi_tools import call_adaguc @@ -161,16 +159,15 @@ def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DI return edr_collections -edr_collections = None +# The edr_collections information is cached locally for a maximum of 2 minutes +# It will be refreshed if older than 2 minutes +edr_cache = TTLCache(maxsize=100, ttl=120) +@cached(cache=edr_cache) def get_edr_collections(): """Returns all EDR collections""" - global edr_collections - if edr_collections is None: - edr_collections = init_edr_collections() - - return edr_collections + return init_edr_collections() async def get_point_value( @@ -259,7 +256,7 @@ async def get_collection_position( ) if resp: dat = json.loads(resp) - ttl = get_age_from_adaguc_headers(headers) + ttl = get_ttl_from_adaguc_headers(headers) if ttl is not None: response.headers["cache-control"] = generate_max_age(ttl) return covjson_from_resp(dat, edr_collections[collection_name]["vertical_name"]) @@ -281,7 +278,7 @@ async def get_collectioninfo_for_id( Returns collection information for a given collection id and or instance Is used to obtain metadata from the dataset configuration and WMS GetCapabilities document. """ - logger.info("get_collection_info_for_id(%s, %s)", edr_collection, instance) + logger.info("get_collectioninfo_for_id(%s, %s)", edr_collection, instance) edr_collectioninfo = get_edr_collections()[edr_collection] dataset = edr_collectioninfo["dataset"] logger.info("%s=>%s", edr_collection, dataset) @@ -309,17 +306,17 @@ async def get_collectioninfo_for_id( wmslayers, ttl = await get_capabilities(edr_collectioninfo["name"]) - bbox = await get_extent(edr_collectioninfo, wmslayers) + bbox = get_extent(edr_collectioninfo, wmslayers) if bbox is None: return None, None crs = 'GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]' spatial = Spatial(bbox=bbox, crs=crs) - (interval, time_values) = await get_times_for_collection( + (interval, time_values) = get_times_for_collection( edr_collectioninfo, wmslayers, edr_collectioninfo["parameters"][0]["name"] ) - customlist: list = await get_custom_dims_for_collection( + customlist: list = get_custom_dims_for_collection( edr_collectioninfo, wmslayers, edr_collectioninfo["parameters"][0]["name"] ) @@ -330,7 +327,7 @@ async def get_collectioninfo_for_id( custom.append(Custom(**custom_el)) vertical = None - vertical_dim = await get_vertical_dim_for_collection( + vertical_dim = get_vertical_dim_for_collection( edr_collectioninfo, wmslayers, edr_collectioninfo["parameters"][0]["name"] ) if vertical_dim: @@ -498,7 +495,7 @@ def get_time_values_for_range(rng: str) -> list[str]: return [f"R{nsteps}/{iso_start}/{step}"] -async def get_times_for_collection( +def get_times_for_collection( edr_collectioninfo: dict, wmslayers, parameter: str = None ) -> tuple[list[list[str]], list[str]]: """ @@ -526,7 +523,7 @@ async def get_times_for_collection( ), ] ] - return (interval, get_time_values_for_range(time_dim["values"][0])) + return interval, get_time_values_for_range(time_dim["values"][0]) interval = [ [ datetime.strptime(time_dim["values"][0], "%Y-%m-%dT%H:%M:%SZ").replace( @@ -538,10 +535,10 @@ async def get_times_for_collection( ] ] return interval, time_dim["values"] - return (None, None) + return None, None -async def get_custom_dims_for_collection( +def get_custom_dims_for_collection( edr_collectioninfo: dict, wmslayers, parameter: str = None ): """ @@ -571,7 +568,7 @@ async def get_custom_dims_for_collection( return custom if len(custom) > 0 else None -async def get_vertical_dim_for_collection( +def get_vertical_dim_for_collection( edr_collectioninfo: dict, wmslayers, parameter: str = None ): """ @@ -609,19 +606,19 @@ async def rest_get_edr_collections(request: Request, response: Response): links.append(self_link) collections: list[Collection] = [] - min_ttl = None + ttl_set = set() edr_collections = get_edr_collections() for edr_coll in edr_collections: coll, ttl = await get_collectioninfo_for_id(edr_coll) if coll: collections.append(coll) if ttl is not None: - min_ttl = ttl if min_ttl is None else min(min_ttl, ttl) + ttl_set.add(ttl) else: logger.warning("Unable to fetch WMS GetCapabilities for %s", edr_coll) collections_data = Collections(links=links, collections=collections) - if min_ttl is not None: - response.headers["cache-control"] = generate_max_age(min_ttl) + if ttl_set: + response.headers["cache-control"] = generate_max_age(min(ttl_set)) return collections_data @@ -635,11 +632,12 @@ async def rest_get_edr_collection_by_id(collection_name: str, response: Response GET Returns collection information for given collection id """ collection, ttl = await get_collectioninfo_for_id(collection_name) - response.headers["cache-control"] = generate_max_age(ttl) + if ttl is not None: + response.headers["cache-control"] = generate_max_age(ttl) return collection -def get_age_from_adaguc_headers(headers): +def get_ttl_from_adaguc_headers(headers): """Derives an age value from the max-age/age cache headers that ADAGUC executable returns""" try: max_age = None @@ -675,7 +673,7 @@ async def get_capabilities(collname): f"dataset={dataset}&service=wms&version=1.3.0&request=getcapabilities" ) status, response, headers = await call_adaguc(url=urlrequest.encode("UTF-8")) - ttl = get_age_from_adaguc_headers(headers) + ttl = get_ttl_from_adaguc_headers(headers) logger.info("status: %d", status) if status == 0: xml = response.getvalue() @@ -714,7 +712,7 @@ async def get_ref_times_for_coll(edr_collectioninfo: dict, layer: str) -> list[s return [] -async def get_extent(edr_collectioninfo: dict, wmslayers): +def get_extent(edr_collectioninfo: dict, wmslayers): """ Get the boundingbox extent from the WMS GetCapabilities """ @@ -754,19 +752,19 @@ async def rest_get_edr_inst_for_coll( ) links: list[Link] = [] links.append(Link(href=instances_url, rel="collection")) - min_ttl = None + ttl_set = set() for instance in list(ref_times): instance_links: list[Link] = [] instance_link = Link(href=f"{instances_url}/{instance}", rel="collection") instance_links.append(instance_link) instance_info, ttl = await get_collectioninfo_for_id(collection_name, instance) if ttl is not None: - min_ttl = ttl if min_ttl is None else min(min_ttl, ttl) + ttl_set.add(ttl) instances.append(instance_info) instances_data = Instances(instances=instances, links=links) - if min_ttl is not None: - response.headers["cache-control"] = generate_max_age(min_ttl) + if ttl_set: + response.headers["cache-control"] = generate_max_age(min(ttl_set)) return instances_data @@ -780,7 +778,8 @@ async def rest_get_collection_info(collection_name: str, instance, response: Res GET "/collections/{collection_name}/instances/{instance}" """ coll, ttl = await get_collectioninfo_for_id(collection_name, instance) - response.headers["cache-control"] = generate_max_age(ttl) + if ttl is not None: + response.headers["cache-control"] = generate_max_age(ttl) return coll From 3618658c90407d78b3e47b332a19d302e2bbf382 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Wed, 13 Mar 2024 16:46:15 +0100 Subject: [PATCH 37/44] Typo --- adagucserverEC/CCreateHistogram.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/adagucserverEC/CCreateHistogram.cpp b/adagucserverEC/CCreateHistogram.cpp index edbe6739..58dd59f4 100644 --- a/adagucserverEC/CCreateHistogram.cpp +++ b/adagucserverEC/CCreateHistogram.cpp @@ -15,7 +15,7 @@ int CCreateHistogram::createHistogram(CDataSource *dataSource, CDrawImage *) { } else { CDBDebug("CREATING JSONP %s", dataSource->srvParams->JSONP.c_str()); printf("%s%s%c%c", "Content-Type: application/javascript", dataSource->srvParams->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); - printf("\n%s()", dataSource->srvParams->JSONP.c_str()); + printf("\n%s(", dataSource->srvParams->JSONP.c_str()); } // puts("{\"a\": 1}"); @@ -262,7 +262,7 @@ int CCreateHistogram::end() { } else { CDBDebug("CREATING JSONP %s", baseDataSource->srvParams->JSONP.c_str()); printf("%s%s%c%c", "Content-Type: application/javascript", baseDataSource->srvParams->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10); - printf("\n%s", baseDataSource->srvParams->JSONP.c_str()); + printf("\n%s(", baseDataSource->srvParams->JSONP.c_str()); } puts(JSONdata.c_str()); From 7d0a1699c878019af1b7e6f5f73f5ca19b026582 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Wed, 13 Mar 2024 16:48:53 +0100 Subject: [PATCH 38/44] Removed temporary Access-Control-Allow-Origin header --- python/python_fastapi_server/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/python_fastapi_server/main.py b/python/python_fastapi_server/main.py index 8c3d2774..6e6407a4 100644 --- a/python/python_fastapi_server/main.py +++ b/python/python_fastapi_server/main.py @@ -65,8 +65,6 @@ async def add_process_time_header(request: Request, call_next): response = await call_next(request) process_time = time.time() - start_time response.headers["X-Process-Time"] = str(process_time) - # TODO: Temporary fix to enable CORS, it seems that CORSMiddleware does not always send this information - response.headers["Access-Control-Allow-Origin"] = "*" return response From 21faf519d0cd8caba1be3b92557dce9bded3bd37 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Wed, 13 Mar 2024 18:46:57 +0100 Subject: [PATCH 39/44] More review comments --- python/lib/adaguc/runAdaguc.py | 34 +++++++------------ .../routers/caching_middleware.py | 7 ++-- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/python/lib/adaguc/runAdaguc.py b/python/lib/adaguc/runAdaguc.py index 76b589f2..0ceb947e 100644 --- a/python/lib/adaguc/runAdaguc.py +++ b/python/lib/adaguc/runAdaguc.py @@ -1,7 +1,5 @@ import asyncio -from ast import Dict import os -from os.path import expanduser from PIL import Image from io import BytesIO import brotli @@ -10,12 +8,9 @@ import string import redis.asyncio as redis # This can also be used to connect to a Redis cluster -# from redis.cluster import RedisCluster as Redis # Cluster client, for testing - -import re import calendar import json -from datetime import datetime, timedelta +from datetime import datetime from adaguc.CGIRunner import CGIRunner @@ -219,7 +214,7 @@ async def runADAGUCServer( runAdaguc.redis_pool, cache_key ) if age is not None: - return [0, data, headers] + return 0, data, headers filetogenerate = BytesIO() status, headers, processErr = await CGIRunner().run( @@ -251,14 +246,13 @@ async def runADAGUCServer( print("Process: No HTTP Headers written") print("--- END ADAGUC DEBUG INFO ---\n") - return status, filetogenerate, headers - else: - if self.cache_wanted(url): - response_to_cache( - runAdaguc.redis_pool, cache_key, headers, filetogenerate - ) - return [status, filetogenerate, headers] + # Only cache if status==0 + if self.cache_wanted(url) and status == 0: + await response_to_cache( + runAdaguc.redis_pool, cache_key, headers, filetogenerate + ) + return status, filetogenerate, headers def writetofile(self, filename, data): with open(filename, "wb") as f: @@ -286,7 +280,7 @@ def mkdir_p(self, directory): headers_to_skip = ["x-process-time", "age"] -def response_to_cache(redis_pool, key, headers: str, data): +async def response_to_cache(redis_pool, key, headers: str, data): cacheable_headers = [] ttl = 0 for header in headers: @@ -302,15 +296,13 @@ def response_to_cache(redis_pool, key, headers: str, data): pass if ttl > 0: - cacheable_headers_json = json.dumps( - cacheable_headers, ensure_ascii=True - ).encode("utf-8") + cacheable_headers_json = json.dumps(cacheable_headers).encode("utf-8") entrytime = f"{calendar.timegm(datetime.utcnow().utctimetuple()):10d}".encode( "utf-8" ) redis_client = redis.Redis(connection_pool=redis_pool) - redis_client.set( + await redis_client.set( key, entrytime + f"{len(cacheable_headers_json):06d}".encode("utf-8") @@ -318,13 +310,13 @@ def response_to_cache(redis_pool, key, headers: str, data): + brotli.compress(data.getvalue()), ex=ttl, ) - redis_client.close() + await redis_client.aclose() async def get_cached_response(redis_pool, key): redis_client = redis.Redis(connection_pool=redis_pool) cached = await redis_client.get(key) - redis_client.aclose() + await redis_client.aclose() if not cached: return None, None, None diff --git a/python/python_fastapi_server/routers/caching_middleware.py b/python/python_fastapi_server/routers/caching_middleware.py index 0108d7f3..df2d61ca 100644 --- a/python/python_fastapi_server/routers/caching_middleware.py +++ b/python/python_fastapi_server/routers/caching_middleware.py @@ -1,6 +1,5 @@ from io import BytesIO import os -from urllib.parse import urlsplit from starlette.middleware.base import BaseHTTPMiddleware from starlette.responses import Response @@ -10,14 +9,12 @@ from datetime import datetime import redis.asyncio as redis # This can also be used to connect to a Redis cluster -# from redis.asyncio.cluster import RedisCluster as Redis # Cluster client, for testing - import json import brotli ADAGUC_REDIS = os.environ.get("ADAGUC_REDIS") -MAX_SIZE_FOR_CACHING = 10000000 +MAX_SIZE_FOR_CACHING = 10_000_000 async def get_cached_response(redis_pool, request): @@ -93,7 +90,7 @@ async def dispatch(self, request, call_next): if data: # Fix Age header - headers["Age"] = "%1d" % (age) + headers["Age"] = f"{age:1d}" headers["adaguc-cache"] = "hit" return Response( content=data, From 9c5666a8b2317c07bbc156c9857769c3c2d7299b Mon Sep 17 00:00:00 2001 From: maartenplieger Date: Thu, 14 Mar 2024 15:59:55 +0100 Subject: [PATCH 40/44] Added () instead of [] after decode function --- python/lib/adaguc/runAdaguc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/adaguc/runAdaguc.py b/python/lib/adaguc/runAdaguc.py index 0ceb947e..a14c6f30 100644 --- a/python/lib/adaguc/runAdaguc.py +++ b/python/lib/adaguc/runAdaguc.py @@ -325,7 +325,7 @@ async def get_cached_response(redis_pool, key): age = currenttime - entrytime headers_len = int(cached[10:16].decode("utf-8")) - headers = json.loads(cached[16 : 16 + headers_len].decode["utf-8"]) + headers = json.loads(cached[16 : 16 + headers_len].decode("utf-8")) headers.append(f"age: {age}") data = brotli.decompress(cached[16 + headers_len :]) From 6ed3f9d0c6f9dcb9cc68db08f3fe30381378f4ae Mon Sep 17 00:00:00 2001 From: maartenplieger Date: Thu, 14 Mar 2024 16:25:13 +0100 Subject: [PATCH 41/44] Removed ADAGUC_REDIS_PORT --- Docker/docker-compose-generate-env.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/Docker/docker-compose-generate-env.sh b/Docker/docker-compose-generate-env.sh index 858ac58d..8f4cb8a4 100644 --- a/Docker/docker-compose-generate-env.sh +++ b/Docker/docker-compose-generate-env.sh @@ -2,7 +2,6 @@ ADAGUC_PORT=443 -ADAGUC_REDIS_PORT=6379 ADAGUC_DATA_DIR=${HOME}/adaguc-docker/adaguc-data ADAGUC_AUTOWMS_DIR=${HOME}/adaguc-docker/adaguc-autowms ADAGUC_DATASET_DIR=${HOME}/adaguc-docker/adaguc-datasets From 4bf614fb01c65134e03e52cafd03720d1c148934 Mon Sep 17 00:00:00 2001 From: maartenplieger Date: Thu, 14 Mar 2024 16:50:59 +0100 Subject: [PATCH 42/44] Updated news and version --- Dockerfile | 2 +- NEWS.md | 9 ++++++++- adagucserverEC/Definitions.h | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0976000d..24bf4af3 100755 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ USER root LABEL maintainer="adaguc@knmi.nl" # Version should be same as in Definitions.h -LABEL version="2.20.2" +LABEL version="2.21.0" # Try to update image packages RUN apt-get -q -y update \ diff --git a/NEWS.md b/NEWS.md index b232b759..31043376 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,10 @@ +**Version 2.21.0 2024-03-14** + +- Added support for Redis caching. Redis caching can be enabled by providing a Redis service via the ADAGUC_REDIS environment and configuring caching settings for a dataset in the [Settings](doc/configuration/Settings.md) element. +- Improved speed of EDR service and added support to cache EDR calls +- Various performance improvements + + **Version 2.20.2 2024-02-28** - Removed locking mechanism @@ -242,7 +249,7 @@ Version 2.6.1 2021-11-09 - Moving to a python wrapper to run adaguc-server. For development with flask, for production with gunicorn. **Version 2.5.15 2021-09-22** -- The /data folder can now completely be mounted from another filesystem, all adaguc internal folders have moved to /adaguc. +- The /data folder can now completely2.20.2 be mounted from another filesystem, all adaguc internal folders have moved to /adaguc. **Version 2.5.14 2021-09-17** - WCS GetCoverage with Native parameters now returns data diff --git a/adagucserverEC/Definitions.h b/adagucserverEC/Definitions.h index af6d6629..a8ec940b 100755 --- a/adagucserverEC/Definitions.h +++ b/adagucserverEC/Definitions.h @@ -28,7 +28,7 @@ #ifndef Definitions_H #define Definitions_H -#define ADAGUCSERVER_VERSION "2.20.2" // Please also update in the Dockerfile to the same version +#define ADAGUCSERVER_VERSION "2.21.0" // Please also update in the Dockerfile to the same version // CConfigReaderLayerType #define CConfigReaderLayerTypeUnknown 0 From da337c48f873a126124157e353fcb3e42deebe86 Mon Sep 17 00:00:00 2001 From: maartenplieger Date: Thu, 14 Mar 2024 17:16:22 +0100 Subject: [PATCH 43/44] Opendap works again --- python/python_fastapi_server/routers/opendap.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/python/python_fastapi_server/routers/opendap.py b/python/python_fastapi_server/routers/opendap.py index f2275159..eef35481 100644 --- a/python/python_fastapi_server/routers/opendap.py +++ b/python/python_fastapi_server/routers/opendap.py @@ -1,4 +1,5 @@ """opendapRouter""" + import logging import os @@ -12,7 +13,7 @@ @opendapRouter.get("/adagucopendap/{opendappath:path}") -def handle_opendap(req: Request, opendappath: str): +async def handle_opendap(req: Request, opendappath: str): logger.info(opendappath) adaguc_instance = setup_adaguc() url = req.url @@ -25,17 +26,19 @@ def handle_opendap(req: Request, opendappath: str): # Set required environment variables base_url = f"{url.scheme}://{url.hostname}:{url.port}" adagucenv["ADAGUC_ONLINERESOURCE"] = ( - os.getenv("EXTERNALADDRESS", base_url) + "/adagucopendap?") + os.getenv("EXTERNALADDRESS", base_url) + "/adagucopendap?" + ) adagucenv["ADAGUC_DB"] = os.getenv( - "ADAGUC_DB", - "user=adaguc password=adaguc host=localhost dbname=adaguc") + "ADAGUC_DB", "user=adaguc password=adaguc host=localhost dbname=adaguc" + ) logger.info("Setting request_uri to %s", base_url) adagucenv["REQUEST_URI"] = url.path adagucenv["SCRIPT_NAME"] = "" - status, data, headers = adaguc_instance.runADAGUCServer( - query_string, env=adagucenv, showLogOnError=False) + status, data, headers = await adaguc_instance.runADAGUCServer( + query_string, env=adagucenv, showLogOnError=False + ) # Obtain logfile logfile = adaguc_instance.getLogFile() From 27f38f10fad92900bf075f2551ff15fca65d2ee7 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Mon, 18 Mar 2024 14:36:35 +0100 Subject: [PATCH 44/44] Remarks from review --- Docker/docker-compose-generate-env.sh | 6 +----- NEWS.md | 2 +- python/python_fastapi_server/routers/caching_middleware.py | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Docker/docker-compose-generate-env.sh b/Docker/docker-compose-generate-env.sh index 8f4cb8a4..b43ff501 100644 --- a/Docker/docker-compose-generate-env.sh +++ b/Docker/docker-compose-generate-env.sh @@ -7,7 +7,7 @@ ADAGUC_AUTOWMS_DIR=${HOME}/adaguc-docker/adaguc-autowms ADAGUC_DATASET_DIR=${HOME}/adaguc-docker/adaguc-datasets ADAGUC_NUMPARALLELPROCESSES=4 -usage() { echo "Usage: $0 -p -e -a -d -f -t -r " 1>&2; exit 1; } +usage() { echo "Usage: $0 -p -e -a -d -f -t " 1>&2; exit 1; } while getopts ":e:p:h:a:d:f:t:" o; do @@ -27,9 +27,6 @@ while getopts ":e:p:h:a:d:f:t:" o; do f) ADAGUC_DATA_DIR=${OPTARG} ;; - r) - REDIS_PORT=${OPTARG} - ;; t) ADAGUC_NUMPARALLELPROCESSES=${OPTARG} ;; @@ -62,7 +59,6 @@ echo "ADAGUC_DATASET_DIR=${ADAGUC_DATASET_DIR}" >> .env echo "ADAGUC_NUMPARALLELPROCESSES=${ADAGUC_NUMPARALLELPROCESSES}" >> .env echo "ADAGUC_PORT=${ADAGUC_PORT}" >> .env echo "EXTERNALADDRESS=${EXTERNALADDRESS}" >> .env -echo "REDIS_PORT=${REDIS_PORT} >>.env echo "############### env file ###############" cat .env echo "############### env file ###############" diff --git a/NEWS.md b/NEWS.md index 31043376..60457b4d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -249,7 +249,7 @@ Version 2.6.1 2021-11-09 - Moving to a python wrapper to run adaguc-server. For development with flask, for production with gunicorn. **Version 2.5.15 2021-09-22** -- The /data folder can now completely2.20.2 be mounted from another filesystem, all adaguc internal folders have moved to /adaguc. +- The /data folder can now completely be mounted from another filesystem, all adaguc internal folders have moved to /adaguc. **Version 2.5.14 2021-09-17** - WCS GetCoverage with Native parameters now returns data diff --git a/python/python_fastapi_server/routers/caching_middleware.py b/python/python_fastapi_server/routers/caching_middleware.py index df2d61ca..0c18de12 100644 --- a/python/python_fastapi_server/routers/caching_middleware.py +++ b/python/python_fastapi_server/routers/caching_middleware.py @@ -90,7 +90,7 @@ async def dispatch(self, request, call_next): if data: # Fix Age header - headers["Age"] = f"{age:1d}" + headers["Age"] = f"{age}" headers["adaguc-cache"] = "hit" return Response( content=data,