From 9ceff56093f151e0823eec2fe43648d42cff1b9b Mon Sep 17 00:00:00 2001 From: David Marteau Date: Tue, 15 Feb 2022 21:18:43 +0100 Subject: [PATCH] Add BAN cache observer --- CHANGELOG.md | 1 + pyqgisserver/config.py | 1 + pyqgisserver/qgscache/observer.py | 6 ++- pyqgisserver/qgscache/observers/ban.py | 54 ++++++++++++++++++++ pyqgisserver/qgscache/observers/test.py | 3 ++ pyqgisserver/qgsworker.py | 8 +-- setup.py | 3 +- tests/Makefile | 16 +++--- tests/docker-compose.amqp.yml | 4 +- tests/docker-compose.postgres.yml | 2 + tests/docker-compose.varnish.yml | 28 +++++++++++ tests/docker-compose.yml | 12 ++++- tests/varnish.secret | 1 + tests/varnish.vcl | 67 +++++++++++++++++++++++++ 14 files changed, 185 insertions(+), 21 deletions(-) create mode 100644 pyqgisserver/qgscache/observers/ban.py create mode 100644 tests/docker-compose.varnish.yml create mode 100644 tests/varnish.secret create mode 100644 tests/varnish.vcl diff --git a/CHANGELOG.md b/CHANGELOG.md index 5975f90..03b3a04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +* Add BAN cache observer * Fix parameter's case in OGC API requests - Fix https://github.com/3liz/py-qgis-server/issues/34 * Implement configurable cache observers diff --git a/pyqgisserver/config.py b/pyqgisserver/config.py index 313275d..7580492 100644 --- a/pyqgisserver/config.py +++ b/pyqgisserver/config.py @@ -245,6 +245,7 @@ def __get_impl( self, _get_fun, section:str, option:str, fallback:Any = NO_DEFAU # Look in environment # Note that the section must exists if self.allow_env: + LOGGER.debug("Looking for option '%s.%s' in environment", section, option) varname = 'QGSRV_%s_%s' % (section.upper(), option.upper()) varname = functools.reduce( lambda s,c: s.replace(c,'_'), ENV_REPLACE_CHARS, varname) varvalue = os.getenv(varname) diff --git a/pyqgisserver/qgscache/observer.py b/pyqgisserver/qgscache/observer.py index 59a5fa2..aee4166 100644 --- a/pyqgisserver/qgscache/observer.py +++ b/pyqgisserver/qgscache/observer.py @@ -109,7 +109,9 @@ def _load_observers(): for name in self._declared_observers: try: LOGGER.debug("*** Loading cache observer '%s'", name) - yield cm.load_entrypoint('py_qgis_server.cache.observers',name) + observer = cm.load_entrypoint('py_qgis_server.cache.observers',name) + observer.init() + yield observer except cm.EntryPointNotFoundError: LOGGER.error("Failed to load cache trigger component: %s", name) @@ -168,7 +170,7 @@ def notify_observers(self, key: str, modified_time: datetime, state = UpdateStat """ for obs in self._observers: try: - obs(key, modified_time, state == UpdateState.INSERTED) + obs.observe(key, modified_time, state == UpdateState.INSERTED) except Exception: LOGGER.critical("Uncaugh error in observer: %s\n%s", obs, traceback.format_exc()) diff --git a/pyqgisserver/qgscache/observers/ban.py b/pyqgisserver/qgscache/observers/ban.py new file mode 100644 index 0000000..21e0ba7 --- /dev/null +++ b/pyqgisserver/qgscache/observers/ban.py @@ -0,0 +1,54 @@ +# Copyright 2022 3liz +# Author David Marteau +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +""" Cache Observer that send a BAN request +""" +import asyncio +import logging + +from datetime import datetime +from tornado.httpclient import AsyncHTTPClient, HTTPRequest + +from pyqgisserver.config import confservice + +LOGGER=logging.getLogger('SRVLOG') + +server_address = None +http_client = None + +def init() -> None: + """ + """ + LOGGER.debug("*** Initializing ban observer") + confservice.add_section('cache.observers:ban') + + global server_address, http_client + server_address = confservice.get('cache.observers:ban','server_address') + http_client = AsyncHTTPClient() + + LOGGER.debug("*** Ban observer: sending_request to %s", server_address) + +async def ban( key: str) -> None: + """ Ban key + """ + LOGGER.info("Sending BAN request to %s", server_address) + + request = HTTPRequest(server_address, method='BAN', + headers={ 'X-Map-Id': key }, + user_agent="py-qgis-server; ban observer", + allow_nonstandard_methods=True) + + response = await http_client.fetch(request, raise_error=False) + if response.code != 200: + LOGGER.error("Ban server returned status code %s", response.code) + + + +def observe(key: str, datetime: datetime, insert: bool) -> None: + asyncio.create_task(ban(key)) + diff --git a/pyqgisserver/qgscache/observers/test.py b/pyqgisserver/qgscache/observers/test.py index 1d52197..dad14d9 100644 --- a/pyqgisserver/qgscache/observers/test.py +++ b/pyqgisserver/qgscache/observers/test.py @@ -16,6 +16,9 @@ notify_data = {} +def init() -> None: + pass + def observe(key: str, datetime: datetime, insert: bool) -> None: LOGGER.debug("*** TEST CACHE OBSERVER: Received update notification for %s %s [Inserted: %s]", key, datetime, insert) notify_data[key] = (key,datetime,insert) diff --git a/pyqgisserver/qgsworker.py b/pyqgisserver/qgsworker.py index 6aefab5..8724bce 100644 --- a/pyqgisserver/qgsworker.py +++ b/pyqgisserver/qgsworker.py @@ -268,7 +268,7 @@ def init_server(cls) -> None: @classmethod def refresh_cache(cls) -> None: - if cls._cache_check_interval <= 0 or time() - cls._cache_check_refresh < cls._cache_check_interval: + if cls._cache_check_interval <= 0 or time() - cls._cache_last_check < cls._cache_check_interval: return LOGGER.debug("Refreshing cache") cls._cache_service.refresh() @@ -397,9 +397,9 @@ def handle_qgis_request(self, ogc_scheme: str, project_location: str, request: R # Set the project uri in separate header, this # is useful for invalidating front-end cache from - # key - response.setExtraHeader('X-Qgis-Project-Uri', project_location) - response.setExtraHeader('Last-Modified' , last_modified.astimezone().isoformat()) + # ke + response.setExtraHeader('X-Map-Id' , project_location) + response.setExtraHeader('Last-Modified' , last_modified.astimezone().isoformat()) # Check etag for OWS requests if ogc_scheme == 'OWS': diff --git a/setup.py b/setup.py index 0e586e7..0772d8d 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,8 @@ def get_version(): 'test = pyqgisserver.monitors.test', ], 'py_qgis_server.cache.observers':[ - 'test = pyqgisserver.qgscache.observers.test:observe', + 'test = pyqgisserver.qgscache.observers.test', + 'ban = pyqgisserver.qgscache.observers.ban', ] }, # Add manifest to main package diff --git a/tests/Makefile b/tests/Makefile index 47bde88..5bf55b7 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -90,18 +90,14 @@ up: stop: docker compose down -v --remove-orphans -amqp-run: export RUN_COMMAND=./tests/run_server.sh -amqp-run: - docker compose \ - -f docker-compose.yml \ - -f docker-compose.amqp.yml up -V --remove-orphans --force-recreate -d - -postgres-run: export RUN_COMMAND=./tests/run_server.sh -postgres-run: +# +# Run server with extra services +# +%-run: export RUN_COMMAND=./tests/run_server.sh +%-run: docker compose \ -f docker-compose.yml \ - -f docker-compose.postgres.yml up -V --remove-orphans --force-recreate -d - + -f docker-compose.$*.yml up -V --remove-orphans --force-recreate -d diff --git a/tests/docker-compose.amqp.yml b/tests/docker-compose.amqp.yml index 239f52c..e2b9e59 100644 --- a/tests/docker-compose.amqp.yml +++ b/tests/docker-compose.amqp.yml @@ -11,5 +11,5 @@ services: ports: - 127.0.0.1:5672:5672 - 127.0.0.1:15672:15672 - - + networks: + - backend diff --git a/tests/docker-compose.postgres.yml b/tests/docker-compose.postgres.yml index 3c802b9..669166e 100644 --- a/tests/docker-compose.postgres.yml +++ b/tests/docker-compose.postgres.yml @@ -10,3 +10,5 @@ services: volumes: - ${PG_RUN}:/var/run/postgresql - ${PGPASSFILE}:/.pgpass + networks: + - backend diff --git a/tests/docker-compose.varnish.yml b/tests/docker-compose.varnish.yml new file mode 100644 index 0000000..d9987a5 --- /dev/null +++ b/tests/docker-compose.varnish.yml @@ -0,0 +1,28 @@ +# +# See https://www.varnish-software.com/developers/tutorials/running-varnish-docker/ +# +# References: +# - https://book.varnish-software.com/4.0/chapters/VCL_Basics.html +# +version: '3.8' +services: + qgis-server: + environment: + QGSRV_CACHE_OBSERVERS: ban + QGSRV_CACHE_OBSERVERS_BAN_SERVER_ADDRESS: "http://varnish:80" + QGSRV_CACHE_CHECK_INTERVAL: 10 + varnish: + image: varnish:7.0.2 + environment: + VARNISH_SIZE: 500M + volumes: + - ${PWD}/varnish.vcl:/etc/varnish/default.vcl:ro + - ${PWD}/varnish.secret:/etc/varnish/secret:ro + command: ["-S", "/etc/varnish/secret"] + tmpfs: + - /var/lib/varnish:exec + ports: + - 127.0.0.1:8889:80 + networks: + - backend + diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 908997c..23f7e46 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -19,7 +19,7 @@ services: QGSRV_PROJECTS_SCHEMES_BAR: file:foobar?data={path} QGSRV_SERVER_PROFILES: /src/tests/profiles.yml QGSRV_SERVER_RESTARTMON: /src/.qgis-restart - QGSRV_SERVER_HTTP_PROXY: 'yes' + QGSRV_SERVER_HTTP_PROXY: 'no' QGSRV_LOGGING_LEVEL: DEBUG QGSRV_DATA_PATH: /.local/share/qgis-server QGSRV_SERVER_STATUS_PAGE: 'yes' @@ -46,9 +46,17 @@ services: ports: - ${SERVER_HTTP_PORT}:8080 - ${MANAGEMENT_HTTP_PORT}:19876 + networks: + - backend deploy: resources: limits: cpus: ${CPU_LIMITS} memory: ${MEMORY_LIMITS} - +networks: + backend: + ipam: + driver: default + config: + - subnet: 172.199.0.0/16 + diff --git a/tests/varnish.secret b/tests/varnish.secret new file mode 100644 index 0000000..d1b6f71 --- /dev/null +++ b/tests/varnish.secret @@ -0,0 +1 @@ +varnishsecretwhateveritcanbe diff --git a/tests/varnish.vcl b/tests/varnish.vcl new file mode 100644 index 0000000..1f73e45 --- /dev/null +++ b/tests/varnish.vcl @@ -0,0 +1,67 @@ +# +# This is an example VCL file for Varnish. +# +# It does not do anything by default, delegating control to the +# builtin VCL. The builtin VCL is called when there is no explicit +# return statement. +# +# See the VCL chapters in the Users Guide for a comprehensive documentation +# at https://www.varnish-cache.org/docs/. + +# Marker to tell the VCL compiler that this VCL has been written with the +# 4.0 or 4.1 syntax. +vcl 4.1; + +import std; + +# acl for administrative requests (i.e BAN) +# Set this to the configured network between admin backend +# and varnish +acl purge { + "172.199.0.0"/16; // Our backend network +} + +# Default backend definition. Set this to point to your content server. +backend default { + .host = "qgis-server"; + .port = "8080"; +} + +sub vcl_recv { + # Happens before we check if we have this in cache already. + # + # Typically you clean up the request here, removing cookies you don't need, + # rewriting the request, etc. + + # Handle BAN request + if (req.method == "BAN") { + if (!client.ip ~ purge) { + return(synth(405,"Not Allowed")); + } + if (std.ban("obj.http.X-Map-Id ~ " + req.http.X-Map-Id)) { + return(synth(200,"Ban Added")); + } else { + return(synth(400, std.ban_error())); + } + } +} + +sub vcl_backend_response { + # Happens after we have read the response headers from the backend. + # + # Here you clean the response headers, removing silly Set-Cookie headers + # and other mistakes your backend does. + + # Set grace period long enough to get + # the response from long loading projects + set beresp.grace = 10m; + return (deliver); +} + +sub vcl_deliver { + # Happens when we have all the pieces we need, and are about to send the + # response to the client. + # + # You can do accounting or modifying the final object here. +} +