Add BAN cache observer

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+* Add BAN cache observer
 * Fix parameter's case in OGC API requests
    - Fix https://github.com/3liz/py-qgis-server/issues/34
 * Implement configurable cache observers

pyqgisserver/config.py

@@ -245,6 +245,7 @@ def __get_impl( self, _get_fun, section:str, option:str, fallback:Any = NO_DEFAU
             # Look in environment
             # Note that the section must exists
             if self.allow_env:
+                LOGGER.debug("Looking for option '%s.%s' in environment", section, option)
                 varname  = 'QGSRV_%s_%s' % (section.upper(), option.upper())
                 varname  = functools.reduce( lambda s,c: s.replace(c,'_'), ENV_REPLACE_CHARS, varname)
                 varvalue = os.getenv(varname)

pyqgisserver/qgscache/observer.py

@@ -109,7 +109,9 @@ def _load_observers():
             for name in self._declared_observers:
                 try:
                     LOGGER.debug("*** Loading cache observer '%s'", name)
-                    yield cm.load_entrypoint('py_qgis_server.cache.observers',name)
+                    observer = cm.load_entrypoint('py_qgis_server.cache.observers',name)
+                    observer.init()
+                    yield observer
                 except cm.EntryPointNotFoundError:
                     LOGGER.error("Failed to load cache trigger component: %s", name)
 
@@ -168,7 +170,7 @@ def notify_observers(self, key: str, modified_time: datetime, state = UpdateStat
         """
         for obs in self._observers:
             try:
-                obs(key, modified_time, state == UpdateState.INSERTED)
+                obs.observe(key, modified_time, state == UpdateState.INSERTED)
             except Exception:
                 LOGGER.critical("Uncaugh error in observer: %s\n%s", obs, traceback.format_exc())
 

pyqgisserver/qgscache/observers/ban.py

@@ -0,0 +1,54 @@
+# Copyright 2022 3liz
+# Author David Marteau
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+""" Cache Observer that send a BAN request
+"""
+import asyncio
+import logging
+
+from datetime import datetime
+from tornado.httpclient import AsyncHTTPClient, HTTPRequest
+
+from pyqgisserver.config import confservice
+
+LOGGER=logging.getLogger('SRVLOG')
+
+server_address = None
+http_client    = None
+
+def init() -> None:
+    """
+    """
+    LOGGER.debug("*** Initializing ban observer")
+    confservice.add_section('cache.observers:ban')
+
+    global server_address, http_client
+    server_address = confservice.get('cache.observers:ban','server_address')
+    http_client = AsyncHTTPClient()
+
+    LOGGER.debug("*** Ban observer: sending_request to %s", server_address)
+
+async def ban( key: str) -> None:
+    """ Ban key
+    """
+    LOGGER.info("Sending BAN request to %s", server_address)
+
+    request = HTTPRequest(server_address, method='BAN', 
+                          headers={ 'X-Map-Id': key },
+                          user_agent="py-qgis-server; ban observer",
+                          allow_nonstandard_methods=True)
+
+    response = await http_client.fetch(request, raise_error=False)
+    if response.code != 200:
+        LOGGER.error("Ban server returned status code %s", response.code)
+
+
+
+def observe(key: str, datetime: datetime, insert: bool) -> None:
+    asyncio.create_task(ban(key))
+

pyqgisserver/qgscache/observers/test.py

@@ -16,6 +16,9 @@
 
 notify_data = {}
 
+def init() -> None:
+    pass
+
 def observe(key: str, datetime: datetime, insert: bool) -> None:
     LOGGER.debug("*** TEST CACHE OBSERVER: Received update notification for %s %s [Inserted: %s]", key, datetime, insert)
     notify_data[key] = (key,datetime,insert)

pyqgisserver/qgsworker.py

@@ -268,7 +268,7 @@ def init_server(cls) -> None:
 
     @classmethod
     def refresh_cache(cls) -> None:
-        if cls._cache_check_interval <= 0 or time() - cls._cache_check_refresh < cls._cache_check_interval:
+        if cls._cache_check_interval <= 0 or time() - cls._cache_last_check < cls._cache_check_interval:
             return
         LOGGER.debug("Refreshing cache")
         cls._cache_service.refresh()
@@ -397,9 +397,9 @@ def handle_qgis_request(self, ogc_scheme: str, project_location: str, request: R
 
             # Set the project uri in separate header, this
             # is useful for invalidating front-end cache from
-            # key
-            response.setExtraHeader('X-Qgis-Project-Uri', project_location)
-            response.setExtraHeader('Last-Modified'     , last_modified.astimezone().isoformat())
+            # ke
+            response.setExtraHeader('X-Map-Id'      , project_location)
+            response.setExtraHeader('Last-Modified' , last_modified.astimezone().isoformat())
 
             # Check etag for OWS requests
             if ogc_scheme == 'OWS':

setup.py

@@ -48,7 +48,8 @@ def get_version():
             'test = pyqgisserver.monitors.test',
         ],
         'py_qgis_server.cache.observers':[
-            'test = pyqgisserver.qgscache.observers.test:observe',
+            'test = pyqgisserver.qgscache.observers.test',
+            'ban  = pyqgisserver.qgscache.observers.ban',
         ]
     },
     # Add manifest to main package

tests/Makefile

@@ -90,18 +90,14 @@ up:
 stop:
 	docker compose down -v --remove-orphans
 
-amqp-run: export RUN_COMMAND=./tests/run_server.sh
-amqp-run:
-	docker compose \
-	-f docker-compose.yml \
-	-f docker-compose.amqp.yml up -V --remove-orphans --force-recreate -d
-
-postgres-run: export RUN_COMMAND=./tests/run_server.sh
-postgres-run:
+#
+# Run server with extra services
+#
+%-run: export RUN_COMMAND=./tests/run_server.sh
+%-run:
 	docker compose \
 	-f docker-compose.yml \
-	-f docker-compose.postgres.yml up -V --remove-orphans --force-recreate -d
-
+	-f docker-compose.$*.yml up -V --remove-orphans --force-recreate -d
 
 
 

tests/docker-compose.amqp.yml

@@ -11,5 +11,5 @@ services:
         ports:
             - 127.0.0.1:5672:5672
             - 127.0.0.1:15672:15672
-
-
+        networks:
+            - backend

tests/docker-compose.postgres.yml

@@ -10,3 +10,5 @@ services:
         volumes:
             - ${PG_RUN}:/var/run/postgresql
             - ${PGPASSFILE}:/.pgpass
+        networks:
+            - backend

tests/docker-compose.varnish.yml

@@ -0,0 +1,28 @@
+#
+# See https://www.varnish-software.com/developers/tutorials/running-varnish-docker/
+# 
+# References:
+# - https://book.varnish-software.com/4.0/chapters/VCL_Basics.html
+#
+version: '3.8'
+services:
+    qgis-server:
+        environment:
+            QGSRV_CACHE_OBSERVERS: ban
+            QGSRV_CACHE_OBSERVERS_BAN_SERVER_ADDRESS: "http://varnish:80"
+            QGSRV_CACHE_CHECK_INTERVAL: 10
+    varnish:
+        image: varnish:7.0.2
+        environment:
+            VARNISH_SIZE: 500M
+        volumes:
+            - ${PWD}/varnish.vcl:/etc/varnish/default.vcl:ro
+            - ${PWD}/varnish.secret:/etc/varnish/secret:ro
+        command: ["-S", "/etc/varnish/secret"]
+        tmpfs:
+            - /var/lib/varnish:exec
+        ports:
+            - 127.0.0.1:8889:80
+        networks:
+            - backend
+

tests/docker-compose.yml

@@ -19,7 +19,7 @@ services:
             QGSRV_PROJECTS_SCHEMES_BAR: file:foobar?data={path}
             QGSRV_SERVER_PROFILES: /src/tests/profiles.yml
             QGSRV_SERVER_RESTARTMON: /src/.qgis-restart
-            QGSRV_SERVER_HTTP_PROXY: 'yes'
+            QGSRV_SERVER_HTTP_PROXY: 'no'
             QGSRV_LOGGING_LEVEL: DEBUG
             QGSRV_DATA_PATH: /.local/share/qgis-server
             QGSRV_SERVER_STATUS_PAGE: 'yes'
@@ -46,9 +46,17 @@ services:
         ports:
             - ${SERVER_HTTP_PORT}:8080
             - ${MANAGEMENT_HTTP_PORT}:19876
+        networks:
+            - backend
         deploy:
             resources:
                 limits:
                     cpus: ${CPU_LIMITS}
                     memory: ${MEMORY_LIMITS}
-
+networks:
+  backend:
+     ipam:
+       driver: default
+       config:
+         - subnet: 172.199.0.0/16
+

tests/varnish.secret

@@ -0,0 +1 @@
+varnishsecretwhateveritcanbe

tests/varnish.vcl

@@ -0,0 +1,67 @@
+#
+# This is an example VCL file for Varnish.
+#
+# It does not do anything by default, delegating control to the
+# builtin VCL. The builtin VCL is called when there is no explicit
+# return statement.
+#
+# See the VCL chapters in the Users Guide for a comprehensive documentation
+# at https://www.varnish-cache.org/docs/.
+
+# Marker to tell the VCL compiler that this VCL has been written with the
+# 4.0 or 4.1 syntax.
+vcl 4.1;
+
+import std;
+
+# acl for administrative requests (i.e BAN)
+# Set this to the configured network between admin backend
+# and varnish
+acl purge {
+  "172.199.0.0"/16; // Our backend network
+}
+
+# Default backend definition. Set this to point to your content server.
+backend default {
+    .host = "qgis-server";
+    .port = "8080";
+}
+
+sub vcl_recv {
+    # Happens before we check if we have this in cache already.
+    #
+    # Typically you clean up the request here, removing cookies you don't need,
+    # rewriting the request, etc.
+
+    # Handle BAN request
+    if (req.method == "BAN") {
+        if (!client.ip ~ purge) {
+            return(synth(405,"Not Allowed"));
+        }
+        if (std.ban("obj.http.X-Map-Id ~ " + req.http.X-Map-Id)) {
+            return(synth(200,"Ban Added"));
+        } else {
+            return(synth(400, std.ban_error()));
+        }
+    }
+}
+
+sub vcl_backend_response {
+    # Happens after we have read the response headers from the backend.
+    #
+    # Here you clean the response headers, removing silly Set-Cookie headers
+    # and other mistakes your backend does.
+
+    # Set grace period long enough to get 
+    # the response from long loading projects 
+    set beresp.grace = 10m; 
+    return (deliver);
+}
+
+sub vcl_deliver {
+    # Happens when we have all the pieces we need, and are about to send the
+    # response to the client.
+    #
+    # You can do accounting or modifying the final object here.
+}
+