Merge pull request #335 from KNMI/fix-redis-caching

Fix redis caching
KNMI · Mar 19, 2024 · 3634130 · 3634130
2 parents 8e48574 + 27f38f1
commit 3634130
Show file tree

Hide file tree

Showing 25 changed files with 680 additions and 342 deletions.
diff --git a/Docker/docker-compose-generate-env.sh b/Docker/docker-compose-generate-env.sh
@@ -48,8 +48,7 @@ if [ -z "${EXTERNALADDRESS}" ]; then
   fi
 fi
 
-mkdir -p ${ADAGUC_DATA_DIR}
-mkdir -p ${ADAGUC_AUTOWMS_DIR}
+            tree = parse(os.path.join(adaguc_data_set_dir, dataset_file))
 mkdir -p ${ADAGUC_DATASET_DIR}
 
 

diff --git a/Docker/docker-compose.yml b/Docker/docker-compose.yml
@@ -43,13 +43,15 @@ services:
       - "ADAGUC_AUTOWMS_DIR=/data/adaguc-autowms"
       - "ADAGUC_DATA_DIR=/data/adaguc-data"
       - "ADAGUC_DATASET_DIR=/data/adaguc-datasets"
+      - "ADAGUC_REDIS=redis://adaguc-redis:6379"
     env_file:
       - .env
     restart: unless-stopped
     links:
       - adaguc-db
     depends_on:
       - adaguc-db
+      - adaguc-redis
     logging:
       driver: "json-file"
       options:
@@ -96,6 +98,12 @@ services:
       options:
         max-size: "200k"
         max-file: "10"
+  adaguc-redis:
+    image: redis:7
+    container_name: adaguc-redis
+    hostname: adaguc-redis
+    networks:
+      - adaguc-network
 volumes:
   adaguc-server-compose-adagucdb:
 networks:

diff --git a/Dockerfile b/Dockerfile
@@ -6,7 +6,7 @@ USER root
 LABEL maintainer="adaguc@knmi.nl"
 
 # Version should be same as in Definitions.h
-LABEL version="2.20.2"
+LABEL version="2.21.0"
 
 # Try to update image packages
 RUN apt-get -q -y update \

diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,10 @@
+**Version 2.21.0 2024-03-14**
+
+- Added support for Redis caching. Redis caching can be enabled by providing a Redis service via the ADAGUC_REDIS environment and configuring caching settings for a dataset in the [Settings](doc/configuration/Settings.md) element.
+- Improved speed of EDR service and added support to cache EDR calls
+- Various performance improvements
+
+
 **Version 2.20.2 2024-02-28**
 - Removed locking mechanism
 

diff --git a/adagucserverEC/CCreateHistogram.cpp b/adagucserverEC/CCreateHistogram.cpp
@@ -11,10 +11,11 @@ int CCreateHistogram::createHistogram(CDataSource *dataSource, CDrawImage *) {
   CT::string resultJSON;
   if (dataSource->srvParams->JSONP.length() == 0) {
     CDBDebug("CREATING JSON");
-    printf("%s%c%c\n", "Content-Type: application/json", 13, 10);
+    printf("%s%s%c%c\n", "Content-Type: application/json", dataSource->srvParams->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10);
   } else {
     CDBDebug("CREATING JSONP %s", dataSource->srvParams->JSONP.c_str());
-    printf("%s%c%c\n%s(", "Content-Type: application/javascript", 13, 10, dataSource->srvParams->JSONP.c_str());
+    printf("%s%s%c%c", "Content-Type: application/javascript", dataSource->srvParams->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10);
+    printf("\n%s(", dataSource->srvParams->JSONP.c_str());
   }
 
   // puts("{\"a\": 1}");
@@ -257,10 +258,11 @@ int CCreateHistogram::end() {
   CT::string resultJSON;
   if (baseDataSource->srvParams->JSONP.length() == 0) {
     CDBDebug("CREATING JSON");
-    printf("%s%c%c\n", "Content-Type: application/json", 13, 10);
+    printf("%s%s%c%c\n", "Content-Type: application/json", baseDataSource->srvParams->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10);
   } else {
     CDBDebug("CREATING JSONP %s", baseDataSource->srvParams->JSONP.c_str());
-    printf("%s%c%c\n%s(", "Content-Type: application/javascript", 13, 10, baseDataSource->srvParams->JSONP.c_str());
+    printf("%s%s%c%c", "Content-Type: application/javascript", baseDataSource->srvParams->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10);
+    printf("\n%s(", baseDataSource->srvParams->JSONP.c_str());
   }
 
   puts(JSONdata.c_str());

diff --git a/adagucserverEC/CDataReader.cpp b/adagucserverEC/CDataReader.cpp
@@ -292,10 +292,10 @@ bool CDataReader::copyCRSFromADAGUCProjectionVariable(CDataSource *dataSource, c
     return false;
   }
 
-  // if (this->_enableReporting) {
-  //   CREPORT_INFO_NODOC(CT::string("Retrieving the projection according to the ADAGUC standards from the proj4_params or proj4 attribute: ") + proj4Attr->toString(),
-  //   CReportMessage::Categories::GENERAL);
-  // }
+  if (this->_enableReporting) {
+    CREPORT_INFO_NODOC(CT::string("Retrieving the projection according to the ADAGUC standards from the proj4_params or proj4 attribute: ") + proj4Attr->toString(),
+                       CReportMessage::Categories::GENERAL);
+  }
   dataSource->nativeProj4.copy(proj4Attr->toString().c_str());
 
   // Fixes issue https://github.com/KNMI/adaguc-server/issues/279
@@ -340,13 +340,15 @@ void CDataReader::copyEPSGCodeFromProjectionVariable(CDataSource *dataSource, co
   // Get EPSG_code
   CDF::Attribute *epsgAttr = projVar->getAttributeNE("EPSG_code");
   if (epsgAttr != NULL) {
-    CREPORT_INFO_NODOC(CT::string("Using EPSG_code defined in projection variable ") + projVar->name, CReportMessage::Categories::GENERAL);
+    if (this->_enableReporting) {
+      CREPORT_INFO_NODOC(CT::string("Using EPSG_code defined in projection variable ") + projVar->name, CReportMessage::Categories::GENERAL);
+    }
     dataSource->nativeEPSG.copy((char *)epsgAttr->data);
   } else {
     // Make a projection code based on PROJ4: namespace
-    // if (this->_enableReporting) {
-    //   CREPORT_INFO_NODOC(CT::string("Using projection string to create EPSG code.") + dataSource->nativeProj4, CReportMessage::Categories::GENERAL);
-    // }
+    if (this->_enableReporting) {
+      CREPORT_INFO_NODOC(CT::string("Using projection string to create EPSG code.") + dataSource->nativeProj4, CReportMessage::Categories::GENERAL);
+    }
     dataSource->nativeEPSG.print("PROJ4:%s", dataSource->nativeProj4.c_str());
     dataSource->nativeEPSG.replaceSelf("\"", "");
     dataSource->nativeEPSG.replaceSelf("\n", "");
@@ -717,12 +719,10 @@ bool CDataReader::determineXandYVars(CDataSource *dataSource, const CDF::Variabl
                         CReportMessage::Categories::GENERAL);
     return false;
   }
-  // if (this->_enableReporting) {
-  //   CREPORT_INFO_NODOC(
-  //       CT::string("Using variable ") + dataSource->varX->name +
-  //       CT::string(" as X variable and variable ") + dataSource->varY->name +
-  //       CT::string(" as Y variable."), CReportMessage::Categories::GENERAL);
-  // }
+  if (this->_enableReporting) {
+    CREPORT_INFO_NODOC(CT::string("Using variable ") + dataSource->varX->name + CT::string(" as X variable and variable ") + dataSource->varY->name + CT::string(" as Y variable."),
+                       CReportMessage::Categories::GENERAL);
+  }
   return true;
 }
 
@@ -746,9 +746,9 @@ void CDataReader::determineStride2DMap(CDataSource *dataSource) const {
   }
 
   dataSource->stride2DMap = 1;
-  // if (this->_enableReporting) {
-  //   CREPORT_INFO_NODOC(CT::string("No stride defined in the RenderSettings, using a default stride of 1."), CReportMessage::Categories::GENERAL);
-  // }
+  if (this->_enableReporting) {
+    CREPORT_INFO_NODOC(CT::string("No stride defined in the RenderSettings, using a default stride of 1."), CReportMessage::Categories::GENERAL);
+  }
   return;
 }
 

diff --git a/adagucserverEC/CDataReader.h b/adagucserverEC/CDataReader.h
@@ -121,7 +121,7 @@ class CDataReader {
   bool calculateCellSizeAndBBox(CDataSource *dataSource, const CDF::Variable *dataSourceVar) const;
 
 public:
-  CDataReader() { _enableReporting = true; }
+  CDataReader() { _enableReporting = false; }
   ~CDataReader() {}
   bool enablePostProcessors = true; // Allows disabling of other postprocessors. For example when a datasource is output of another postprocessor.
   bool enableObjectCache = true;

diff --git a/adagucserverEC/CImageDataWriter.cpp b/adagucserverEC/CImageDataWriter.cpp
@@ -2414,10 +2414,11 @@ int CImageDataWriter::end() {
           CT::string resultJSON;
           if (srvParam->JSONP.length() == 0) {
             CDBDebug("CREATING JSON");
-            printf("%s%c%c\n", "Content-Type: application/json", 13, 10);
+            printf("%s%s%c%c\n", "Content-Type: application/json", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10);
           } else {
             CDBDebug("CREATING JSONP %s", srvParam->JSONP.c_str());
-            printf("%s%c%c\n%s(", "Content-Type: application/javascript", 13, 10, srvParam->JSONP.c_str());
+            printf("%s%s%c%c", "Content-Type: application/javascript", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10);
+            printf("\n%s(", srvParam->JSONP.c_str());
           }
 
           puts(data.c_str());
@@ -2438,9 +2439,9 @@ int CImageDataWriter::end() {
     if (resultFormat == textplain || resultFormat == texthtml) {
       CT::string resultHTML;
       if (resultFormat == textplain) {
-        resultHTML.print("%s%c%c\n", "Content-Type:text/plain", 13, 10);
+        resultHTML.print("%s%s%c%c\n", "Content-Type: text/plain", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10);
       } else {
-        resultHTML.print("%s%c%c\n", "Content-Type:text/html", 13, 10);
+        resultHTML.print("%s%s%c%c\n", "Content-Type: text/html", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10);
       }
 
       if (resultFormat == texthtml) resultHTML.printconcat("<html>\n");
@@ -2568,7 +2569,8 @@ int CImageDataWriter::end() {
     if (resultFormat == applicationvndogcgml) {
       CDBDebug("CREATING GML");
       CT::string resultXML;
-      resultXML.print("%s%c%c\n", "Content-Type:text/xml", 13, 10);
+      resultXML.print("%s%s%c%c\n", "Content-Type: text/xml", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10);
+
       resultXML.printconcat("<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>\n");
       resultXML.printconcat("  <FeatureCollection\n");
       resultXML.printconcat("          xmlns:gml=\"http://www.opengis.net/gml\"\n");
@@ -2733,10 +2735,11 @@ int CImageDataWriter::end() {
       CT::string resultJSON;
       if (srvParam->JSONP.length() == 0) {
         CDBDebug("CREATING JSON");
-        resultJSON.print("%s%c%c\n", "Content-Type: application/json", 13, 10);
+        resultJSON.print("%s%s%c%c\n", "Content-Type: application/json", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10);
       } else {
         CDBDebug("CREATING JSONP %s", srvParam->JSONP.c_str());
-        resultJSON.print("%s%c%c\n", "Content-Type: application/javascript", 13, 10);
+        resultJSON.print("%s%s%c%c", "Content-Type: application/javascript", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10);
+        resultJSON.print("\n%s(", srvParam->JSONP.c_str());
       }
 
       CXMLParser::XMLElement rootElement;
@@ -3448,7 +3451,6 @@ int CImageDataWriter::end() {
     printf("%s%s%c%c\n", "Content-Type:image/png", cacheControl.c_str(), 13, 10);
     status = drawImage.printImagePng32();
   } else if (srvParam->imageFormat == IMAGEFORMAT_IMAGEWEBP) {
-    CDBDebug("Creating 32 bit webp");
     printf("%s%s%c%c\n", "Content-Type:image/webp", cacheControl.c_str(), 13, 10);
     int webPQuality = srvParam->imageQuality;
     if (!srvParam->Format.empty()) {
@@ -3461,7 +3463,7 @@ int CImageDataWriter::end() {
         }
       }
     }
-    CDBDebug("webPQuality = %d", webPQuality);
+    CDBDebug("Creating 32 bit webp quality = %d", webPQuality);
     status = drawImage.printImageWebP32(webPQuality);
   } else if (srvParam->imageFormat == IMAGEFORMAT_IMAGEGIF) {
     // CDBDebug("LegendGraphic GIF");

diff --git a/adagucserverEC/CRequest.cpp b/adagucserverEC/CRequest.cpp
@@ -741,10 +741,10 @@ int CRequest::generateGetReferenceTimes(CDataSource *dataSource) {
     if (status == CXMLGEN_FATAL_ERROR_OCCURED) return 1;
   }
   if (srvParam->JSONP.length() == 0) {
-    printf("%s%c%c\n", "Content-Type: application/json ", 13, 10);
+    printf("%s%s%c%c\n", "Content-Type: application/json ", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10);
     printf("%s", XMLdocument.c_str());
   } else {
-    printf("%s%c%c\n", "Content-Type: application/javascript ", 13, 10);
+    printf("%s%s%c%c\n", "Content-Type: application/javascript ", srvParam->getCacheControlHeader(CSERVERPARAMS_CACHE_CONTROL_OPTION_SHORTCACHE).c_str(), 13, 10);
     printf("%s(%s)", srvParam->JSONP.c_str(), XMLdocument.c_str());
   }
 
@@ -1496,7 +1496,7 @@ int CRequest::queryDimValuesForDataSource(CDataSource *dataSource, CServerParams
           maxQueryResultLimit = dataSource->cfgLayer->FilePath[0]->attr.maxquerylimit.toInt();
         }
       }
-      CDBDebug("Using maxquerylimit %d", maxQueryResultLimit);
+      // CDBDebug("Using maxquerylimit %d", maxQueryResultLimit);
       store = CDBFactory::getDBAdapter(srvParam->cfg)->getFilesAndIndicesForDimensions(dataSource, maxQueryResultLimit);
     }
 

diff --git a/adagucserverEC/Definitions.h b/adagucserverEC/Definitions.h
@@ -28,7 +28,7 @@
 #ifndef Definitions_H
 #define Definitions_H
 
-#define ADAGUCSERVER_VERSION "2.20.2" // Please also update in the Dockerfile to the same version
+#define ADAGUCSERVER_VERSION "2.21.0" // Please also update in the Dockerfile to the same version
 
 // CConfigReaderLayerType
 #define CConfigReaderLayerTypeUnknown 0