Merge pull request #401 from KNMI/store-layer-metadata

Store layer metadata

CCDFDataModel/CCDFVariable.h

@@ -292,20 +292,20 @@ namespace CDF {
     size_t getSize() { return currentSize; }
 
     Attribute *getAttribute(const char *name) const {
+      Attribute *a = getAttributeNE(name);
+      if (a == nullptr) {
+        throw(CDF_E_ATTNOTFOUND);
+      }
+      return a;
+    }
+
+    Attribute *getAttributeNE(const char *name) const {
       for (size_t j = 0; j < attributes.size(); j++) {
         if (attributes[j]->name.equals(name)) {
           return attributes[j];
         }
       }
-      throw(CDF_E_ATTNOTFOUND);
-      return NULL;
-    }
-    Attribute *getAttributeNE(const char *name) const {
-      try {
-        return getAttribute(name);
-      } catch (int e) {
-        return NULL;
-      }
+      return nullptr;
     }
 
     /**

CCDFDataModel/CTime.cpp

@@ -878,7 +878,7 @@ CT::string CTime::currentDateTime() {
   strftime(buffer, 80, "%Y-%m-%dT%H:%M:%S", gmtime(&curTime.tv_sec));
 
   char currentTime[100] = "";
-  snprintf(currentTime, 99, "%s:%03dZ", buffer, milli);
+  snprintf(currentTime, 99, "%s.%03dZ", buffer, milli);
 
   return currentTime;
 }

Dockerfile

@@ -6,7 +6,7 @@ USER root
 LABEL maintainer="adaguc@knmi.nl"
 
 # Version should be same as in Definitions.h
-LABEL version="2.27.0"
+LABEL version="2.28.0"
 
 # Try to update image packages
 RUN apt-get -q -y update \
@@ -88,7 +88,7 @@ COPY python /adaguc/adaguc-server-master/python
 # To run the tests against a postgres db, see docs/test_postgesql.md
 FROM base AS test
 
-ENV TEST_IN_CONTAINER 1
+ENV TEST_IN_CONTAINER=1
 
 COPY requirements-dev.txt /adaguc/adaguc-server-master/requirements-dev.txt
 RUN pip install --no-cache-dir -r requirements-dev.txt

NEWS.md

@@ -1,3 +1,8 @@
+**Version 2.28.0 2024-09-11**
+- Metadata for layer items like variables, projections, dimensions and styles are now stored in a database table called `layermetadata`. This can be disabled via the `enablemetadatacache` property in [Settings](doc/configuration/Settings.md).
+
+
+
 **Version 2.27.0 2024-09-02**
 - PostgreSQL query from `getFilesAndIndicesForDimensions` has been rewritten, which fixes https://github.com/KNMI/adaguc-server/issues/341.
 - Optimized existing PostgreSQL queries and reduced number of PostgreSQL queries in general. This results in better performance, the benchmark tool runs 9% faster.

adagucserverEC/CAutoConfigure.cpp

@@ -463,31 +463,13 @@ int CAutoConfigure::autoConfigureStyles(CDataSource *dataSource) {
   return 0;
 }
 
-int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
+int CAutoConfigure::getFileNameForDataSource(CDataSource *dataSource, std::string &fileName) {
 
-  if (dataSource == NULL) {
-    CDBError("datasource == NULL");
-    return 1;
-  }
-  if (dataSource->cfgLayer == NULL) {
-    CDBError("datasource->cfgLayer == NULL");
-    return 1;
-  }
-  if (dataSource->getNumDataObjects() == 0) {
-    CDBError("dataSource->getNumDataObjects()==0");
-    return 1;
-  }
-  if (dataSource->getDataObject(0)->cdfVariable != NULL) {
-#ifdef CAUTOCONFIGURE_DEBUG
-    CDBDebug("already loaded: dataSource->getDataObject(0)->cdfVariable!=NULL");
-#endif
-    return 0;
+  CT::string foundFileName = dataSource->getFileName();
+  if (foundFileName.empty()) {
+    /* Use the file specified as header file */
+    foundFileName = dataSource->headerFileName.c_str();
   }
-
-  CT::string foundFileName;
-
-  /* Use the file specified as header file */
-  foundFileName = dataSource->headerFileName.c_str();
   if (foundFileName.empty()) {
 
     /* Try to get a file from DB */
@@ -498,7 +480,12 @@ int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
     if (dataSource->requiredDims.size() == 0) {
       removeRequiredDims = true;
       if (dataSource->cfgLayer->Dimension.size() > 0) {
-        CRequest::fillDimValuesForDataSource(dataSource, dataSource->srvParams);
+        try {
+          CRequest::fillDimValuesForDataSource(dataSource, dataSource->srvParams);
+        } catch (ServiceExceptionCode e) {
+          CDBDebug("Unable to fillDimValuesForDataSource");
+          return 1;
+        }
       } else {
         CDBDebug("Required dims is still zero, add none now");
         COGCDims *ogcDim = new COGCDims();
@@ -509,7 +496,7 @@ int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
         ogcDim->netCDFDimName.copy("none");
       }
     }
-    CDBStore::Store *store = CDBFactory::getDBAdapter(dataSource->srvParams->cfg)->getFilesAndIndicesForDimensions(dataSource, 1000);
+    CDBStore::Store *store = CDBFactory::getDBAdapter(dataSource->srvParams->cfg)->getFilesAndIndicesForDimensions(dataSource, 1);
     if (store != NULL && store->getSize() > 0) {
       CT::string fileNamestr = store->getRecord(0)->get(0)->c_str();
       // CDBDebug("fileName from DB: %s", fileNamestr.c_str());
@@ -531,7 +518,36 @@ int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
       return 1;
     }
   }
+  fileName = foundFileName.c_str();
+  return 0;
+}
 
+int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
+  if (dataSource == NULL) {
+    CDBError("datasource == NULL");
+    return 1;
+  }
+  if (dataSource->cfgLayer == NULL) {
+    CDBError("datasource->cfgLayer == NULL");
+    return 1;
+  }
+  if (dataSource->getNumDataObjects() == 0) {
+    CDBError("dataSource->getNumDataObjects()==0");
+    return 1;
+  }
+  if (dataSource->getDataObject(0)->cdfVariable != NULL) {
+#ifdef CAUTOCONFIGURE_DEBUG
+    CDBDebug("already loaded: dataSource->getDataObject(0)->cdfVariable!=NULL");
+#endif
+    return 0;
+  }
+
+  std::string foundFileName;
+
+  if (getFileNameForDataSource(dataSource, foundFileName) != 0) {
+    CDBDebug("Unable to getFileNameForDataSource");
+    return 1;
+  }
   /* Open a file */
   try {
     // CDBDebug("Loading header [%s]", foundFileName.c_str());

adagucserverEC/CAutoConfigure.h

@@ -56,5 +56,13 @@ class CAutoConfigure {
 public:
   static int autoConfigureDimensions(CDataSource *dataSource);
   static int autoConfigureStyles(CDataSource *dataSource);
+
+  /**
+   * Find the default filename for a datasource from the database
+   * @param dataSource
+   * @param fileName - The filename to return
+   * @returns Zero on success.
+   */
+  static int getFileNameForDataSource(CDataSource *dataSource, std::string &fileName);
 };
 #endif

adagucserverEC/CDBAdapter.h

@@ -16,7 +16,7 @@
 class CDBAdapter {
 public:
   CDBAdapter() {}
-  virtual ~CDBAdapter(){};
+  virtual ~CDBAdapter() {};
 
   class GeoOptions {
   public:
@@ -96,6 +96,13 @@ class CDBAdapter {
 
   /** First use setFile<type> as many times as you whish, second use addFilesToDataBase to make it final*/
   virtual int addFilesToDataBase() = 0;
+
+  virtual int storeLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey, const char *metadatablob) = 0;
+  virtual CDBStore::Store *getLayerMetadataStore(const char *datasetName) = 0;
+  virtual int dropLayerFromLayerMetadataStore(const char *datasetName, const char *layerName) = 0;
+
+  virtual bool tryAdvisoryLock(size_t key) = 0;
+  virtual bool advisoryUnLock(size_t key) = 0;
 };
 
 #endif

adagucserverEC/CDBAdapterPostgreSQL.cpp

@@ -44,6 +44,10 @@ CDBAdapterPostgreSQL::~CDBAdapterPostgreSQL() {
 #ifdef CDBAdapterPostgreSQL_DEBUG
   CDBDebug("~CDBAdapterPostgreSQL()");
 #endif
+  if (layerMetaDataStore != nullptr) {
+    delete layerMetaDataStore;
+    layerMetaDataStore = nullptr;
+  }
   if (dataBaseConnection != NULL) {
     dataBaseConnection->close2();
   }
@@ -423,7 +427,7 @@ CDBStore::Store *CDBAdapterPostgreSQL::getFilesAndIndicesForDimensions(CDataSour
     CT::string whereStatement;
     std::vector<CT::string> requestedDimVals = requestedDimMap[m.first];
 
-    for (int i = 0; i < requestedDimVals.size(); ++i) {
+    for (size_t i = 0; i < requestedDimVals.size(); ++i) {
       if (requestedDimVals[i].equals("*")) continue;
 
       if (i != 0) {
@@ -620,6 +624,7 @@ void CDBAdapterPostgreSQL::addToLookupTable(const char *path, const char *filter
 CT::string CDBAdapterPostgreSQL::generateRandomTableName() {
   CT::string tableName;
   tableName.print("t%s_%s", CTime::currentDateTime().c_str(), CServerParams::randomString(20).c_str());
+  tableName.replaceSelf(".", "");
   tableName.replaceSelf(":", "");
   tableName.replaceSelf("-", "");
   tableName.replaceSelf("Z", "");
@@ -717,7 +722,7 @@ std::map<CT::string, DimInfo> CDBAdapterPostgreSQL::getTableNamesForPathFilterAn
   }
 
   // Found all tablenames for requested dimensions in lookup table
-  int found = tableDimStore->size();
+  size_t found = tableDimStore->size();
   delete tableDimStore;
   if (found == mapping.size()) {
 #ifdef MEASURETIME
@@ -1080,3 +1085,139 @@ int CDBAdapterPostgreSQL::addFilesToDataBase() {
 #endif
   return 0;
 }
+
+int CDBAdapterPostgreSQL::storeLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey, const char *metadataBlob) {
+#ifdef MEASURETIME
+  StopWatch_Stop(">CDBAdapterPostgreSQL::storeLayerMetadata");
+#endif
+  CPGSQLDB *dataBaseConnection = getDataBaseConnection();
+  if (dataBaseConnection == NULL) {
+    return -1;
+  }
+
+  CT::string tableColumns("datasetname varchar (511), layername varchar (511), metadatakey varchar (255), updatetime TIMESTAMPTZ, blob JSONB, PRIMARY KEY (datasetname, layername, metadatakey)");
+
+  int status = dataBaseConnection->checkTable("layermetadata", tableColumns.c_str());
+  if (status == 1) {
+    CDBError("\nFAIL: Table layermetadata could not be created: %s", tableColumns.c_str());
+    throw(__LINE__);
+  }
+
+  CT::string updateTime = CTime::currentDateTime().c_str();
+  CT::string query;
+  query.print("INSERT INTO layermetadata (datasetname, layername, metadatakey, updatetime, blob) "
+              "VALUES (E'%s',E'%s', E'%s', E'%s', E'%s') "
+              "ON CONFLICT (datasetname, layername, metadatakey) "
+              "DO UPDATE SET blob = excluded.blob, updatetime = excluded.updatetime;",
+              datasetName, layerName, metadataKey, updateTime.c_str(), metadataBlob);
+
+  status = dataBaseConnection->query(query.c_str());
+  if (status != 0) {
+    CDBError("Unable to insert records: \"%s\"", query.c_str());
+    throw(__LINE__);
+  }
+#ifdef MEASURETIME
+  StopWatch_Stop("<CDBAdapterPostgreSQL::storeLayerMetadata");
+#endif
+  return 0;
+}
+
+CDBStore::Store *CDBAdapterPostgreSQL::getLayerMetadataStore(const char *datasetName) {
+#ifdef MEASURETIME
+  StopWatch_Stop(">CDBAdapterPostgreSQL::getLayerMetadata");
+#endif
+
+  if (this->layerMetaDataStore == nullptr) {
+#ifdef CDBAdapterPostgreSQL_DEBUG
+    CDBDebug("Need to query layer metadata for %s/%s/%s", datasetName, layerName, metadataKey);
+#endif
+    CPGSQLDB *dataBaseConnection = getDataBaseConnection();
+    if (dataBaseConnection == NULL) {
+      return nullptr;
+    }
+
+    CT::string query;
+    if (datasetName != nullptr) {
+      if (CServerParams::checkForValidTokens(datasetName, "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_-:/.") == false) {
+        CDBError("Invalid dataset name. ");
+        throw(__LINE__);
+      }
+      query.print("SELECT datasetname, layername, metadatakey, blob FROM layermetadata WHERE datasetname = '%s';", datasetName);
+    } else {
+      query.print("SELECT datasetname, layername, metadatakey, blob FROM layermetadata");
+    }
+
+    this->layerMetaDataStore = dataBaseConnection->queryToStore(query.c_str());
+    if (layerMetaDataStore == nullptr) {
+#ifdef CDBAdapterPostgreSQL_DEBUG
+      CDBDebug("Unable query: \"%s\"", query.c_str());
+#endif
+      return nullptr;
+    }
+  }
+
+#ifdef MEASURETIME
+  StopWatch_Stop("<CDBAdapterPostgreSQL::getLayerMetadata");
+#endif
+  return layerMetaDataStore;
+}
+
+int CDBAdapterPostgreSQL::dropLayerFromLayerMetadataStore(const char *datasetName, const char *layerName) {
+  CPGSQLDB *dataBaseConnection = getDataBaseConnection();
+  if (dataBaseConnection == NULL) {
+    return -1;
+  }
+
+  CT::string query;
+  query.print("DELETE FROM layermetadata "
+              "WHERE datasetname='%s' AND layername = '%s';",
+              datasetName, layerName);
+  return dataBaseConnection->query(query.c_str());
+}
+
+bool CDBAdapterPostgreSQL::tryAdvisoryLock(size_t key) {
+  CPGSQLDB *dataBaseConnection = getDataBaseConnection();
+  if (dataBaseConnection == NULL) {
+    return false;
+  }
+  CT::string query;
+  query.print("SELECT pg_try_advisory_lock(%d) as \"result\";", key);
+  auto *store = dataBaseConnection->queryToStore(query.c_str());
+  if (store == nullptr || store->getSize() != 1) {
+    CDBError("Query failed [%s]:", dataBaseConnection->getError());
+    return false;
+  }
+  auto result = store->getRecord(0)->get("result");
+  bool succesfullylocked = result != nullptr && result->equals("t");
+  if (succesfullylocked) {
+    CDBDebug("pg_try_advisory_lock succesfullylocked");
+  } else {
+    CDBDebug("pg_try_advisory_lock NOT succesfullylocked");
+  }
+  delete store;
+  return succesfullylocked;
+}
+
+bool CDBAdapterPostgreSQL::advisoryUnLock(size_t key) {
+  CPGSQLDB *dataBaseConnection = getDataBaseConnection();
+  if (dataBaseConnection == NULL) {
+    return false;
+  }
+  CT::string query;
+
+  query.print("SELECT pg_advisory_unlock(%d) as \"result\";", key);
+  auto store = dataBaseConnection->queryToStore(query.c_str());
+  if (store == nullptr || store->getSize() != 1) {
+    CDBError("Query failed [%s]:", dataBaseConnection->getError());
+    return false;
+  }
+  auto result = store->getRecord(0)->get("result");
+  bool succesfullyunlocked = result != nullptr && result->equals("t");
+  if (succesfullyunlocked) {
+    CDBDebug("pg_advisory_unlock succesfullyunlocked");
+  } else {
+    CDBWarning("pg_advisory_unlock NOT succesfullyunlocked");
+  }
+  delete store;
+  return succesfullyunlocked;
+}