Merge pull request #330 from KNMI/postgres-optimize1

Optimizing PostgreSQL queries

CMakeLists.txt

@@ -35,7 +35,7 @@ add_definitions("-DSOURCE_PATH_SIZE=${SOURCE_PATH_SIZE}")
 
 
 
-add_definitions(-DENABLE_CURL -DADAGUC_USE_GDAL -DADAGUC_USE_SQLITE -DADAGUC_USE_POSTGRESQL -DADAGUC_USE_WEBP)
+add_definitions(-DENABLE_CURL -DADAGUC_USE_GDAL -DADAGUC_USE_SQLITE -DADAGUC_USE_WEBP)
 
 
 

Docker/adaguc-server-updatedatasets.sh

@@ -16,6 +16,11 @@ THISSCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )
 # Unbufferd logging for realtime output
 export ADAGUC_ENABLELOGBUFFER=FALSE
 
+
+# Do export ADAGUC_VERBOSE="--verboseon" to enable verbose logging
+
+ADAGUC_VERBOSE="${ADAGUC_VERBOSE:=--verboseoff}" 
+
 if [[ $1 ]]; then
   # Update a specific dataset
   for configfile in /data/adaguc-datasets/$1 ;do
@@ -28,7 +33,7 @@ if [[ $1 ]]; then
       OUT=$?
     else
       echo "*** Starting update for ${filename}" 
-      ${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${filename}
+      ${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${filename} ${ADAGUC_VERBOSE}
       OUT=$?
     fi
 
@@ -49,7 +54,7 @@ else
     fi
     echo ""
     echo "Starting update for ${filename}" 
-    ${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${filename}
+    ${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${filename} ${ADAGUC_VERBOSE}
     OUT=$?
      done
 

Dockerfile

@@ -6,7 +6,7 @@ USER root
 LABEL maintainer="adaguc@knmi.nl"
 
 # Version should be same as in Definitions.h
-LABEL version="2.14.3"
+LABEL version="2.15.0"
 
 # Try to update image packages
 RUN apt-get -q -y update \

NEWS.md

@@ -1,3 +1,6 @@
+**Version 2.15.0 2024-01-29**
+- PostgreSQL queries have been optimized
+
 **Version 2.14.3 2024-01-19**
 - Opendap services are accessible again in the Layer configuration: https://github.com/KNMI/adaguc-server/issues/315
 

adagucserverEC/CAutoConfigure.cpp

@@ -51,6 +51,11 @@ int CAutoConfigure::autoConfigureDimensions(CDataSource *dataSource) {
     return 1;
   }
 
+  if (dataSource->cfgLayer->FilePath.size() != 1 && dataSource->cfgLayer->FilePath[0] != nullptr) {
+    CDBDebug("(dataSource->cfgLayer->FilePath.size() != 1");
+    return 1;
+  }
+
   /**
    * Load dimension information about the layer from the autoconfigure_dimensions table
    * This table stores only layerid, netcdf dimname, adaguc dimname and units
@@ -61,9 +66,12 @@ int CAutoConfigure::autoConfigureDimensions(CDataSource *dataSource) {
 
   CT::string layerTableId;
   try {
-
-    layerTableId = CDBFactory::getDBAdapter(dataSource->srvParams->cfg)
-                       ->getTableNameForPathFilterAndDimension(dataSource->cfgLayer->FilePath[0]->value.c_str(), dataSource->cfgLayer->FilePath[0]->attr.filter.c_str(), NULL, dataSource);
+    auto dbAdapter = CDBFactory::getDBAdapter(dataSource->srvParams->cfg);
+    if (dbAdapter == nullptr) {
+      CDBError("Unable to get a getDBAdapter");
+      return 1;
+    }
+    layerTableId = dbAdapter->getTableNameForPathFilterAndDimension(dataSource->cfgLayer->FilePath[0]->value.c_str(), dataSource->cfgLayer->FilePath[0]->attr.filter.c_str(), NULL, dataSource);
 
   } catch (int e) {
     CDBError("Unable to get layerTableId for autoconfigure_dimensions");
@@ -84,7 +92,6 @@ int CAutoConfigure::autoConfigureDimensions(CDataSource *dataSource) {
   CT::string layerIdentifier = dataSource->getLayerName();
   CDBStore::Store *store = CDBFactory::getDBAdapter(dataSource->srvParams->cfg)->getDimensionInfoForLayerTableAndLayerName(layerTableId.c_str(), layerIdentifier.c_str());
   if (store != NULL) {
-
     try {
 
       for (size_t j = 0; j < store->size(); j++) {

adagucserverEC/CDBAdapterPostgreSQL.cpp

@@ -24,7 +24,6 @@
  ******************************************************************************/
 #include "CDBAdapterPostgreSQL.h"
 
-#ifdef ADAGUC_USE_POSTGRESQL
 #include <set>
 #include "CDebugger.h"
 
@@ -257,9 +256,6 @@ CDBStore::Store *CDBAdapterPostgreSQL::getClosestDataTimeToSystemTime(const char
 };
 
 CDBStore::Store *CDBAdapterPostgreSQL::getFilesForIndices(CDataSource *dataSource, size_t *start, size_t *count, ptrdiff_t *, int) {
-#ifdef MEASURETIME
-  StopWatch_Stop(">CDBAdapterPostgreSQL::getFilesForIndices");
-#endif
 #ifdef CDBAdapterPostgreSQL_DEBUG
   CDBDebug("getFilesForIndices");
 #endif
@@ -1135,5 +1131,3 @@ int CDBAdapterPostgreSQL::addFilesToDataBase() {
 #endif
   return 0;
 }
-
-#endif

adagucserverEC/CDBAdapterPostgreSQL.h

@@ -23,7 +23,6 @@
  *
  ******************************************************************************/
 
-#ifdef ADAGUC_USE_POSTGRESQL
 #include "CDBAdapter.h"
 #include "CDebugger.h"
 #include "CPGSQLDB.h"
@@ -76,4 +75,3 @@ class CDBAdapterPostgreSQL : public CDBAdapter {
   int setFileTimeStamp(const char *tablename, const char *file, const char *dimvalue, int dimindex, const char *filedate, GeoOptions *geoOptions);
   int addFilesToDataBase();
 };
-#endif

adagucserverEC/CDBFactory.cpp

@@ -34,7 +34,7 @@ CDBAdapter *CDBFactory::getDBAdapter(CServerConfig::XMLE_Configuration *cfg) {
   if (staticCDBAdapter == NULL) {
     if (cfg->DataBase.size() != 1) {
       CDBError("DataBase not properly configured");
-      return NULL;
+      exit(1);
     }
     if (cfg->DataBase[0]->attr.dbtype.equals("sqlite")) {
       CDBDebug("Using sqlite");
@@ -44,12 +44,8 @@ CDBAdapter *CDBFactory::getDBAdapter(CServerConfig::XMLE_Configuration *cfg) {
       CDBError("SQLITE is not compiled for ADAGUC, not available!");
 #endif
     } else {
-// CDBDebug("Using postgresql");
-#ifdef ADAGUC_USE_POSTGRESQL
+      // CDBDebug("Using postgresql");
       staticCDBAdapter = new CDBAdapterPostgreSQL();
-#else
-      CDBError("POSTGRESQL is not compiled for ADAGUC, not available!");
-#endif
     }
 
     staticCDBAdapter->setConfig(cfg);

adagucserverEC/CDBFileScanner.cpp

@@ -441,6 +441,7 @@ int CDBFileScanner::DBLoopFiles(CDataSource *dataSource, int removeNonExistingFi
           break;
         }
       }
+      size_t numberOfFilesAddedToDbStore = 0;
       for (size_t d = 0; d < dataSource->cfgLayer->Dimension.size(); d++) {
         if (skipDim[d] == true) {
 #ifdef CDBFILESCANNER_DEBUG
@@ -449,7 +450,7 @@ int CDBFileScanner::DBLoopFiles(CDataSource *dataSource, int removeNonExistingFi
           continue;
         }
         {
-
+          numberOfFilesAddedToDbStore += 1;
           numberOfFilesAddedFromDB = 0;
           int fileExistsInDB = 0;
 
@@ -831,7 +832,7 @@ int CDBFileScanner::DBLoopFiles(CDataSource *dataSource, int removeNonExistingFi
         }
       }
       // End of dimloop, start inserting our collected records in one statement
-      if (j % 50 == 0) dbAdapter->addFilesToDataBase();
+      if (numberOfFilesAddedToDbStore % 50 == 0) dbAdapter->addFilesToDataBase();
     }
 
     // End of dimloop, start inserting our collected records in one statement

adagucserverEC/CPGSQLDB.cpp

@@ -22,8 +22,9 @@
  * limitations under the License.
  *
  ******************************************************************************/
-#ifdef ADAGUC_USE_POSTGRESQL
+
 #include "CPGSQLDB.h"
+// #define CPGSQLDB_DEBUG_H
 const char *CPGSQLDB::className = "CPGSQLDB";
 void CPGSQLDB::clearResult() {
   if (result != NULL) PQclear(result);
@@ -72,55 +73,49 @@ int CPGSQLDB::checkTable(const char *pszTableName, const char *pszColumns) {
   // 2 = table created
 
   LastErrorMsg[0] = '\0';
-  int i;
+
   if (dConnected == 0) {
     CDBError("checkTable: Not connected to DB");
     return 1;
   }
-  char query_string[1024];
-  snprintf(query_string, 1023, "select * from pg_tables where schemaname='public';");
-  result = PQexec(connection, query_string);     /* send the query */
-  if (PQresultStatus(result) != PGRES_TUPLES_OK) /* did the query fail? */
+
+  CT::string queryString;
+  queryString.print("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = '%s') AS table_existence;", pszTableName);
+#ifdef CPGSQLDB_DEBUG_H
+  CDBDebug("checkTable PQexec SELECT EXISTS  %s", pszTableName);
+#endif
+  result = PQexec(connection, queryString.c_str()); /* send the query */
+  if (PQresultStatus(result) != PGRES_TUPLES_OK)    /* did the query fail? */
   {
     CDBError("checkTable: select from pg_tables failed");
     clearResult();
     return 1;
   }
 
-  for (i = 0; i < PQntuples(result); i++) {
-    char *pqval = PQgetvalue(result, i, 1);
-    if (strncmp(pszTableName, pqval, strlen(pszTableName)) == 0 && strlen(pqval) == strlen(pszTableName)) {
-      clearResult();
-      // CDBDebug("Found: %s == %s",pqval,pszTableName);
-      return 0;
-    } else {
-      // CDBDebug("Found: [%s] != [%s]",pqval,pszTableName);
-    }
-  }
-  // No table exists yet
-  if (i == PQntuples(result)) {
+  if (PQntuples(result) == 1 && PQgetvalue(result, 0, 0)[0] == 't') {
     clearResult();
+    return 0;
+  }
 
-    snprintf(query_string, 1023, "CREATE TABLE %s (%s)", pszTableName, pszColumns);
-
-    result = PQexec(connection, query_string);      /* send the query */
-    if (PQresultStatus(result) != PGRES_COMMAND_OK) /* did the query fail? */
-    {
+  // No table exists yet
+  queryString.print("CREATE TABLE %s (%s)", pszTableName, pszColumns);
+#ifdef CPGSQLDB_DEBUG_H
+  CDBDebug("checkTable PQexec CREATE TABLE %s", pszTableName);
+#endif
+  result = PQexec(connection, queryString.c_str()); /* send the query */
+  if (PQresultStatus(result) != PGRES_COMMAND_OK)   /* did the query fail? */
+  {
 
-      snprintf(LastErrorMsg, CPGSQLDB_MAX_STR_LEN, "%s: %s (%s)", PQresStatus(PQresultStatus(result)), PQresultErrorMessage(result), query_string);
+    snprintf(LastErrorMsg, CPGSQLDB_MAX_STR_LEN, "%s: %s (%s)", PQresStatus(PQresultStatus(result)), PQresultErrorMessage(result), queryString.c_str());
 
-      // snprintf(szTemp,CPGSQLDB_MAX_STR_LEN,"checkTable: CREATE TABLE %s failed",pszTableName);
-      // CDBError(LastErrorMsg);
-      clearResult();
-      return 1;
-    }
-    // Table created set status to 2
+    // snprintf(szTemp,CPGSQLDB_MAX_STR_LEN,"checkTable: CREATE TABLE %s failed",pszTableName);
+    // CDBError(LastErrorMsg);
     clearResult();
-    return 2;
+    return 1;
   }
-
+  // Table created set status to 2
   clearResult();
-  return 0;
+  return 2;
 }
 
 int CPGSQLDB::query(const char *pszQuery) {
@@ -129,6 +124,9 @@ int CPGSQLDB::query(const char *pszQuery) {
     CDBError("query: Not connected to DB");
     return 1;
   }
+#ifdef CPGSQLDB_DEBUG_H
+  CDBDebug("query PQexec %s", pszQuery);
+#endif
   result = PQexec(connection, pszQuery);
   if (PQresultStatus(result) != PGRES_COMMAND_OK) /* did the query fail? */
   {
@@ -141,40 +139,9 @@ int CPGSQLDB::query(const char *pszQuery) {
   clearResult();
   return 0;
 }
-// CT::string* CPGSQLDB::query_select_deprecated(const char *pszQuery,int dColumn){
-// //  CDBDebug("query_select %d %s",dColumn,pszQuery);
-//   LastErrorMsg[0]='\0';
-//   int i;
-//   if(dConnected == 0){
-//     CDBError("query_select: Not connected to DB");
-//     return NULL;
-//   }
-//
-//   result = PQexec(connection, pszQuery);
-//
-//   if (PQresultStatus(result) != PGRES_TUPLES_OK) // did the query fail?
-//   {
-//     //snprintf(szTemp,CPGSQLDB_MAX_STR_LEN,"query_select: %s failed",pszQuery);
-//     //CDBError(szTemp);
-//     clearResult();
-//     return NULL;
-//   }
-//   int n=PQntuples(result);
-//   CT::string *strings=new CT::string[n+1];
-//   for(i=0;i<n;i++){
-//     strings[i].copy(PQgetvalue(result, i, dColumn));
-//     strings[i].count=n;
-//   }
-//   CT::CTlink<CT::string>(strings,n);
-//   clearResult();
-//   return strings;
-// }
-// CT::string* CPGSQLDB::query_select_deprecated(const char *pszQuery){
-//   return query_select_deprecated(pszQuery,0);
-// }
 
 CDBStore::Store *CPGSQLDB::queryToStore(const char *pszQuery, bool throwException) {
-  // CDBDebug("query_select %s",pszQuery);
+
   LastErrorMsg[0] = '\0';
 
   if (dConnected == 0) {
@@ -184,7 +151,9 @@ CDBStore::Store *CPGSQLDB::queryToStore(const char *pszQuery, bool throwExceptio
     CDBError("queryToStore: Not connected to DB");
     return NULL;
   }
-
+#ifdef CPGSQLDB_DEBUG_H
+  CDBDebug("queryToStore PQexec %s", pszQuery);
+#endif
   result = PQexec(connection, pszQuery);
 
   if (PQresultStatus(result) != PGRES_TUPLES_OK) // did the query fail?
@@ -225,4 +194,3 @@ CDBStore::Store *CPGSQLDB::queryToStore(const char *pszQuery, bool throwExceptio
   return store;
   ;
 }
-#endif

adagucserverEC/CPGSQLDB.h

@@ -22,7 +22,7 @@
  * limitations under the License.
  *
  ******************************************************************************/
-#ifdef ADAGUC_USE_POSTGRESQL
+
 #ifndef CPGSQLDB_H
 #define CPGSQLDB_H
 #include <stdio.h>
@@ -51,6 +51,13 @@ class CPGSQLDB {
   ~CPGSQLDB();
   int close2();
   int connect(const char *pszOptions);
+  /**
+   * Checks if a table exists, if not it will be created with the specified columns.
+   * @param pszTableName The tablename to check for existence
+   * @param pszColumns The columns needed to create the table
+   * @returns  0: no change, table exists. 1: error,  2: table was created
+
+  */
   int checkTable(const char *pszTableName, const char *pszColumns);
   int query(const char *pszQuery);
   //     CT::string* query_select_deprecated(const char *pszQuery);
@@ -71,5 +78,3 @@ class CPGSQLDB {
   CDBStore::Store *queryToStore(const char *pszQuery) { return queryToStore(pszQuery, false); }
 };
 #endif
-
-#endif