Update configuration

Author: ChrisspyB

src/gribjump/Config.cc

@@ -17,22 +17,20 @@
 namespace gribjump {   
 
 // Config options:
-// - `type`          : Whether GribJump will work locally or forward work to a server. Allowed values are `local` or `remote`.
-// - `server`        : Configuration for the remote server.
-//   - `host`        : The hostname of the server.
-//   - `port`        : The port number of the server.
-// - `threads`       : The number of worker threads created for gribjump.extract. Default is 1.
-//                     NOTE Setting env GRIBJUMP_THREADS will override this setting.
-// - `cache`         : Configuration of the cache.
-//   - `directory`   : The directory where the cache will be stored. Default is `~gribjump/cache/etc/gribjump/cache`.
-//                     Note can be overridden by env var GRIBJUMP_CACHE_DIR.  
-//   - `shadowfdb`   : If true, the cache files will be stored in the same directory as data files, instead of `directory`.
-//   - `enable`      : If false, the caching will be disabled.
-// - `plugin`        : Configuration for using GribJump as a plugin to FDB, which generates jumpinfos on the fly for fdb.archive()
-//                     NOTE Plugin cannot be enabled from config, one must set the envar FDB_ENABLE_GRIBJUMP
-//                     NOTE Setting env FDB_DISABLE_GRIBJUMP will override this setting and disable the plugin.
-//   - `select`      : Defines regex for selecting which FDB keys to generate jumpinfo for. If unset, no jumpinfos will be generated.
-//                   : example `select: date=(20*),stream=(oper|test)`.
+// - type          // Whether GribJump will work locally or forward work to a remote server. Allowed values are `local` or `remote`.
+// - server        // Configuration for gribjump-server.
+//   - port        // The port to listen on for incoming work.
+// - uri           // host:port of remote server to forward work to (requires type:remote)
+// - threads       // The number of worker threads for gribjump.extract. Default is 1.
+// - cache         // Configuration of the cache.
+//   - shadowfdb   // If true, the cache files will be stored in the same directory as data files. DEFAULT=true
+//   - directory   // The directory where the cache will be stored, instead of shadowing the FDB.
+//   - enable      // Whether to look at the cache at all. DEFAULT=true
+// - plugin        // Configuration for using GribJump as a plugin to FDB, which generates jumpinfos on the fly for fdb.archive()
+//                 // NOTE Plugin cannot be enabled from config, one must set the envar FDB_ENABLE_GRIBJUMP
+//                 // NOTE Setting env FDB_DISABLE_GRIBJUMP will override this setting and disable the plugin.
+//   - select      // Defines regex for selecting which FDB keys to generate jumpinfo for. If unset, no jumpinfos will be generated.
+//                 // example `select: date=(20*),stream=(oper|test)`.
 
 Config::Config() {
 }

src/gribjump/info/InfoCache.cc

@@ -42,28 +42,32 @@ InfoCache::~InfoCache() {
 InfoCache::InfoCache(): cacheDir_(eckit::PathName()) {
 
     const Config& config = LibGribJump::instance().config();
+    
+    bool enabled = config.getBool("cache.enabled", true);
+    if (!enabled) {
+        persistentCache_ = false; 
+        LOG_DEBUG_LIB(LibGribJump) << "Cache disabled" << std::endl;
+        return;
+    }
 
-    shadowCache_ = config.getBool("cache.shadowfdb", false);
+    std::string cache_str = config.getString("cache.directory", "");
+    shadowCache_ = config.getBool("cache.shadowfdb", cache_str.empty());
     if (shadowCache_) {
         LOG_DEBUG_LIB(LibGribJump) << "Shadow FDB cache enabled" << std::endl;    
         return;
     }
 
-    std::string cache = eckit::Resource<std::string>("$GRIBJUMP_CACHE_DIR", config.getString("cache.directory", ""));
-    bool enabled = config.getBool("cache.enabled", true);
-
-    if(!enabled || cache.empty()) {
-        persistentCache_ = false; 
-        LOG_DEBUG_LIB(LibGribJump) << "Warning, cache persistence is disabled" << std::endl;
-        return;
+    // Shadow FDB has been explicitly disabled
+    if (cache_str.empty()) {
+        throw eckit::BadValue("Cache directory not set");
     }
 
-    cacheDir_ = eckit::PathName(cache);
-    LOG_DEBUG_LIB(LibGribJump) << "Cache directory " << cacheDir_ << std::endl;
-
+    cacheDir_ = eckit::PathName(cache_str);
     if (!cacheDir_.exists()) {
         throw eckit::BadValue("Cache directory " + cacheDir_ + " does not exist");
     }
+
+    LOG_DEBUG_LIB(LibGribJump) << "Using cache directory: " << cacheDir_ << std::endl;
 }
 
 eckit::PathName InfoCache::cacheFilePath(const eckit::PathName& path) const {

src/gribjump/remote/RemoteGribJump.cc

@@ -19,12 +19,25 @@
 
 namespace gribjump {
 
-RemoteGribJump::RemoteGribJump(const Config& config): GribJumpBase(config) {
-    if (!config.get("remote.host", host_))
-        throw eckit::UserError("RemoteGribJump requires host to be set in config", Here());
+RemoteGribJump::RemoteGribJump(const Config& config): GribJumpBase(config){
+    std::string uri = config.getString("uri", "");
 
-    if (!config.get("remote.port", port_))
-        throw eckit::UserError("RemoteGribJump requires port to be set in config", Here());
+    if (uri.empty())
+        throw eckit::UserError("RemoteGribJump requires uri to be set in config (format host:port)", Here());
+
+    // parse uri, expect format "host:port"
+    size_t pos = uri.find(':');
+
+    if (pos == std::string::npos)
+        throw eckit::UserError("RemoteGribJump uri must be in the format 'host:port'", Here());
+
+    host_ = uri.substr(0, pos);
+    if (host_.empty())
+        throw eckit::UserError("RemoteGribJump requires host to be set in uri", Here());
+
+    port_ = std::stoi(uri.substr(pos+1));
+    if (port_ == 0)
+        throw eckit::UserError("RemoteGribJump requires port to be set in uri", Here());
 }
 
 RemoteGribJump::~RemoteGribJump() {}

tests/tools/callback_vs_scan.sh.in

@@ -20,6 +20,7 @@ fdbread="$<TARGET_FILE:fdb-read>"
 gjscan="$<TARGET_FILE:gribjump-scan>"
 gjextract="$<TARGET_FILE:gribjump-extract>"
 gjinfo="$<TARGET_FILE:gribjump-dump-info>"
+gribset="$<TARGET_FILE:grib_set>"
 
 srcdir=@CMAKE_CURRENT_SOURCE_DIR@
 bindir=@CMAKE_CURRENT_BINARY_DIR@
@@ -49,15 +50,21 @@ spaces:
 EOF
 done
 
-# Get the mars requests corresponding to the data, for scanning/extracting
+# Data from nexus
 supporteddata="${bindir}/supported.grib"
 unsupporteddata="${bindir}/unsupported.grib"
-testdata=${bindir}/alldata.grib
 
-cat $supporteddata $unsupporteddata > $testdata
+# Group data based on whether we expect it to be selected by the filter.
+# We will be selecting expver=xxxx, and not xxxy.
+selected=${bindir}/selected.grib # data to be selected (all other data is filtered)
+unselected=${bindir}/unselected.grib # data to be filtered out
+testdata=${bindir}/alldata.grib
+cat $supporteddata $unsupporteddata > $selected
+$gribset -s "expver=xxxy" $selected $unselected
+cat $selected $unselected > $testdata
 
-requests=${bindir}/requests
-$fdbread --extract ${testdata} /dev/null > $requests
+selectedrequests=${bindir}/requests
+$fdbread --extract ${selected} /dev/null > $selectedrequests
 
 # -------------------------------------
 # 1: Reference FDB
@@ -77,7 +84,7 @@ unset FDB_ENABLE_GRIBJUMP
 echo "Writing to FDB without gribjump plugin, then scanning"
 $fdbwrite ${testdata}
 
-$gjscan $requests
+$gjscan $selectedrequests
 
 # -------------------------------------
 # 3: FDB with gribjump plugin

tests/tools/gribjump-fdb.yaml

@@ -1,5 +1,3 @@
 ---
 plugin:
-  select: date=(2*)
-cache:
-  shadowfdb: true
+  select: expver=(xxxx)