DS-1801: WIP - writeHook to extract searchable values

xyz-hub-service/src/main/java/com/here/xyz/hub/Config.java

@@ -473,5 +473,5 @@ public List<String> getDefaultStorageIds(String region) {
    */
   public ConnectorMapping DEFAULT_CONNECTOR_MAPPING_STRATEGY = RANDOM;
 
-  public boolean USE_WRITE_FEATURES_EVENT = false;
+  public boolean USE_WRITE_FEATURES_EVENT = true;
 }

xyz-hub-service/src/main/java/com/here/xyz/hub/rest/FeatureApi.java

@@ -586,9 +586,45 @@ private UpdateStrategy toUpdateStrategy(Space space, IfExists ifExists, IfNotExi
     );
   }
 
+  private String writeHook()
+  {
+   return    // TODO:  paths = [name, jsonpath] needs to be filled from index configuration
+    """
+  feature => {
+
+  const paths = [
+  ["$alias1", "$.properties.name"],
+  ["$alias2", "$.properties.location.lat"],
+  ["$alias3", "$.properties.tags[*]"]
+];
+
+function extractJsonPaths(feature, paths) {
+  const result = {};
+
+  for (const [name, jsonPath] of paths) {
+    try {
+      const value = jp.query(feature, jsonPath);
+      // decide whether to store arrays or single values:
+      result[name] = value.length === 1 ? value[0] : value;
+    } catch (err) {
+      result[name] = null; // or undefined / error message
+      console.error(`Error evaluating JSONPath for ${name}:`, err.message);
+    }
+  }
+
+  return result;
+}
+
+ return ["searchable",extractJsonPaths(feature, paths) ];
+}
+
+    """;
+}
+
   private Set<Modification> toModifications(RoutingContext context, Space space, FeatureModificationList featureModificationList,
       boolean versionConflictDetectionEnabled) {
     List<String> featureHooks = new ArrayList<>();
+    List<String> writeHooks = new ArrayList<>();
     List<String> addTags = getAddTags(context);
     List<String> removeTags = getRemoveTags(context);
     String idPrefix = getIdPrefix(context);
@@ -599,13 +635,20 @@ private Set<Modification> toModifications(RoutingContext context, Space space, F
     if (idPrefix != null)
       featureHooks.add("feature => feature.id = \"" + idPrefix + "\" + feature.id");
 
+    boolean useWriteHook = true;
+
+    if( useWriteHook )
+     writeHooks.add(writeHook()); // todo: pass searchable aliases to use within writehook
+
     return featureModificationList.getModifications().stream()
         .map(modification -> new Modification()
             .withFeatureData(modification.getFeatureData())
             .withUpdateStrategy(toUpdateStrategy(space, modification.getOnFeatureExists(), modification.getOnFeatureNotExists(),
                 modification.getOnMergeConflict(), versionConflictDetectionEnabled))
             .withPartialUpdates(modification.getOnFeatureExists() == PATCH)
-            .withFeatureHooks(featureHooks.isEmpty() ? null : featureHooks))
+            .withFeatureHooks(featureHooks.isEmpty() ? null : featureHooks)
+            .withWriteHooks(writeHooks.isEmpty() ? null : writeHooks)
+            )
         .collect(Collectors.toSet());
   }
 

xyz-models/src/main/java/com/here/xyz/events/WriteFeaturesEvent.java

@@ -59,6 +59,7 @@ public static class Modification {
     private List<String> featureIds; //To be used only for deletions
     private boolean partialUpdates;
     private List<String> featureHooks; //NOTE: The featureHooks will be applied in the given order
+    private List<String> writeHooks; //NOTE: The Hooks will be applied in the given order
 
     public UpdateStrategy getUpdateStrategy() {
       return updateStrategy;
@@ -124,5 +125,20 @@ public Modification withFeatureHooks(List<String> featureHooks) {
       setFeatureHooks(featureHooks);
       return this;
     }
+
+    public List<String> getWriteHooks() {
+      return writeHooks;
+    }
+
+    public void setWriteHooks(List<String> writeHooks) {
+      this.writeHooks = writeHooks;
+    }
+
+    public Modification withWriteHooks(List<String> writeHooks) {
+      setWriteHooks(writeHooks);
+      return this;
+    }
+
+
   }
 }

xyz-util/src/main/java/com/here/xyz/util/db/pg/Script.java

@@ -131,19 +131,55 @@ public void install() {
       }
     }
     catch (SQLException | IOException e) {
-      logger.warn("Unable to install script {} on DB {}. Falling back to previous version if possible.", getScriptName(),
-          getDbId(), e);
+      logger.warn("Unable to install script {} on DB {}. Falling back to previous version if possible.", getScriptName(), getDbId(), e);
     }
   }
 
   private String getDbId() {
     return dataSourceProvider.getDatabaseSettings() == null ? "unknown" : dataSourceProvider.getDatabaseSettings().getId();
   }
 
+  private SQLQuery addJsLibRegisterFunctions(SQLQuery scriptContent) throws IOException
+  {
+   String
+    libPath = String.format("%s/lib-js",getScriptResourceFolder()),
+    registerSqlFunc =
+   """
+     CREATE OR REPLACE FUNCTION libjs_${{regFunctionName}}() RETURNS text AS
+     $body$
+      with indata as
+      ( select $rfc$${{regFunctionCode}}$rfc$ as code )
+      select regexp_replace(code, '^var\\s+[^\\(]+','') as code from indata
+     $body$
+     LANGUAGE sql IMMUTABLE PARALLEL SAFE;
+   """;
+
+   List<SQLQuery> queries = new ArrayList<>();
+   queries.add(scriptContent);
+
+   List<Script> jsScripts = loadJsScripts( libPath );
+
+   for (Script jsScript : jsScripts ) {
+
+    String scriptName = jsScript.getScriptName(),
+           regFunctionName = scriptName.substring(0, scriptName.indexOf('.'));
+
+    queries.add( new SQLQuery(registerSqlFunc)
+                      .withQueryFragment("regFunctionName", regFunctionName)
+                      .withQueryFragment("regFunctionCode", jsScript.loadScriptContent()) );
+   }
+
+   return SQLQuery.join(queries," ");
+  }
+
   private void install(String targetSchema, boolean deleteBefore) throws SQLException, IOException {
     logger.info("Installing script {} on DB {} into schema {} ...", getScriptName(), getDbId(), targetSchema);
 
     SQLQuery scriptContent = loadSubstitutedScriptContent();
+
+    if( "common.sql".equals(getScriptName()) )
+     scriptContent = addJsLibRegisterFunctions(scriptContent);
+
     List<SQLQuery> installationQueries = new ArrayList<>();
     if (deleteBefore) {
       //TODO: Remove following workaround once "drop schema cascade"-bug creating orphaned functions is fixed in postgres
@@ -264,15 +300,20 @@ private String readResource(String resourceLocation) throws IOException {
   private static List<String> scanResourceFolderWA(String resourceFolder, String fileSuffix) throws IOException {
     return ((List<String>) switch (fileSuffix) {
       case ".sql" -> List.of("/sql/common.sql",  "/sql/geo.sql", "/sql/feature_writer.sql",  "/sql/h3.sql", "/sql/ext.sql", "/jobs/transport.sql");
-      case ".js" -> List.of("/sql/Exception.js", "/sql/FeatureWriter.js", "/sql/DatabaseWriter.js");
+      case ".js" -> List.of(
+                          "/sql/Exception.js", "/sql/FeatureWriter.js", "/sql/DatabaseWriter.js",
+                          "/sql/lib-js/jsonpath_rfc9535.min.js", "/sql/lib-js/sample_hello.min.js"
+                           );
       default -> List.of();
     }).stream().filter(filePath -> filePath.startsWith(resourceFolder)).toList();
   }
 
   private static List<String> scanResourceFolder(ScriptResourcePath scriptResourcePath, String fileSuffix) throws IOException {
     String resourceFolder = scriptResourcePath.path();
     //TODO: Remove this workaround once the actual implementation of this method supports scanning folders inside a JAR
-    if ("/sql".equals(resourceFolder) || "/jobs".equals(resourceFolder))
+    if (    "/sql".equals(resourceFolder)
+         || "/sql/lib-js".equals(resourceFolder)
+         || "/jobs".equals(resourceFolder))
       return ensureInitScriptIsFirst(scanResourceFolderWA(resourceFolder, fileSuffix), scriptResourcePath.initScript());
 
     final InputStream folderResource = Script.class.getResourceAsStream(resourceFolder);
@@ -316,6 +357,10 @@ private SQLQuery loadSubstitutedScriptContent() throws IOException {
     //Load JS-scripts to be injected
     for (Script jsScript : loadJsScripts(getScriptResourceFolder())) {
       String relativeJsScriptPath = jsScript.getScriptResourceFolder().substring(getScriptResourceFolder().length());
+
+      if( relativeJsScriptPath != null && relativeJsScriptPath.length() > 0 && !relativeJsScriptPath.endsWith("/") )
+       relativeJsScriptPath = relativeJsScriptPath + "/";
+
       scriptContent
           .withQueryFragment(relativeJsScriptPath + jsScript.getScriptName(), jsScript.loadScriptContent())
           .withQueryFragment("./" + relativeJsScriptPath + jsScript.getScriptName(), jsScript.loadScriptContent());

xyz-util/src/main/resources/sql/FeatureWriter.js

@@ -43,6 +43,7 @@ class FeatureWriter {
   isPartial;
   baseVersion;
   featureHooks;
+  writeHooks;
 
   onExists;
   onNotExists;
@@ -923,11 +924,11 @@ class FeatureWriter {
   /**
    * @returns {FeatureCollection}
    */
-  static writeFeatures(inputFeatures, author, onExists, onNotExists, onVersionConflict, onMergeConflict, isPartial, featureHooks, version = FeatureWriter.getNextVersion()) {
+  static writeFeatures(inputFeatures, author, onExists, onNotExists, onVersionConflict, onMergeConflict, isPartial, featureHooks, writeHooks, version = FeatureWriter.getNextVersion()) {
     FeatureWriter.dbWriter = new DatabaseWriter(queryContext().schema, FeatureWriter._targetTable(), FeatureWriter._tableBaseVersions().at(-1), FW_BATCH_MODE(), queryContext().tableLayout);
     let result = this.newFeatureCollection();
     for (let feature of inputFeatures) {
-      let execution = new FeatureWriter(feature, version, author, onExists, onNotExists, onVersionConflict, onMergeConflict, isPartial, featureHooks).writeFeature();
+      let execution = new FeatureWriter(feature, version, author, onExists, onNotExists, onVersionConflict, onMergeConflict, isPartial, featureHooks, writeHooks).writeFeature();
       this._collectResult(execution, result);
     }
 
@@ -969,16 +970,18 @@ class FeatureWriter {
     let featureCollections = featureModifications.map(modification => FeatureWriter.writeFeatures(this.toFeatureList(modification),
         author, modification.updateStrategy.onExists, modification.updateStrategy.onNotExists,
         modification.updateStrategy.onVersionConflict, modification.updateStrategy.onMergeConflict, modification.partialUpdates,
-        modification.featureHooks && modification.featureHooks.map(hook => eval(hook)), version));
+        modification.featureHooks && modification.featureHooks.map(hook => eval(hook)),
+        modification.writeHooks && modification.writeHooks.map(hook => eval(hook)),
+        version));
     return this.combineResults(featureCollections);
   }
 
   /**
    * @returns {FeatureCollection}
    */
-  static writeFeature(inputFeature, author, onExists, onNotExists, onVersionConflict, onMergeConflict, isPartial, featureHooks, version = undefined) {
+  static writeFeature(inputFeature, author, onExists, onNotExists, onVersionConflict, onMergeConflict, isPartial, featureHooks, writeHooks, version = undefined) {
     return FeatureWriter.writeFeatures([inputFeature], author, onExists, onNotExists, onVersionConflict, onMergeConflict,
-        isPartial, featureHooks, version);
+        isPartial, featureHooks, writeHooks, version);
   }
 }
 

xyz-util/src/main/resources/sql/common.sql

@@ -229,3 +229,53 @@ END;
 $BODY$
 LANGUAGE plpgsql STABLE
 PARALLEL SAFE;
+
+/**
+
+ *  register code of requirerd js libs in gobalThis
+ *  select require( 'libmod1', 'libmod2', 'libmod3' )
+ *  select require(variadic array['libmod1','libmod2','libmod3'] )
+
+ */
+
+CREATE OR REPLACE FUNCTION require(VARIADIC modules text[])
+RETURNS void AS $body$
+
+    for (let i = 0; i < modules.length; i++) {
+        const name = modules[i];
+
+        if (!name || typeof name !== 'string') continue;
+
+        if (!globalThis[name]) {
+
+            plv8.elog(NOTICE, `Loading module: ${name}`);
+			try {
+             const libSqlName = 'libjs_' + name;
+			 const res = plv8.execute(`select ${libSqlName}()`);
+			 globalThis[name] = res.length > 0 ? new Function("return " + res[0][libSqlName])() : `Unable to load code for module: ${name}`;
+             plv8.elog(NOTICE, `Loaded module source: ${res[0][libSqlName].substring(0,25)}`);
+			} catch (err) {
+			 plv8.elog(NOTICE, `Loading module ${name} failed: ` + err.message);
+			}
+        }
+    }
+$body$
+LANGUAGE plv8 IMMUTABLE
+PARALLEL UNSAFE;
+
+/*
+   sample sql function to verify/demonstrate use of require
+   module sample_hello provides a function hello( input ) which simply returns "Hello + input"
+*/
+
+CREATE OR REPLACE FUNCTION sample_hello_test( sometext text)
+    RETURNS text AS
+$BODY$
+
+    plv8.execute("SELECT require( 'sample_hello' )");
+
+    return sample_hello.hello(sometext);
+
+$BODY$
+LANGUAGE 'plv8' IMMUTABLE
+PARALLEL UNSAFE;

xyz-util/src/main/resources/sql/feature_writer.sql

@@ -51,6 +51,9 @@ $BODY$
         return _queryContext;
       };
 
+      // load required js libs
+      plv8.execute("SELECT require( 'jsonpath_rfc9535' )");
+
       //Init block of internal feature_writer functionality
       ${{FeatureWriter.js}}
       ${{DatabaseWriter.js}}

xyz-util/src/main/resources/sql/lib-js/sample_hello.min.js

@@ -0,0 +1 @@
+var HelloModule=(()=>{var p=(n)=>"Hello "+n;return{hello:p}})();

xyz-util/src/test/js/db/featurewriter/TestFeatureWriter.js

@@ -88,7 +88,7 @@ class TestFeatureWriter {
   }
 
   run() {
-    let result = FeatureWriter.writeFeature(this.inputFeature, null, "REPLACE", "CREATE", null, null, false, null);
+    let result = FeatureWriter.writeFeature(this.inputFeature, null, "REPLACE", "CREATE", null, null, false, null, null);
     console.log("Returned result from FeatureWriter: ", result);
   }