introduce DebuggableTransformerFunctions to debug each step in tranfo…

…rmer operation. (java-0.6.0, js-1.1.0, js-core-1.1.1)
nlighten-oss · Jan 25, 2025 · 6791f1c · 6791f1c
1 parent 9542199
commit 6791f1c
Show file tree

Hide file tree

Showing 46 changed files with 667 additions and 176 deletions.
diff --git a/docs/docs/functions/date.md b/docs/docs/functions/date.md
@@ -140,6 +140,58 @@ Date part of ISO 8601,
 </div>
 ```
 
+## DIFF
+
+Calculate the difference between two dates in specified units.
+
+### Usage
+```transformers
+"$$date(DIFF,{units},{end}):{input}"
+```
+### Returns
+`integer`
+### Arguments
+| Argument | Type                  | Values                            | Required / Default&nbsp;Value | Description                    |
+|----------|-----------------------|-----------------------------------|-------------------------------|--------------------------------|
+| `units`  | `Enum` (`ChronoUnit`) | `SECONDS`/`MINUTES`/ ... /`DAYS`  | Yes                           | The units of calculated result |
+| `end`    | `Date`                |                                   | Yes                           | End date                       |
+
+### Examples
+
+```mdx-code-block
+<div className="examples_grid">
+```
+
+**Input**
+
+**Definition**
+
+**Output**
+
+```json
+"2024-01-01"
+```
+```transformers
+"$$date(DIFF,DAYS,2025-01-01):$"
+```
+```json
+366
+```
+
+```json
+"2025-01-01"
+```
+```transformers
+"$$date(DIFF,DAYS,2026-01-01):$"
+```
+```json
+365
+```
+
+```mdx-code-block
+</div>
+```
+
 ## EPOCH
 
 Seconds passed since 1970-01-01; unless `type`=`MS` then milliseconds,

diff --git a/java/json-transform/build.gradle b/java/json-transform/build.gradle
@@ -9,7 +9,7 @@ plugins {
 }
 
 group 'co.nlighten'
-version = '0.5.2'
+version = '0.6.0'
 
 ext {
     gsonVersion = "2.10.1"

diff --git a/...son-transform/src/main/java/co/nlighten/jsontransform/DebuggableTransformerFunctions.java b/...son-transform/src/main/java/co/nlighten/jsontransform/DebuggableTransformerFunctions.java
@@ -0,0 +1,44 @@
+package co.nlighten.jsontransform;
+
+import co.nlighten.jsontransform.adapters.JsonAdapter;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class DebuggableTransformerFunctions<JE, JA extends Iterable<JE>, JO extends JE> extends TransformerFunctions<JE, JA, JO>{
+    private final Map<String, TransformerDebugInfo> debugResults;
+
+    public record TransformerDebugInfo(Object result) {}
+
+    public DebuggableTransformerFunctions(JsonAdapter<JE, JA, JO> adapter) {
+        super(adapter);
+        debugResults = new HashMap<>();
+    }
+
+    private TransformerFunctions.FunctionMatchResult<Object> auditAndReturn(String path, TransformerFunctions.FunctionMatchResult<Object> matchResult) {
+        if (matchResult == null) {
+            return null;
+        }
+        // if the function result is the transformer's output, don't audit it
+        if ("$".equals(path)) return matchResult;
+
+        if (matchResult.result() instanceof JsonElementStreamer<?,?,?> streamer) {
+            debugResults.put(matchResult.resultPath(), new TransformerDebugInfo(streamer.toJsonArray()));
+            return matchResult;
+        }
+        debugResults.put(matchResult.resultPath(), new TransformerDebugInfo(matchResult.result()));
+        return matchResult;
+    }
+
+    public TransformerFunctions.FunctionMatchResult<Object> matchObject(String path, JO definition, ParameterResolver resolver, JsonTransformerFunction<JE> transformer) {
+        return auditAndReturn(path, super.matchObject(path, definition, resolver, transformer));
+    }
+
+    public TransformerFunctions.FunctionMatchResult<Object> matchInline(String path, String value, ParameterResolver resolver, JsonTransformerFunction<JE> transformer) {
+        return auditAndReturn(path, super.matchInline(path, value, resolver, transformer));
+    }
+
+    public Map<String, TransformerDebugInfo> getDebugResults() {
+        return debugResults;
+    }
+}
diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonElementStreamer.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonElementStreamer.java
@@ -8,7 +8,7 @@
 public class JsonElementStreamer<JE, JA extends Iterable<JE>, JO extends JE> {
     private final FunctionContext<JE, JA, JO> context;
     private final boolean transformed;
-    private final JA value;
+    private JA value;
     private final Stream<JE> stream;
 
     private JsonElementStreamer(FunctionContext<JE, JA, JO> context, JA arr, boolean transformed) {
@@ -34,7 +34,7 @@ public Stream<JE> stream() {
     }
 
     public Stream<JE> stream(Long skip, Long limit) {
-        if (this.stream != null) {
+        if (this.stream != null && this.value == null) {
             var skipped = skip != null ? this.stream.skip(skip) : this.stream;
             return limit != null ? skipped.limit(limit) : skipped;
         }
@@ -72,6 +72,7 @@ public JA toJsonArray() {
         if (stream != null) {
             stream.forEach(item -> context.jArray.add(ja, item));
         }
+        value = ja;
         return ja;
     }
 }
diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformer.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformer.java
@@ -4,26 +4,27 @@
 import com.google.gson.JsonNull;
 
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * A transformer is used to transform data from one layout to another
  */
 public abstract class JsonTransformer<JE, JA extends Iterable<JE>, JO extends JE> implements Transformer {
 
     static final String OBJ_DESTRUCT_KEY = "*";
-    static final String FUNCTION_PREFIX = "$$";
+    static final String NULL_VALUE = "#null";
 
     private final JsonAdapter<JE, JA, JO> adapter;
     protected final JE definition;
 
     private final JsonTransformerFunction<JE> JSON_TRANSFORMER;
-    private final TransformerFunctions<JE, JA, JO> transformerFunctions;
+    private final TransformerFunctionsAdapter<JE, JA, JO> transformerFunctions;
 
     public JsonTransformer(
-            final TransformerFunctions<JE, JA, JO> transformerFunctions,
             final JsonAdapter<JE, JA, JO> adapter,
-            final JE definition) {
-        this.transformerFunctions = transformerFunctions;
+            final JE definition,
+            final TransformerFunctionsAdapter<JE, JA, JO> functionsAdapter) {
+        this.transformerFunctions = functionsAdapter;
         this.adapter = adapter;
         this.definition = definition;
         this.JSON_TRANSFORMER = this::fromJsonElement;
@@ -34,18 +35,18 @@ public Object transform(Object payload, Map<String, Object> additionalContext, b
             return JsonNull.INSTANCE;
         }
         var resolver = adapter.createPayloadResolver(payload, additionalContext, false);
-        return fromJsonElement(definition, resolver, allowReturningStreams);
+        return fromJsonElement("$", definition, resolver, allowReturningStreams);
     }
 
-    protected Object fromJsonPrimitive(JE definition, co.nlighten.jsontransform.ParameterResolver resolver, boolean allowReturningStreams) {
+    protected Object fromJsonPrimitive(String path, JE definition, co.nlighten.jsontransform.ParameterResolver resolver, boolean allowReturningStreams) {
         if (!adapter.isJsonString(definition))
             return definition;
         try {
             var val = adapter.getAsString(definition);
             // test for inline function (e.g. $$function:...)
-            var match = transformerFunctions.matchInline(val, resolver, JSON_TRANSFORMER);
+            var match = transformerFunctions.matchInline(path, val, resolver, JSON_TRANSFORMER);
             if (match != null) {
-                var matchResult = match.getResult();
+                var matchResult = match.result();
                 if (matchResult instanceof JsonElementStreamer streamer) {
                     return allowReturningStreams ? streamer : streamer.toJsonArray();
                 }
@@ -61,10 +62,10 @@ protected Object fromJsonPrimitive(JE definition, co.nlighten.jsontransform.Para
     }
 
 
-    protected Object fromJsonObject(JO definition, co.nlighten.jsontransform.ParameterResolver resolver, boolean allowReturningStreams) {
-        var match = transformerFunctions.matchObject(definition, resolver, JSON_TRANSFORMER);
+    protected Object fromJsonObject(String path, JO definition, co.nlighten.jsontransform.ParameterResolver resolver, boolean allowReturningStreams) {
+        var match = transformerFunctions.matchObject(path, definition, resolver, JSON_TRANSFORMER);
         if (match != null) {
-            var res = match.getResult();
+            var res = match.result();
             return res instanceof JsonElementStreamer s
                     ? (allowReturningStreams ? s : s.toJsonArray())
                     : adapter.wrap(res);
@@ -73,7 +74,7 @@ protected Object fromJsonObject(JO definition, co.nlighten.jsontransform.Paramet
         var result = adapter.jObject.create();
         if (adapter.jObject.has(definition, OBJ_DESTRUCT_KEY)) {
             var val = adapter.jObject.get(definition, OBJ_DESTRUCT_KEY);
-            var res = (JE) fromJsonElement(val, resolver, false);
+            var res = (JE) fromJsonElement(path + "[\"*\"]", val, resolver, false);
             if (res != null) {
                 var isArray = adapter.jArray.is(val);
                 if (isArray && adapter.jArray.is(res)) {
@@ -98,13 +99,15 @@ protected Object fromJsonObject(JO definition, co.nlighten.jsontransform.Paramet
             if (kv.getKey().equals(OBJ_DESTRUCT_KEY)) continue;
             var localKey = kv.getKey();
             var localValue = kv.getValue();
-            if (adapter.isJsonString(localValue) && adapter.getAsString(localValue).equals("#null")) {
+            if (adapter.isJsonString(localValue) && adapter.getAsString(localValue).equals(NULL_VALUE)) {
                 // don't define key if #null was used
                 // might already exist, so try removing it
                 adapter.jObject.remove(result, localKey);
                 continue;
             }
-            var value = (JE) fromJsonElement(localValue, resolver, false);
+            var value = (JE) fromJsonElement(
+                    path + JsonTransformerUtils.toObjectFieldPath(adapter, localKey),
+                    localValue, resolver, false);
             if (!adapter.isNull(value) || adapter.jObject.has(result, localKey) /* we allow overriding with null*/) {
                 adapter.jObject.add(result, localKey, value);
             }
@@ -113,20 +116,21 @@ protected Object fromJsonObject(JO definition, co.nlighten.jsontransform.Paramet
         return result;
     }
 
-    protected Object fromJsonElement(JE definition, ParameterResolver resolver, boolean allowReturningStreams) {
+    protected Object fromJsonElement(String path, JE definition, ParameterResolver resolver, boolean allowReturningStreams) {
         if (adapter.isNull(definition))
             return adapter.jsonNull();
         if (adapter.jArray.is(definition)) {
             var result = adapter.jArray.create();
+            var index = new AtomicInteger(0);
             adapter.jArray.stream((JA)definition)
-                    .map(d -> (JE)fromJsonElement(d, resolver, false))
+                    .map(d -> (JE)fromJsonElement(path + "[" + index.getAndIncrement() + "]", d, resolver, false))
                     .forEachOrdered(item -> adapter.jArray.add(result, item));
             return result;
         }
         if (adapter.jObject.is(definition)) {
-            return fromJsonObject((JO)definition, resolver, allowReturningStreams);
+            return fromJsonObject(path, (JO)definition, resolver, allowReturningStreams);
         }
-        return fromJsonPrimitive(definition, resolver, allowReturningStreams);
+        return fromJsonPrimitive(path, definition, resolver, allowReturningStreams);
     }
 
     public JE getDefinition() {

diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformerFunction.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformerFunction.java
@@ -5,5 +5,5 @@ public interface JsonTransformerFunction<JE> {
     /**
      * @return JsonElement | JsonElementStreamer
      */
-    Object transform(JE definition, ParameterResolver resolver, boolean allowReturningStreams);
+    Object transform(String path, JE definition, ParameterResolver resolver, boolean allowReturningStreams);
 }
diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformerUtils.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformerUtils.java
@@ -10,6 +10,7 @@
 public class JsonTransformerUtils {
 
     private static Pattern variableDetectionRegExp = variableDetectionRegExpFactory(null, null);
+    static final Pattern validIdRegExp = Pattern.compile("^[a-zA-Z_$][a-zA-Z0-9_$]*$");
 
     public static Pattern variableDetectionRegExpFactory(Integer flags, List<String> altNames) {
         var altPrefixes =  altNames != null && !altNames.isEmpty()
@@ -78,4 +79,8 @@ public static void setVariableDetectionRegExp(Integer flags, List<String> altNam
     public static Pattern getVariableDetectionRegExp() {
         return variableDetectionRegExp;
     }
+
+    public static <JE, JA extends Iterable<JE>, JO extends JE> String toObjectFieldPath(JsonAdapter<JE, JA, JO> adapter, String key) {
+        return validIdRegExp.matcher(key).matches() ? "." + key : "[" + adapter.toString(key) + "]";
+    }
 }
diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/TransformerFunctions.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/TransformerFunctions.java
@@ -14,7 +14,7 @@
 import java.util.regex.Pattern;
 import java.util.stream.Stream;
 
-public class TransformerFunctions<JE, JA extends Iterable<JE>, JO extends JE> {
+public class TransformerFunctions<JE, JA extends Iterable<JE>, JO extends JE> implements TransformerFunctionsAdapter<JE, JA, JO>{
     static final Logger log = LoggerFactory.getLogger(TransformerFunctions.class);
 
     private static final Pattern inlineFunctionRegex = Pattern.compile("^\\$\\$(\\w+)(\\((.*?)\\))?(:|$)");
@@ -135,7 +135,7 @@ public void registerFunctions(Map.Entry<String, TransformerFunction<JE, JA, JO>>
     /**
      * Checks the context for a registered object function and returns the result if matched
      */
-    public FunctionMatchResult<Object> matchObject(JO definition, co.nlighten.jsontransform.ParameterResolver resolver, JsonTransformerFunction<JE> transformer) {
+    public FunctionMatchResult<Object> matchObject(String path, JO definition, co.nlighten.jsontransform.ParameterResolver resolver, JsonTransformerFunction<JE> transformer) {
         if (definition == null) {
             return null;
         }
@@ -145,23 +145,25 @@ public FunctionMatchResult<Object> matchObject(JO definition, co.nlighten.jsontr
             if (jsonAdapter.jObject.has(definition, FUNCTION_KEY_PREFIX + key)) {
                 var func = functions.get(key);
                 var context = new ObjectFunctionContext<>(
+                        path,
                         definition,
                         jsonAdapter,
                         FUNCTION_KEY_PREFIX + key,
                         func, resolver, transformer);
+                var resolvedPath = path + "." + FUNCTION_KEY_PREFIX + key;
                 try {
-                    return new FunctionMatchResult<>(func.apply(context));
+                    return new FunctionMatchResult<>(func.apply(context), resolvedPath);
                 } catch (Throwable ex) {
-                    log.warn("Failed running object function ", ex);
-                    return new FunctionMatchResult<>(null);
+                    log.warn("Failed running object function (at {})", resolvedPath, ex);
+                    return new FunctionMatchResult<>(null, resolvedPath);
                 }
             }
         }
         // didn't find an object function
         return null;
     }
 
-    private InlineFunctionContext<JE, JA, JO> tryParseInlineFunction(String value, co.nlighten.jsontransform.ParameterResolver resolver,
+    private InlineFunctionContext<JE, JA, JO> tryParseInlineFunction(String path, String value, co.nlighten.jsontransform.ParameterResolver resolver,
                                                                      JsonTransformerFunction<JE> transformer) {
         var matcher = inlineFunctionRegex.matcher(value);
         if (matcher.find()) {
@@ -195,6 +197,7 @@ private InlineFunctionContext<JE, JA, JO> tryParseInlineFunction(String value, c
                     input = value.substring(matchEndIndex);
                 }
                 return new InlineFunctionContext<>(
+                        path + "/" + FUNCTION_KEY_PREFIX + functionKey,
                         input, args,
                         jsonAdapter,
                         functionKey,
@@ -205,20 +208,21 @@ private InlineFunctionContext<JE, JA, JO> tryParseInlineFunction(String value, c
         return null;
     }
 
-    public FunctionMatchResult<Object> matchInline(String value, ParameterResolver resolver, JsonTransformerFunction<JE> transformer) {
+    public FunctionMatchResult<Object> matchInline(String path, String value, ParameterResolver resolver, JsonTransformerFunction<JE> transformer) {
         if (value == null) return null;
-        var context = tryParseInlineFunction(value, resolver, transformer);
+        var context = tryParseInlineFunction(path, value, resolver, transformer);
         if (context == null) {
             return null;
         }
         // at this point we detected an inline function, we must return a match result
+        var resolvedPath = context.getPathFor(null);
         try {
             var result = functions.get(context.getAlias()).apply(context);
-            return new FunctionMatchResult<>(result);
+            return new FunctionMatchResult<>(result, resolvedPath);
         } catch (Throwable ex) {
-            log.warn("Failed running inline function ", ex);
+            log.warn("Failed running inline function (at {})", resolvedPath, ex);
         }
-        return new FunctionMatchResult<>(null);
+        return new FunctionMatchResult<>(null, resolvedPath);
     }
 
     public Map<String, TransformerFunction<JE, JA, JO>> getFunctions() {
@@ -227,24 +231,8 @@ public Map<String, TransformerFunction<JE, JA, JO>> getFunctions() {
 
     /**
      * The purpose of this class is to differentiate between null and null result
+     *
      * @param <T>
      */
-    static class FunctionMatchResult<T> {
-        final T result;
-
-        /**
-         * Wrap a function result.
-         * @param result the result of the function
-         */
-        public FunctionMatchResult(T result) {
-            this.result = result;
-        }
-
-        /**
-         * @return the result of the function
-         */
-        public T getResult() {
-            return result;
-        }
-    }
+    public record FunctionMatchResult<T>(T result, String resultPath) {}
 }