diff --git a/docs/docs/functions/date.md b/docs/docs/functions/date.md index a37f9d7..88a674d 100644 --- a/docs/docs/functions/date.md +++ b/docs/docs/functions/date.md @@ -140,6 +140,58 @@ Date part of ISO 8601, ``` +## DIFF + +Calculate the difference between two dates in specified units. + +### Usage +```transformers +"$$date(DIFF,{units},{end}):{input}" +``` +### Returns +`integer` +### Arguments +| Argument | Type | Values | Required / Default Value | Description | +|----------|-----------------------|-----------------------------------|-------------------------------|--------------------------------| +| `units` | `Enum` (`ChronoUnit`) | `SECONDS`/`MINUTES`/ ... /`DAYS` | Yes | The units of calculated result | +| `end` | `Date` | | Yes | End date | + +### Examples + +```mdx-code-block +
+``` + +**Input** + +**Definition** + +**Output** + +```json +"2024-01-01" +``` +```transformers +"$$date(DIFF,DAYS,2025-01-01):$" +``` +```json +366 +``` + +```json +"2025-01-01" +``` +```transformers +"$$date(DIFF,DAYS,2026-01-01):$" +``` +```json +365 +``` + +```mdx-code-block +
+``` + ## EPOCH Seconds passed since 1970-01-01; unless `type`=`MS` then milliseconds, diff --git a/java/json-transform/build.gradle b/java/json-transform/build.gradle index 240a156..36c0c6a 100644 --- a/java/json-transform/build.gradle +++ b/java/json-transform/build.gradle @@ -9,7 +9,7 @@ plugins { } group 'co.nlighten' -version = '0.5.2' +version = '0.6.0' ext { gsonVersion = "2.10.1" diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/DebuggableTransformerFunctions.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/DebuggableTransformerFunctions.java new file mode 100644 index 0000000..01a54b4 --- /dev/null +++ b/java/json-transform/src/main/java/co/nlighten/jsontransform/DebuggableTransformerFunctions.java @@ -0,0 +1,44 @@ +package co.nlighten.jsontransform; + +import co.nlighten.jsontransform.adapters.JsonAdapter; + +import java.util.HashMap; +import java.util.Map; + +public class DebuggableTransformerFunctions, JO extends JE> extends TransformerFunctions{ + private final Map debugResults; + + public record TransformerDebugInfo(Object result) {} + + public DebuggableTransformerFunctions(JsonAdapter adapter) { + super(adapter); + debugResults = new HashMap<>(); + } + + private TransformerFunctions.FunctionMatchResult auditAndReturn(String path, TransformerFunctions.FunctionMatchResult matchResult) { + if (matchResult == null) { + return null; + } + // if the function result is the transformer's output, don't audit it + if ("$".equals(path)) return matchResult; + + if (matchResult.result() instanceof JsonElementStreamer streamer) { + debugResults.put(matchResult.resultPath(), new TransformerDebugInfo(streamer.toJsonArray())); + return matchResult; + } + debugResults.put(matchResult.resultPath(), new TransformerDebugInfo(matchResult.result())); + return matchResult; + } + + public TransformerFunctions.FunctionMatchResult matchObject(String path, JO definition, ParameterResolver resolver, JsonTransformerFunction transformer) { + return auditAndReturn(path, super.matchObject(path, definition, resolver, transformer)); + } + + public TransformerFunctions.FunctionMatchResult matchInline(String path, String value, ParameterResolver resolver, JsonTransformerFunction transformer) { + return auditAndReturn(path, super.matchInline(path, value, resolver, transformer)); + } + + public Map getDebugResults() { + return debugResults; + } +} diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonElementStreamer.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonElementStreamer.java index fe3e4a2..dc25cb6 100644 --- a/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonElementStreamer.java +++ b/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonElementStreamer.java @@ -8,7 +8,7 @@ public class JsonElementStreamer, JO extends JE> { private final FunctionContext context; private final boolean transformed; - private final JA value; + private JA value; private final Stream stream; private JsonElementStreamer(FunctionContext context, JA arr, boolean transformed) { @@ -34,7 +34,7 @@ public Stream stream() { } public Stream stream(Long skip, Long limit) { - if (this.stream != null) { + if (this.stream != null && this.value == null) { var skipped = skip != null ? this.stream.skip(skip) : this.stream; return limit != null ? skipped.limit(limit) : skipped; } @@ -72,6 +72,7 @@ public JA toJsonArray() { if (stream != null) { stream.forEach(item -> context.jArray.add(ja, item)); } + value = ja; return ja; } } diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformer.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformer.java index c555ebf..9f2ddce 100644 --- a/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformer.java +++ b/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformer.java @@ -4,6 +4,7 @@ import com.google.gson.JsonNull; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; /** * A transformer is used to transform data from one layout to another @@ -11,19 +12,19 @@ public abstract class JsonTransformer, JO extends JE> implements Transformer { static final String OBJ_DESTRUCT_KEY = "*"; - static final String FUNCTION_PREFIX = "$$"; + static final String NULL_VALUE = "#null"; private final JsonAdapter adapter; protected final JE definition; private final JsonTransformerFunction JSON_TRANSFORMER; - private final TransformerFunctions transformerFunctions; + private final TransformerFunctionsAdapter transformerFunctions; public JsonTransformer( - final TransformerFunctions transformerFunctions, final JsonAdapter adapter, - final JE definition) { - this.transformerFunctions = transformerFunctions; + final JE definition, + final TransformerFunctionsAdapter functionsAdapter) { + this.transformerFunctions = functionsAdapter; this.adapter = adapter; this.definition = definition; this.JSON_TRANSFORMER = this::fromJsonElement; @@ -34,18 +35,18 @@ public Object transform(Object payload, Map additionalContext, b return JsonNull.INSTANCE; } var resolver = adapter.createPayloadResolver(payload, additionalContext, false); - return fromJsonElement(definition, resolver, allowReturningStreams); + return fromJsonElement("$", definition, resolver, allowReturningStreams); } - protected Object fromJsonPrimitive(JE definition, co.nlighten.jsontransform.ParameterResolver resolver, boolean allowReturningStreams) { + protected Object fromJsonPrimitive(String path, JE definition, co.nlighten.jsontransform.ParameterResolver resolver, boolean allowReturningStreams) { if (!adapter.isJsonString(definition)) return definition; try { var val = adapter.getAsString(definition); // test for inline function (e.g. $$function:...) - var match = transformerFunctions.matchInline(val, resolver, JSON_TRANSFORMER); + var match = transformerFunctions.matchInline(path, val, resolver, JSON_TRANSFORMER); if (match != null) { - var matchResult = match.getResult(); + var matchResult = match.result(); if (matchResult instanceof JsonElementStreamer streamer) { return allowReturningStreams ? streamer : streamer.toJsonArray(); } @@ -61,10 +62,10 @@ protected Object fromJsonPrimitive(JE definition, co.nlighten.jsontransform.Para } - protected Object fromJsonObject(JO definition, co.nlighten.jsontransform.ParameterResolver resolver, boolean allowReturningStreams) { - var match = transformerFunctions.matchObject(definition, resolver, JSON_TRANSFORMER); + protected Object fromJsonObject(String path, JO definition, co.nlighten.jsontransform.ParameterResolver resolver, boolean allowReturningStreams) { + var match = transformerFunctions.matchObject(path, definition, resolver, JSON_TRANSFORMER); if (match != null) { - var res = match.getResult(); + var res = match.result(); return res instanceof JsonElementStreamer s ? (allowReturningStreams ? s : s.toJsonArray()) : adapter.wrap(res); @@ -73,7 +74,7 @@ protected Object fromJsonObject(JO definition, co.nlighten.jsontransform.Paramet var result = adapter.jObject.create(); if (adapter.jObject.has(definition, OBJ_DESTRUCT_KEY)) { var val = adapter.jObject.get(definition, OBJ_DESTRUCT_KEY); - var res = (JE) fromJsonElement(val, resolver, false); + var res = (JE) fromJsonElement(path + "[\"*\"]", val, resolver, false); if (res != null) { var isArray = adapter.jArray.is(val); if (isArray && adapter.jArray.is(res)) { @@ -98,13 +99,15 @@ protected Object fromJsonObject(JO definition, co.nlighten.jsontransform.Paramet if (kv.getKey().equals(OBJ_DESTRUCT_KEY)) continue; var localKey = kv.getKey(); var localValue = kv.getValue(); - if (adapter.isJsonString(localValue) && adapter.getAsString(localValue).equals("#null")) { + if (adapter.isJsonString(localValue) && adapter.getAsString(localValue).equals(NULL_VALUE)) { // don't define key if #null was used // might already exist, so try removing it adapter.jObject.remove(result, localKey); continue; } - var value = (JE) fromJsonElement(localValue, resolver, false); + var value = (JE) fromJsonElement( + path + JsonTransformerUtils.toObjectFieldPath(adapter, localKey), + localValue, resolver, false); if (!adapter.isNull(value) || adapter.jObject.has(result, localKey) /* we allow overriding with null*/) { adapter.jObject.add(result, localKey, value); } @@ -113,20 +116,21 @@ protected Object fromJsonObject(JO definition, co.nlighten.jsontransform.Paramet return result; } - protected Object fromJsonElement(JE definition, ParameterResolver resolver, boolean allowReturningStreams) { + protected Object fromJsonElement(String path, JE definition, ParameterResolver resolver, boolean allowReturningStreams) { if (adapter.isNull(definition)) return adapter.jsonNull(); if (adapter.jArray.is(definition)) { var result = adapter.jArray.create(); + var index = new AtomicInteger(0); adapter.jArray.stream((JA)definition) - .map(d -> (JE)fromJsonElement(d, resolver, false)) + .map(d -> (JE)fromJsonElement(path + "[" + index.getAndIncrement() + "]", d, resolver, false)) .forEachOrdered(item -> adapter.jArray.add(result, item)); return result; } if (adapter.jObject.is(definition)) { - return fromJsonObject((JO)definition, resolver, allowReturningStreams); + return fromJsonObject(path, (JO)definition, resolver, allowReturningStreams); } - return fromJsonPrimitive(definition, resolver, allowReturningStreams); + return fromJsonPrimitive(path, definition, resolver, allowReturningStreams); } public JE getDefinition() { diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformerFunction.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformerFunction.java index 08c43d0..12edff4 100644 --- a/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformerFunction.java +++ b/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformerFunction.java @@ -5,5 +5,5 @@ public interface JsonTransformerFunction { /** * @return JsonElement | JsonElementStreamer */ - Object transform(JE definition, ParameterResolver resolver, boolean allowReturningStreams); + Object transform(String path, JE definition, ParameterResolver resolver, boolean allowReturningStreams); } diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformerUtils.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformerUtils.java index 5d78f57..6a6d32b 100644 --- a/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformerUtils.java +++ b/java/json-transform/src/main/java/co/nlighten/jsontransform/JsonTransformerUtils.java @@ -10,6 +10,7 @@ public class JsonTransformerUtils { private static Pattern variableDetectionRegExp = variableDetectionRegExpFactory(null, null); + static final Pattern validIdRegExp = Pattern.compile("^[a-zA-Z_$][a-zA-Z0-9_$]*$"); public static Pattern variableDetectionRegExpFactory(Integer flags, List altNames) { var altPrefixes = altNames != null && !altNames.isEmpty() @@ -78,4 +79,8 @@ public static void setVariableDetectionRegExp(Integer flags, List altNam public static Pattern getVariableDetectionRegExp() { return variableDetectionRegExp; } + + public static , JO extends JE> String toObjectFieldPath(JsonAdapter adapter, String key) { + return validIdRegExp.matcher(key).matches() ? "." + key : "[" + adapter.toString(key) + "]"; + } } diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/TransformerFunctions.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/TransformerFunctions.java index 092fe71..48b1abe 100644 --- a/java/json-transform/src/main/java/co/nlighten/jsontransform/TransformerFunctions.java +++ b/java/json-transform/src/main/java/co/nlighten/jsontransform/TransformerFunctions.java @@ -14,7 +14,7 @@ import java.util.regex.Pattern; import java.util.stream.Stream; -public class TransformerFunctions, JO extends JE> { +public class TransformerFunctions, JO extends JE> implements TransformerFunctionsAdapter{ static final Logger log = LoggerFactory.getLogger(TransformerFunctions.class); private static final Pattern inlineFunctionRegex = Pattern.compile("^\\$\\$(\\w+)(\\((.*?)\\))?(:|$)"); @@ -135,7 +135,7 @@ public void registerFunctions(Map.Entry> /** * Checks the context for a registered object function and returns the result if matched */ - public FunctionMatchResult matchObject(JO definition, co.nlighten.jsontransform.ParameterResolver resolver, JsonTransformerFunction transformer) { + public FunctionMatchResult matchObject(String path, JO definition, co.nlighten.jsontransform.ParameterResolver resolver, JsonTransformerFunction transformer) { if (definition == null) { return null; } @@ -145,15 +145,17 @@ public FunctionMatchResult matchObject(JO definition, co.nlighten.jsontr if (jsonAdapter.jObject.has(definition, FUNCTION_KEY_PREFIX + key)) { var func = functions.get(key); var context = new ObjectFunctionContext<>( + path, definition, jsonAdapter, FUNCTION_KEY_PREFIX + key, func, resolver, transformer); + var resolvedPath = path + "." + FUNCTION_KEY_PREFIX + key; try { - return new FunctionMatchResult<>(func.apply(context)); + return new FunctionMatchResult<>(func.apply(context), resolvedPath); } catch (Throwable ex) { - log.warn("Failed running object function ", ex); - return new FunctionMatchResult<>(null); + log.warn("Failed running object function (at {})", resolvedPath, ex); + return new FunctionMatchResult<>(null, resolvedPath); } } } @@ -161,7 +163,7 @@ public FunctionMatchResult matchObject(JO definition, co.nlighten.jsontr return null; } - private InlineFunctionContext tryParseInlineFunction(String value, co.nlighten.jsontransform.ParameterResolver resolver, + private InlineFunctionContext tryParseInlineFunction(String path, String value, co.nlighten.jsontransform.ParameterResolver resolver, JsonTransformerFunction transformer) { var matcher = inlineFunctionRegex.matcher(value); if (matcher.find()) { @@ -195,6 +197,7 @@ private InlineFunctionContext tryParseInlineFunction(String value, c input = value.substring(matchEndIndex); } return new InlineFunctionContext<>( + path + "/" + FUNCTION_KEY_PREFIX + functionKey, input, args, jsonAdapter, functionKey, @@ -205,20 +208,21 @@ private InlineFunctionContext tryParseInlineFunction(String value, c return null; } - public FunctionMatchResult matchInline(String value, ParameterResolver resolver, JsonTransformerFunction transformer) { + public FunctionMatchResult matchInline(String path, String value, ParameterResolver resolver, JsonTransformerFunction transformer) { if (value == null) return null; - var context = tryParseInlineFunction(value, resolver, transformer); + var context = tryParseInlineFunction(path, value, resolver, transformer); if (context == null) { return null; } // at this point we detected an inline function, we must return a match result + var resolvedPath = context.getPathFor(null); try { var result = functions.get(context.getAlias()).apply(context); - return new FunctionMatchResult<>(result); + return new FunctionMatchResult<>(result, resolvedPath); } catch (Throwable ex) { - log.warn("Failed running inline function ", ex); + log.warn("Failed running inline function (at {})", resolvedPath, ex); } - return new FunctionMatchResult<>(null); + return new FunctionMatchResult<>(null, resolvedPath); } public Map> getFunctions() { @@ -227,24 +231,8 @@ public Map> getFunctions() { /** * The purpose of this class is to differentiate between null and null result + * * @param */ - static class FunctionMatchResult { - final T result; - - /** - * Wrap a function result. - * @param result the result of the function - */ - public FunctionMatchResult(T result) { - this.result = result; - } - - /** - * @return the result of the function - */ - public T getResult() { - return result; - } - } + public record FunctionMatchResult(T result, String resultPath) {} } diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/TransformerFunctionsAdapter.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/TransformerFunctionsAdapter.java new file mode 100644 index 0000000..442c241 --- /dev/null +++ b/java/json-transform/src/main/java/co/nlighten/jsontransform/TransformerFunctionsAdapter.java @@ -0,0 +1,7 @@ +package co.nlighten.jsontransform; + +public interface TransformerFunctionsAdapter, JO extends JE> { + TransformerFunctions.FunctionMatchResult matchInline(String path, String value, ParameterResolver resolver, JsonTransformerFunction transformer); + + TransformerFunctions.FunctionMatchResult matchObject(String path, JO definition, ParameterResolver resolver, JsonTransformerFunction transformer); +} diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/adapters/gson/GsonJsonTransformer.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/adapters/gson/GsonJsonTransformer.java index dc623ad..33c3a6d 100644 --- a/java/json-transform/src/main/java/co/nlighten/jsontransform/adapters/gson/GsonJsonTransformer.java +++ b/java/json-transform/src/main/java/co/nlighten/jsontransform/adapters/gson/GsonJsonTransformer.java @@ -1,6 +1,8 @@ package co.nlighten.jsontransform.adapters.gson; +import co.nlighten.jsontransform.DebuggableTransformerFunctions; import co.nlighten.jsontransform.JsonTransformer; +import co.nlighten.jsontransform.TransformerFunctionsAdapter; import co.nlighten.jsontransform.adapters.JsonAdapter; import co.nlighten.jsontransform.TransformerFunctions; import com.google.gson.*; @@ -11,6 +13,14 @@ public class GsonJsonTransformer extends JsonTransformer FUNCTIONS = new TransformerFunctions<>(ADAPTER); public GsonJsonTransformer(final JsonElement definition) { - super(FUNCTIONS, ADAPTER, definition); + this(definition, FUNCTIONS); + } + + public GsonJsonTransformer(final JsonElement definition, TransformerFunctionsAdapter functionsAdapter) { + super(ADAPTER, definition, functionsAdapter); + } + + public static DebuggableTransformerFunctions getDebuggableAdapter() { + return new DebuggableTransformerFunctions<>(ADAPTER); } } diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/adapters/jsonorg/JsonOrgJsonTransformer.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/adapters/jsonorg/JsonOrgJsonTransformer.java index b8646e1..4738177 100644 --- a/java/json-transform/src/main/java/co/nlighten/jsontransform/adapters/jsonorg/JsonOrgJsonTransformer.java +++ b/java/json-transform/src/main/java/co/nlighten/jsontransform/adapters/jsonorg/JsonOrgJsonTransformer.java @@ -1,7 +1,9 @@ package co.nlighten.jsontransform.adapters.jsonorg; +import co.nlighten.jsontransform.DebuggableTransformerFunctions; import co.nlighten.jsontransform.JsonTransformer; import co.nlighten.jsontransform.TransformerFunctions; +import co.nlighten.jsontransform.TransformerFunctionsAdapter; import co.nlighten.jsontransform.adapters.JsonAdapter; import org.json.JSONArray; import org.json.JSONObject; @@ -12,6 +14,14 @@ class JsonOrgJsonTransformer extends JsonTransformer FUNCTIONS = new TransformerFunctions<>(ADAPTER); public JsonOrgJsonTransformer(final Object definition) { - super(FUNCTIONS, ADAPTER, definition); + this(definition, FUNCTIONS); } -} + + public JsonOrgJsonTransformer(final Object definition, TransformerFunctionsAdapter functionsAdapter) { + super(ADAPTER, definition, functionsAdapter); + } + + public static DebuggableTransformerFunctions getDebuggableAdapter() { + return new DebuggableTransformerFunctions<>(ADAPTER); + } +} \ No newline at end of file diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionDate.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionDate.java index 5214557..db6013f 100644 --- a/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionDate.java +++ b/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionDate.java @@ -7,9 +7,7 @@ import co.nlighten.jsontransform.functions.annotations.ArgumentType; import java.math.BigDecimal; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; +import java.time.*; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.temporal.ChronoField; @@ -29,6 +27,7 @@ @ArgumentType(value = "pattern", type = ArgType.String, position = 1) @ArgumentType(value = "timezone", type = ArgType.String, position = 2, defaultString = "UTC") @ArgumentType(value = "zone", type = ArgType.String, position = 1, defaultString = "UTC") +@ArgumentType(value = "end", type = ArgType.String, position = 2) public class TransformerFunctionDate, JO extends JE> extends TransformerFunction { public static final DateTimeFormatter ISO_INSTANT_0 = new DateTimeFormatterBuilder().appendInstant(0).toFormatter(); public static final DateTimeFormatter ISO_INSTANT_3 = new DateTimeFormatterBuilder().appendInstant(3).toFormatter(); @@ -117,6 +116,21 @@ public Object apply(FunctionContext context) { default -> DateTimeFormatter.ISO_INSTANT.format(instant); }; case "DATE" -> DateTimeFormatter.ISO_INSTANT.format(instant).substring(0, 10); + case "DIFF" -> { + var end = parseInstant(context.getUnwrapped("end")); + var units = ChronoUnit.valueOf(context.getEnum("units")); + if (ChronoUnit.MONTHS.equals(units)) { + var endLocalDate = LocalDate.ofInstant(end, ZoneId.of("UTC")); + var startLocalDate = LocalDate.ofInstant(instant, ZoneId.of("UTC")); + yield BigDecimal.valueOf(Period.between(startLocalDate, endLocalDate).toTotalMonths()); + } else if (ChronoUnit.YEARS.equals(units)) { + var endLocalDate = LocalDate.ofInstant(end, ZoneId.of("UTC")); + var startLocalDate = LocalDate.ofInstant(instant, ZoneId.of("UTC")); + yield BigDecimal.valueOf(Period.between(startLocalDate, endLocalDate).getYears()); + } else { + yield BigDecimal.valueOf(instant.until(end, units)); + } + } case "EPOCH" -> switch (context.getEnum("resolution")) { case "MS" -> BigDecimal.valueOf(instant.toEpochMilli()); default -> BigDecimal.valueOf(instant.getEpochSecond()); diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionEval.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionEval.java index 9721f9a..c7f65af 100644 --- a/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionEval.java +++ b/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionEval.java @@ -16,6 +16,6 @@ public TransformerFunctionEval(JsonAdapter adapter) { @Override public Object apply(FunctionContext context) { var eval = context.getJsonElement(null,true); - return context.transform(eval, true); + return context.transform(context.getPath() + "/.", eval, true); } } diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionIf.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionIf.java index b05adcb..4b065cc 100644 --- a/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionIf.java +++ b/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionIf.java @@ -36,13 +36,13 @@ public Object apply(FunctionContext context) { } else if (adapter.isJsonBoolean(cje)) { condition = adapter.getBoolean(cje); } else { - condition = adapter.isTruthy(context.transform(cje)); + condition = adapter.isTruthy(context.transform(context.getPathFor(0), cje)); } if (condition) { - return context.transform(jArray.get(arr, 1)); + return context.transform(context.getPathFor(1), jArray.get(arr, 1)); } else if (jArray.size(arr) > 2) { - return context.transform(jArray.get(arr, 2)); + return context.transform(context.getPathFor(2), jArray.get(arr, 2)); } } return null; // default falsy value diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionLookup.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionLookup.java index 087c95e..dbc9391 100644 --- a/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionLookup.java +++ b/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionLookup.java @@ -57,7 +57,7 @@ public Object apply(FunctionContext context) { var withDef = jObject.get(using, "with"); if (adapter.isNull(withDef)) continue; // with - null - var with = context.transform(withDef); + var with = context.transform(context.getPathFor("with"), withDef); if (!jArray.is(with)) continue; // with - not array usingMap.put(w, new UsingEntry((JA)with, as, jObject.get(using, "on"))); diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionMap.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionMap.java index a2d8b76..001a192 100644 --- a/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionMap.java +++ b/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionMap.java @@ -38,7 +38,7 @@ public Object apply(FunctionContext context) { var arr = context.getJsonArray(null, false); // we don't transform definitions to prevent premature evaluation if (arr == null) return null; - var inputEl = context.transform(jArray.get(arr, 0)); + var inputEl = context.transform(context.getPathFor(0), jArray.get(arr, 0)); if (!jArray.is(inputEl)) { logger.warn("{} was not specified with an array of items", context.getAlias()); return null; diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/common/FunctionContext.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/common/FunctionContext.java index 7e400bc..c93d563 100644 --- a/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/common/FunctionContext.java +++ b/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/common/FunctionContext.java @@ -1,5 +1,6 @@ package co.nlighten.jsontransform.functions.common; +import co.nlighten.jsontransform.JsonTransformerUtils; import co.nlighten.jsontransform.ParameterResolver; import co.nlighten.jsontransform.adapters.JsonAdapter; import co.nlighten.jsontransform.adapters.JsonArrayAdapter; @@ -19,6 +20,7 @@ public abstract class FunctionContext, JO extends JE protected final String DOUBLE_HASH_INDEX = "##index"; protected final String DOLLAR = "$"; + protected final String path; /** * The function's name as it appeared in the transformer (e.g. "$$filter") */ @@ -31,7 +33,8 @@ public abstract class FunctionContext, JO extends JE public final JsonObjectAdapter jObject; protected final JsonAdapter adapter; - public FunctionContext(JsonAdapter jsonAdapter, + public FunctionContext(String path, + JsonAdapter jsonAdapter, String alias, co.nlighten.jsontransform.functions.common.TransformerFunction function, ParameterResolver resolver, JsonTransformerFunction extractor, @@ -39,6 +42,7 @@ public FunctionContext(JsonAdapter jsonAdapter, this.jArray = jsonAdapter.jArray; this.jObject = jsonAdapter.jObject; this.adapter = jsonAdapter; + this.path = path; this.alias = alias; this.function = function; this.extractor = extractor; @@ -49,8 +53,8 @@ public FunctionContext(JsonAdapter jsonAdapter, } } - public FunctionContext(JsonAdapter jsonAdapter, String alias, TransformerFunction function, ParameterResolver resolver, JsonTransformerFunction extractor) { - this(jsonAdapter, alias, function, resolver, extractor, null); + public FunctionContext(String path, JsonAdapter jsonAdapter, String alias, TransformerFunction function, ParameterResolver resolver, JsonTransformerFunction extractor) { + this(path, jsonAdapter, alias, function, resolver, extractor, null); } private ParameterResolver recalcResolver(JO definition, ParameterResolver resolver, JsonTransformerFunction extractor) { @@ -61,7 +65,7 @@ private ParameterResolver recalcResolver(JO definition, ParameterResolver resolv var addCtx = adapter.jObject.entrySet(ctx).stream().collect( Collectors.toMap( Map.Entry::getKey, - kv -> adapter.getDocumentContext(extractor.transform(kv.getValue(), resolver, false)) + kv -> adapter.getDocumentContext(extractor.transform(path + JsonTransformerUtils.toObjectFieldPath(adapter, kv.getKey()),kv.getValue(), resolver, false)) ) ); return name -> { @@ -91,6 +95,10 @@ public String getAlias() { return alias; } + public String getPath() { + return alias; + } + public ParameterResolver getResolver() { return resolver; } @@ -103,6 +111,12 @@ public Object get(String name) { return get(name, true); } + public String getPathFor(String name) { + return this.path; + } + public String getPathFor(int index) { + return this.path + "[" + index + "]"; + } public boolean isNull(JE value) { return adapter.isNull(value); @@ -291,7 +305,7 @@ public JsonElementStreamer getJsonElementStreamer(String name) { // in case val is already an array we don't transform it to prevent evaluation of its result values // so if is not an array, we must transform it and check after-wards (not lazy anymore) if (!adapter.jArray.is(value)) { - value = extractor.transform(wrap(value), resolver, true); + value = extractor.transform(getPathFor(name), wrap(value), resolver, true); if (value instanceof JsonElementStreamer jes) { return jes; } @@ -304,12 +318,17 @@ public JsonElementStreamer getJsonElementStreamer(String name) { return null; } + // TODO: replace this with something public JE transform(JE definition){ - return (JE) extractor.transform(definition, resolver, false); + return (JE) extractor.transform(path, definition, resolver, false); + } + + public Object transform(String path, JE definition){ + return extractor.transform(path, definition, resolver, false); } - public Object transform(JE definition, boolean allowReturningStreams){ - return extractor.transform(definition, resolver, allowReturningStreams); + public Object transform(String path, JE definition, boolean allowReturningStreams){ + return extractor.transform(path, definition, resolver, allowReturningStreams); } public JE transformItem(JE definition, JE current) { @@ -318,7 +337,7 @@ public JE transformItem(JE definition, JE current) { pathOfVar(DOUBLE_HASH_CURRENT, name) ? currentContext.read(DOLLAR + name.substring(9)) : resolver.get(name); - return (JE) extractor.transform(definition, itemResolver, false); + return (JE) extractor.transform("$", definition, itemResolver, false); } public JE transformItem(JE definition, JE current, Integer index) { @@ -329,7 +348,7 @@ public JE transformItem(JE definition, JE current, Integer index) { : pathOfVar(DOUBLE_HASH_CURRENT, name) ? currentContext.read(DOLLAR + name.substring(9)) : resolver.get(name); - return (JE) extractor.transform(definition, itemResolver, false); + return (JE) extractor.transform("$", definition, itemResolver, false); } public JE transformItem(JE definition, JE current, Integer index, String additionalName, JE additional) { @@ -343,7 +362,7 @@ public JE transformItem(JE definition, JE current, Integer index, String additio : pathOfVar(additionalName, name) ? additionalContext.read(DOLLAR + name.substring(additionalName.length())) : resolver.get(name); - return (JE) extractor.transform(definition, itemResolver, false); + return (JE) extractor.transform("$", definition, itemResolver, false); } public JE transformItem(JE definition, JE current, Integer index, Map additionalContexts) { @@ -360,6 +379,6 @@ public JE transformItem(JE definition, JE current, Integer index, Map, JO extends JE> e protected final String stringInput; protected final ArrayList args; - public InlineFunctionContext(String input, ArrayList args, + public InlineFunctionContext(String path, String input, ArrayList args, JsonAdapter jsonAdapter, String functionKey, TransformerFunction function, ParameterResolver resolver, JsonTransformerFunction extractor) { - super(jsonAdapter, functionKey, function, resolver, extractor, null); + super(path, jsonAdapter, functionKey, function, resolver, extractor, null); this.stringInput = input; this.args = args; } @@ -34,12 +34,17 @@ public Object get(String name, boolean transform) { } var argValue = name == null ? stringInput : args.get(argument.position()); if (argValue instanceof String s && transform) { - return extractor.transform(adapter.wrap(s), resolver, true); + return extractor.transform(getPathFor(name), adapter.wrap(s), resolver, true); } if (adapter.is(argValue)) { var je = (JE)argValue; - return transform ? extractor.transform(je, resolver, true) : je; + return transform ? extractor.transform(getPathFor(name), je, resolver, true) : je; } return argValue; } + + @Override + public String getPathFor(String name) { + return path + (name == null ? "" : "(" + name + ")"); + } } diff --git a/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/common/ObjectFunctionContext.java b/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/common/ObjectFunctionContext.java index 1f6a45b..e3b39e7 100644 --- a/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/common/ObjectFunctionContext.java +++ b/java/json-transform/src/main/java/co/nlighten/jsontransform/functions/common/ObjectFunctionContext.java @@ -1,5 +1,6 @@ package co.nlighten.jsontransform.functions.common; +import co.nlighten.jsontransform.JsonTransformerUtils; import co.nlighten.jsontransform.adapters.JsonAdapter; import co.nlighten.jsontransform.JsonTransformerFunction; import co.nlighten.jsontransform.ParameterResolver; @@ -7,11 +8,12 @@ public class ObjectFunctionContext, JO extends JE> extends FunctionContext { private final JO definition; - public ObjectFunctionContext(JO definition, JsonAdapter jsonAdapter, + public ObjectFunctionContext(String path, + JO definition, JsonAdapter jsonAdapter, String functionKey, TransformerFunction function, ParameterResolver resolver, JsonTransformerFunction extractor) { - super(jsonAdapter, functionKey, function, resolver, extractor, definition); + super(path, jsonAdapter, functionKey, function, resolver, extractor, definition); this.definition = definition; } @@ -26,6 +28,11 @@ public Object get(String name, boolean transform) { if (adapter.isNull(el)) { return function.getDefaultValue(name); } - return transform ? extractor.transform(el, resolver, true) : el; + return transform ? extractor.transform(getPathFor(name), el, resolver, true) : el; + } + + @Override + public String getPathFor(String name) { + return path + JsonTransformerUtils.toObjectFieldPath(adapter, name == null ? getAlias() : name); } } diff --git a/java/playground/src/main/java/co/nlighten/jsontransform/playground/ApiController.java b/java/playground/src/main/java/co/nlighten/jsontransform/playground/ApiController.java index ffeb137..52a1ffe 100644 --- a/java/playground/src/main/java/co/nlighten/jsontransform/playground/ApiController.java +++ b/java/playground/src/main/java/co/nlighten/jsontransform/playground/ApiController.java @@ -1,5 +1,6 @@ package co.nlighten.jsontransform.playground; +import co.nlighten.jsontransform.DebuggableTransformerFunctions; import co.nlighten.jsontransform.adapters.gson.GsonJsonTransformer; import org.springframework.web.bind.annotation.*; @@ -9,8 +10,9 @@ public class ApiController { @PostMapping("/v1/transform") public TransformTestResponse v1Transform(@RequestBody TransformTestRequest request){ - var transformer = new GsonJsonTransformer(request.definition); + var adapter = GsonJsonTransformer.getDebuggableAdapter(); + var transformer = new GsonJsonTransformer(request.definition, adapter); var result = transformer.transform(request.input, request.additionalContext); - return new TransformTestResponse(result); + return new TransformTestResponse(result, request.debug ? adapter.getDebugResults() : null); } } \ No newline at end of file diff --git a/java/playground/src/main/java/co/nlighten/jsontransform/playground/BigDecimalTypeAdapter.java b/java/playground/src/main/java/co/nlighten/jsontransform/playground/BigDecimalTypeAdapter.java index 53ea09a..db63ec5 100644 --- a/java/playground/src/main/java/co/nlighten/jsontransform/playground/BigDecimalTypeAdapter.java +++ b/java/playground/src/main/java/co/nlighten/jsontransform/playground/BigDecimalTypeAdapter.java @@ -8,8 +8,8 @@ public class BigDecimalTypeAdapter implements JsonSerializer, JsonDeserializer { @Override public BigDecimal deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext jsonDeserializationContext) throws JsonParseException { - if (jsonElement instanceof JsonPrimitive jp && jp.isString() && jp.getAsString().startsWith("bd#")) { - return new BigDecimal(jp.getAsString().substring(3)); + if (jsonElement instanceof JsonPrimitive jp) { + return new BigDecimal(jp.isString() ? jp.getAsString() : jp.getAsNumber().toString()); } return null; } @@ -19,6 +19,6 @@ public JsonElement serialize(BigDecimal bigDecimal, Type type, JsonSerialization if (bigDecimal == null) { return JsonNull.INSTANCE; } - return new JsonPrimitive("bd#" + bigDecimal); + return new JsonPrimitive(bigDecimal); } } diff --git a/java/playground/src/main/java/co/nlighten/jsontransform/playground/TransformTestRequest.java b/java/playground/src/main/java/co/nlighten/jsontransform/playground/TransformTestRequest.java index 8c3b359..6d9cc2c 100644 --- a/java/playground/src/main/java/co/nlighten/jsontransform/playground/TransformTestRequest.java +++ b/java/playground/src/main/java/co/nlighten/jsontransform/playground/TransformTestRequest.java @@ -8,4 +8,5 @@ public class TransformTestRequest { public JsonElement input; public JsonElement definition; public Map additionalContext; + public boolean debug; } diff --git a/java/playground/src/main/java/co/nlighten/jsontransform/playground/TransformTestResponse.java b/java/playground/src/main/java/co/nlighten/jsontransform/playground/TransformTestResponse.java index ecd7442..7753d7c 100644 --- a/java/playground/src/main/java/co/nlighten/jsontransform/playground/TransformTestResponse.java +++ b/java/playground/src/main/java/co/nlighten/jsontransform/playground/TransformTestResponse.java @@ -1,8 +1,14 @@ package co.nlighten.jsontransform.playground; +import co.nlighten.jsontransform.DebuggableTransformerFunctions; + +import java.util.Map; + public class TransformTestResponse { Object result; - public TransformTestResponse(Object result) { + Map debugInfo; + public TransformTestResponse(Object result, Map debugInfo) { this.result = result; + this.debugInfo = debugInfo; } } diff --git a/javascript/json-transform-core/package.json b/javascript/json-transform-core/package.json index 9904ddb..f389c1f 100644 --- a/javascript/json-transform-core/package.json +++ b/javascript/json-transform-core/package.json @@ -1,7 +1,7 @@ { "name": "@nlighten/json-transform-core", "description": "Core types and utilities for handling JSON transformers", - "version": "1.0.4", + "version": "1.1.1", "main": "dist/json-transform-core.js", "umd:main": "dist/json-transform-core.umd.js", "module": "dist/json-transform-core.module.js", diff --git a/javascript/json-transform-core/src/functions/embeddedFunctions.ts b/javascript/json-transform-core/src/functions/embeddedFunctions.ts index 811e0fa..669668e 100644 --- a/javascript/json-transform-core/src/functions/embeddedFunctions.ts +++ b/javascript/json-transform-core/src/functions/embeddedFunctions.ts @@ -85,6 +85,34 @@ export const DateFunctionArgBasedSchemas: Record = p { name: "timezone", description: "(FORMAT)", type: "string", position: 2, default: "UTC" }, ], }, + DIFF: { + outputSchema: { type: "integer" }, + description: "Calculate the difference between two dates in specified units.", + arguments: [ + { + name: "DIFF", + type: "enum", + enum: ["DIFF"], + position: 0, + required: true, + }, + { + name: "units", + description: "(DIFF) Units to use (ChronoUnit)", + type: "enum", + enum: ["NANOS", "MICROS", "MILLIS", "SECONDS", "MINUTES", "HOURS", "HALF_DAYS", "DAYS"], + position: 1, + required: true, + }, + { + name: "end", + description: "(DIFF) End date", + type: "string", + position: 2, + required: true, + }, + ], + }, EPOCH: { outputSchema: { type: "integer" }, description: "Seconds passed since 1970-01-01; unless `resolution`=`MS` then milliseconds,", @@ -434,7 +462,7 @@ export const embeddedFunctions: Record { return args.reduce((a, c) => { if (!a) return c ?? ""; diff --git a/javascript/json-transform-core/src/parse.ts b/javascript/json-transform-core/src/parse.ts index 0968bf5..85fea5c 100644 --- a/javascript/json-transform-core/src/parse.ts +++ b/javascript/json-transform-core/src/parse.ts @@ -7,10 +7,9 @@ import { import { EmbeddedTransformerFunction } from "./functions/types"; import { functions } from "./functions/functions"; import { matchJsonPathFunction } from "./jsonpath/jsonpathFunctions"; -import { jsonpathJoin } from "./jsonpath/jsonpathJoin"; +import { jsonpathJoin, VALID_ID_REGEXP } from "./jsonpath/jsonpathJoin"; import ParseContext, { HandleFunctionMethod, ParseMethod } from "./ParseContext"; -const VALID_ID_REGEXP = /^[a-z_$][a-z0-9_$]*$/i; const ALL_DIGITS = /^\d+$/; class TransformerParser { diff --git a/javascript/json-transform/package.json b/javascript/json-transform/package.json index b10ba12..73d98ce 100644 --- a/javascript/json-transform/package.json +++ b/javascript/json-transform/package.json @@ -1,7 +1,7 @@ { "name": "@nlighten/json-transform", "description": "JSON transformers JavaScript implementation", - "version": "1.0.1", + "version": "1.1.0", "main": "dist/json-transform.js", "umd:main": "dist/json-transform.umd.js", "module": "dist/json-transform.module.js", diff --git a/javascript/json-transform/src/DebuggableTransformerFunctions.ts b/javascript/json-transform/src/DebuggableTransformerFunctions.ts new file mode 100644 index 0000000..c071ee0 --- /dev/null +++ b/javascript/json-transform/src/DebuggableTransformerFunctions.ts @@ -0,0 +1,54 @@ +import { FunctionMatchResult, TransformerFunctions } from "./transformerFunctions"; +import JsonElementStreamer from "./JsonElementStreamer"; +import { ParameterResolver } from "./ParameterResolver"; +import { JsonTransformerFunction } from "./JsonTransformerFunction"; + +export type TransformerDebugInfo = { + result: any; +}; + +export default class DebuggableTransformerFunctions extends TransformerFunctions { + private readonly debugResults: Record; + + constructor() { + super(); + this.debugResults = {}; + } + + public getDebugResults(): Record { + return this.debugResults; + } + + private async auditAndReturn(path: string, matchResult: FunctionMatchResult | null) { + if (!matchResult) { + return null; + } + // if the function result is the transformer's output, don't audit it + if (path === "$") return matchResult; + + const value = matchResult.getResult(); + if (value instanceof JsonElementStreamer) { + this.debugResults[matchResult.getResultPath()] = { result: await value.toJsonArray() }; + return matchResult; + } + this.debugResults[matchResult.getResultPath()] = { result: value }; + return matchResult; + } + + override async matchObject( + path: string, + definition: any, + resolver: ParameterResolver, + transformer: JsonTransformerFunction, + ) { + return super.matchObject(path, definition, resolver, transformer).then(result => this.auditAndReturn(path, result)); + } + override async matchInline( + path: string, + value: string, + resolver: ParameterResolver, + transformer: JsonTransformerFunction, + ) { + return super.matchInline(path, value, resolver, transformer).then(result => this.auditAndReturn(path, result)); + } +} diff --git a/javascript/json-transform/src/JsonElementStreamer.ts b/javascript/json-transform/src/JsonElementStreamer.ts index d4db158..65eb7a6 100644 --- a/javascript/json-transform/src/JsonElementStreamer.ts +++ b/javascript/json-transform/src/JsonElementStreamer.ts @@ -4,7 +4,7 @@ import { asAsyncSequence, AsyncSequence, emptyAsyncSequence } from "@wortise/seq class JsonElementStreamer { private readonly context: FunctionContext; private readonly transformed: boolean; - private readonly value?: any[]; + private value?: any[]; private readonly _stream?: AsyncSequence; private constructor(context: FunctionContext, value: any[] | AsyncSequence, transformed: boolean) { @@ -24,7 +24,7 @@ class JsonElementStreamer { } public stream(skip: number = 0, limit: number = -1) { - if (this._stream != null) { + if (this._stream != null && !this.value) { const skipped = skip > 0 ? this._stream.drop(skip) : this._stream; return limit > -1 ? skipped.take(limit) : skipped; } @@ -39,7 +39,7 @@ class JsonElementStreamer { valueStream = valueStream.take(limit); } if (!this.transformed) { - valueStream = valueStream.map(el => this.context.transform(el)); + valueStream = valueStream.map(el => this.context.transform(undefined, el)); } return valueStream; } @@ -57,7 +57,10 @@ class JsonElementStreamer { return this.value; } if (this._stream) { - return this._stream.toArray(); + return this._stream.toArray().then(arr => { + this.value = arr; + return arr; + }); } return []; } diff --git a/javascript/json-transform/src/JsonHelpers.ts b/javascript/json-transform/src/JsonHelpers.ts index 3118ee0..307730b 100644 --- a/javascript/json-transform/src/JsonHelpers.ts +++ b/javascript/json-transform/src/JsonHelpers.ts @@ -358,6 +358,12 @@ function createComparator(type: string | null) { return comparator; } +const VALID_ID_REGEXP = /^[a-z_$][a-z0-9_$]*$/i; + +function toObjectFieldPath(key: string) { + return VALID_ID_REGEXP.test(key) ? "." + key : "[" + JSON.stringify(key) + "]"; +} + export { isNullOrUndefined, isNullOrEmpty, @@ -373,4 +379,5 @@ export { isTruthy, isEqual, mergeInto, + toObjectFieldPath, }; diff --git a/javascript/json-transform/src/JsonTransformer.ts b/javascript/json-transform/src/JsonTransformer.ts index 3ecc271..c80aa48 100644 --- a/javascript/json-transform/src/JsonTransformer.ts +++ b/javascript/json-transform/src/JsonTransformer.ts @@ -1,43 +1,53 @@ import { Transformer } from "./Transformer"; import { JsonTransformerFunction } from "./JsonTransformerFunction"; import { ParameterResolver } from "./ParameterResolver"; -import { createPayloadResolver, isNullOrUndefined } from "./JsonHelpers"; -import transformerFunctions, { TransformerFunctions } from "./transformerFunctions"; +import { createPayloadResolver, isNullOrUndefined, toObjectFieldPath } from "./JsonHelpers"; +import transformerFunctions, { TransformerFunctionsAdapter } from "./transformerFunctions"; import JsonElementStreamer from "./JsonElementStreamer"; import BigNumber from "bignumber.js"; import { BigDecimal } from "./functions/common/FunctionHelpers"; +import DebuggableTransformerFunctions from "./DebuggableTransformerFunctions"; class JsonTransformer implements Transformer { static readonly OBJ_DESTRUCT_KEY = "*"; - static readonly FUNCTION_PREFIX = "$$"; + static readonly NULL_VALUE = "#null"; - private transformerFunctions: TransformerFunctions; + private transformerFunctions: TransformerFunctionsAdapter; private definition: any; private JSON_TRANSFORMER: JsonTransformerFunction; - constructor(definition: any) { - this.transformerFunctions = transformerFunctions; + constructor(definition: any, functionsAdapter?: TransformerFunctionsAdapter) { + this.transformerFunctions = functionsAdapter ?? transformerFunctions; this.definition = definition; this.JSON_TRANSFORMER = { transform: this.fromJsonElement.bind(this), }; } + static getDebuggableAdapter() { + return new DebuggableTransformerFunctions(); + } + async transform(payload: any = null, additionalContext: Record = {}) { if (isNullOrUndefined(this.definition)) { return null; } const resolver: ParameterResolver = createPayloadResolver(payload, additionalContext); - return this.fromJsonElement(this.definition, resolver, false); + return this.fromJsonElement("$", this.definition, resolver, false); } - async fromJsonPrimitive(definition: any, resolver: ParameterResolver, allowReturningStreams: boolean): Promise { + async fromJsonPrimitive( + path: string, + definition: any, + resolver: ParameterResolver, + allowReturningStreams: boolean, + ): Promise { if (typeof definition !== "string") { return definition ?? null; } try { // test for inline function (e.g. $$function:...) - const match = await this.transformerFunctions.matchInline(definition, resolver, this.JSON_TRANSFORMER); + const match = await this.transformerFunctions.matchInline(path, definition, resolver, this.JSON_TRANSFORMER); if (match != null) { const matchResult = match.getResult(); if (matchResult instanceof JsonElementStreamer) { @@ -53,8 +63,13 @@ class JsonTransformer implements Transformer { } } - async fromJsonObject(definition: any, resolver: ParameterResolver, allowReturningStreams: boolean): Promise { - const match = await this.transformerFunctions.matchObject(definition, resolver, this.JSON_TRANSFORMER); + async fromJsonObject( + path: string, + definition: any, + resolver: ParameterResolver, + allowReturningStreams: boolean, + ): Promise { + const match = await this.transformerFunctions.matchObject(path, definition, resolver, this.JSON_TRANSFORMER); if (match != null) { const res = match.getResult(); if (res instanceof JsonElementStreamer) { @@ -66,7 +81,7 @@ class JsonTransformer implements Transformer { let result: Record = {}; if (Object.prototype.hasOwnProperty.call(definition, JsonTransformer.OBJ_DESTRUCT_KEY)) { const val = definition[JsonTransformer.OBJ_DESTRUCT_KEY]; - const res = await this.fromJsonElement(val, resolver, false); + const res = await this.fromJsonElement(path + '["*"]', val, resolver, false); if (res != null) { const isArray = Array.isArray(val); if (isArray && Array.isArray(res)) { @@ -85,7 +100,12 @@ class JsonTransformer implements Transformer { for (const key in definition) { if (key === JsonTransformer.OBJ_DESTRUCT_KEY) continue; - const value = await this.fromJsonElement(definition[key], resolver, false); + const localValue = definition[key]; + if (localValue === JsonTransformer.NULL_VALUE) { + delete result[key]; + continue; + } + const value = await this.fromJsonElement(path + toObjectFieldPath(key), localValue, resolver, false); if ( !isNullOrUndefined(value) || Object.prototype.hasOwnProperty.call(result, key) /* we allow overriding with null*/ @@ -97,17 +117,22 @@ class JsonTransformer implements Transformer { return result; } - async fromJsonElement(definition: any, resolver: ParameterResolver, allowReturningStreams: boolean): Promise { + async fromJsonElement( + path: string, + definition: any, + resolver: ParameterResolver, + allowReturningStreams: boolean, + ): Promise { if (isNullOrUndefined(definition)) { return null; } if (Array.isArray(definition)) { - return Promise.all(definition.map((d: any) => this.fromJsonElement(d, resolver, false))); + return Promise.all(definition.map((d: any, i) => this.fromJsonElement(`${path}[${i}]`, d, resolver, false))); } if (typeof definition === "object" && !(definition instanceof BigNumber || definition instanceof BigDecimal)) { - return this.fromJsonObject(definition, resolver, allowReturningStreams); + return this.fromJsonObject(path, definition, resolver, allowReturningStreams); } - return this.fromJsonPrimitive(definition, resolver, allowReturningStreams); + return this.fromJsonPrimitive(path, definition, resolver, allowReturningStreams); } getDefinition() { diff --git a/javascript/json-transform/src/JsonTransformerFunction.ts b/javascript/json-transform/src/JsonTransformerFunction.ts index c255939..104f0b4 100644 --- a/javascript/json-transform/src/JsonTransformerFunction.ts +++ b/javascript/json-transform/src/JsonTransformerFunction.ts @@ -1,5 +1,5 @@ -import {ParameterResolver} from "./ParameterResolver"; +import { ParameterResolver } from "./ParameterResolver"; export interface JsonTransformerFunction { - transform(definition: any, resolver: ParameterResolver, allowReturningStreams?: boolean): Promise; -} \ No newline at end of file + transform(path: string, definition: any, resolver: ParameterResolver, allowReturningStreams?: boolean): Promise; +} diff --git a/javascript/json-transform/src/functions/TransformerFunctionDate.ts b/javascript/json-transform/src/functions/TransformerFunctionDate.ts index f3c636f..6639d81 100644 --- a/javascript/json-transform/src/functions/TransformerFunctionDate.ts +++ b/javascript/json-transform/src/functions/TransformerFunctionDate.ts @@ -1,4 +1,20 @@ -import { add, addMilliseconds, format, formatISO, fromUnixTime, sub, subMilliseconds, parseJSON } from "date-fns"; +import { + add, + addMilliseconds, + format, + formatISO, + fromUnixTime, + sub, + subMilliseconds, + parseJSON, + differenceInMilliseconds, + differenceInSeconds, + differenceInMinutes, + differenceInHours, + differenceInDays, + differenceInMonths, + differenceInYears, +} from "date-fns"; import { tz, TZDate } from "@date-fns/tz"; import BigNumber from "bignumber.js"; import TransformerFunction from "./common/TransformerFunction"; @@ -40,6 +56,7 @@ class TransformerFunctionDate extends TransformerFunction { pattern: { type: ArgType.String, position: 1 }, timezone: { type: ArgType.String, position: 2, defaultString: "UTC" }, zone: { type: ArgType.String, position: 1, defaultString: "UTC" }, + end: { type: ArgType.String, position: 2 }, }, }); } @@ -47,7 +64,12 @@ class TransformerFunctionDate extends TransformerFunction { private static parseInstant(value: any): Date { if (value instanceof Date) return value; if (typeof value === "string") { - if (value.includes("T") || value.includes("-")) return parseJSON(value); + if (value.includes("T")) { + return parseJSON(value); + } + if (value.includes("-")) { + return parseJSON(`${value}T00:00:00Z`); + } if (value.includes(":")) return parseJSON(`1970-01-01T${value}`); value = parseInt(value); } @@ -130,6 +152,33 @@ class TransformerFunctionDate extends TransformerFunction { case "DATE": { return formatISO(instant, { representation: "date" }); } + case "DIFF": { + const units = await context.getEnum("units"); + const end = TransformerFunctionDate.parseInstant(await context.get("end")); + switch (units) { + case "NANOS": + return differenceInMilliseconds(end, instant) * 1e6; + case "MICROS": + return differenceInMilliseconds(end, instant) * 1e3; + case "MILLIS": + return differenceInMilliseconds(end, instant); + case "SECONDS": + return differenceInSeconds(end, instant); + case "MINUTES": + return differenceInMinutes(end, instant); + case "HOURS": + return differenceInHours(end, instant); + case "HALF_DAYS": + return Math.trunc(differenceInHours(end, instant) / 12); + case "DAYS": + return differenceInDays(end, instant); + case "MONTHS": + return differenceInMonths(end, instant); + case "YEARS": + return differenceInYears(end, instant); + } + return null; + } case "EPOCH": { switch (await context.getEnum("resolution")) { case "MS": diff --git a/javascript/json-transform/src/functions/TransformerFunctionEval.ts b/javascript/json-transform/src/functions/TransformerFunctionEval.ts index 54dcd4d..50771ef 100644 --- a/javascript/json-transform/src/functions/TransformerFunctionEval.ts +++ b/javascript/json-transform/src/functions/TransformerFunctionEval.ts @@ -8,7 +8,7 @@ class TransformerFunctionEval extends TransformerFunction { override async apply(context: FunctionContext): Promise { const _eval = await context.getJsonElement(null); - return await context.transform(_eval, true); + return await context.transform(context.getPath() + "/.", _eval, true); } } diff --git a/javascript/json-transform/src/functions/TransformerFunctionIf.ts b/javascript/json-transform/src/functions/TransformerFunctionIf.ts index ba89e28..4cc8533 100644 --- a/javascript/json-transform/src/functions/TransformerFunctionIf.ts +++ b/javascript/json-transform/src/functions/TransformerFunctionIf.ts @@ -31,13 +31,13 @@ class TransformerFunctionIf extends TransformerFunction { } else if (typeof cje === "boolean") { condition = cje; } else { - condition = isTruthy(await context.transform(cje)); + condition = isTruthy(await context.transform(context.getPathFor(0), cje)); } if (condition) { - return await context.transform(arr[1]); + return await context.transform(context.getPathFor(1), arr[1]); } else if (arr.length > 2) { - return await context.transform(arr[2]); + return await context.transform(context.getPathFor(2), arr[2]); } } return null; // default falsy value diff --git a/javascript/json-transform/src/functions/TransformerFunctionLookup.ts b/javascript/json-transform/src/functions/TransformerFunctionLookup.ts index d5620dc..724228a 100644 --- a/javascript/json-transform/src/functions/TransformerFunctionLookup.ts +++ b/javascript/json-transform/src/functions/TransformerFunctionLookup.ts @@ -41,7 +41,7 @@ class TransformerFunctionLookup extends TransformerFunction { // collect using const withDef = using.with; if (isNullOrUndefined(withDef)) continue; // with - null - const $with = await context.transform(withDef); + const $with = await context.transform(context.getPathFor("with"), withDef); if (!Array.isArray($with)) continue; // with - not array usingMap[w] = { with: $with, as, on: using.on }; } diff --git a/javascript/json-transform/src/functions/TransformerFunctionMap.ts b/javascript/json-transform/src/functions/TransformerFunctionMap.ts index 24c2ed4..6e1eaa5 100644 --- a/javascript/json-transform/src/functions/TransformerFunctionMap.ts +++ b/javascript/json-transform/src/functions/TransformerFunctionMap.ts @@ -25,7 +25,7 @@ class TransformerFunctionMap extends TransformerFunction { // [ input, to ] const arr = await context.getJsonArray(null, false); // we don't transform definitions to prevent premature evaluation if (arr == null) return null; - const inputEl = await context.transform(arr[0]); + const inputEl = await context.transform(context.getPathFor(0), arr[0]); if (!Array.isArray(inputEl)) { console.warn(`${context.getAlias()} was not specified with an array of items`); return null; diff --git a/javascript/json-transform/src/functions/common/FunctionContext.ts b/javascript/json-transform/src/functions/common/FunctionContext.ts index 94ac442..57b4852 100644 --- a/javascript/json-transform/src/functions/common/FunctionContext.ts +++ b/javascript/json-transform/src/functions/common/FunctionContext.ts @@ -1,7 +1,14 @@ import TransformerFunction from "./TransformerFunction"; import { ParameterResolver } from "../../ParameterResolver"; import { JsonTransformerFunction } from "../../JsonTransformerFunction"; -import { compareTo, isNullOrUndefined, getAsString, getDocumentContext, isMap } from "../../JsonHelpers"; +import { + compareTo, + isNullOrUndefined, + getAsString, + getDocumentContext, + isMap, + toObjectFieldPath, +} from "../../JsonHelpers"; import { BigDecimal, MAX_SCALE_ROUNDING, RoundingModes } from "./FunctionHelpers"; import JsonElementStreamer from "../../JsonElementStreamer"; import BigNumber from "bignumber.js"; @@ -13,17 +20,20 @@ class FunctionContext { protected static readonly DOUBLE_HASH_INDEX = "##index"; protected static readonly DOLLAR = "$"; + protected readonly path: string; protected readonly alias: string; protected readonly function: TransformerFunction; protected readonly extractor: JsonTransformerFunction; protected resolver: ParameterResolver; protected constructor( + path: string, alias: string, func: TransformerFunction, resolver: ParameterResolver, extractor: JsonTransformerFunction, ) { + this.path = path; this.alias = alias; this.function = func; this.extractor = extractor; @@ -31,13 +41,16 @@ class FunctionContext { } protected static async recalcResolver( + path: string, contextElement: any, resolver: ParameterResolver, extractor: JsonTransformerFunction, ): Promise { const addCtx: Record = {}; for (const key in contextElement) { - addCtx[key] = getDocumentContext(await extractor.transform(contextElement[key], resolver, false)); + addCtx[key] = getDocumentContext( + await extractor.transform(path + toObjectFieldPath(key), contextElement[key], resolver, false), + ); } return { get: name => { @@ -65,6 +78,10 @@ class FunctionContext { return this.alias; } + public getPath() { + return this.path; + } + public getResolver() { return this.resolver; } @@ -77,6 +94,10 @@ class FunctionContext { return null; } + public getPathFor(key: number | string | null) { + return this.path + (!key ? "" : typeof key === "number" ? "[" + key + "]" : toObjectFieldPath(key)); + } + public isNull(value: any) { return isNullOrUndefined(value); } @@ -216,7 +237,7 @@ class FunctionContext { // in case val is already an array we don't transform it to prevent evaluation of its result values // so if is not an array, we must transform it and check after-wards (not lazy anymore) if (!Array.isArray(value)) { - value = await this.extractor.transform(value, this.resolver, true); + value = await this.extractor.transform(this.getPathFor(name), value, this.resolver, true); if (value instanceof JsonElementStreamer) { return value; } @@ -229,8 +250,8 @@ class FunctionContext { return null; } - public async transform(definition: any, allowReturningStreams: boolean = false) { - return await this.extractor.transform(definition, this.resolver, allowReturningStreams); + public async transform(path: string | undefined, definition: any, allowReturningStreams: boolean = false) { + return await this.extractor.transform(path ?? this.path, definition, this.resolver, allowReturningStreams); } public async transformItem( @@ -292,7 +313,7 @@ class FunctionContext { }, }; } - return this.extractor.transform(definition, itemResolver, false); + return this.extractor.transform("$", definition, itemResolver, false); } } diff --git a/javascript/json-transform/src/functions/common/InlineFunctionContext.ts b/javascript/json-transform/src/functions/common/InlineFunctionContext.ts index 7a6077d..6e6f2f0 100644 --- a/javascript/json-transform/src/functions/common/InlineFunctionContext.ts +++ b/javascript/json-transform/src/functions/common/InlineFunctionContext.ts @@ -5,6 +5,7 @@ class InlineFunctionContext extends FunctionContext { private args: any[]; private constructor( + path: string, input: string | null, args: any[], functionKey: string, @@ -12,12 +13,13 @@ class InlineFunctionContext extends FunctionContext { resolver: any, extractor: any, ) { - super(functionKey, func, resolver, extractor); + super(path, functionKey, func, resolver, extractor); this.stringInput = input; this.args = args; } public static create( + path: string, input: string | null, args: any[], functionKey: string, @@ -25,7 +27,7 @@ class InlineFunctionContext extends FunctionContext { resolver: any, extractor: any, ) { - return new InlineFunctionContext(input, args, functionKey, func, resolver, extractor); + return new InlineFunctionContext(path, input, args, functionKey, func, resolver, extractor); } override has(name: string): boolean { @@ -52,9 +54,13 @@ class InlineFunctionContext extends FunctionContext { } const argValue = name == null ? this.stringInput : this.args[argument?.position ?? -1]; if (typeof argValue === "string" && transform) { - return await this.extractor.transform(argValue, this.resolver, true); + return await this.extractor.transform(this.getPathFor(name), argValue, this.resolver, true); } - return !transform ? argValue : await this.extractor.transform(argValue, this.resolver, true); + return !transform ? argValue : await this.extractor.transform(this.getPathFor(name), argValue, this.resolver, true); + } + + override getPathFor(key: number | string | null): string { + return this.path + (key == null ? "" : `(${key})`); } } diff --git a/javascript/json-transform/src/functions/common/ObjectFunctionContext.ts b/javascript/json-transform/src/functions/common/ObjectFunctionContext.ts index 070abb6..eadc9ea 100644 --- a/javascript/json-transform/src/functions/common/ObjectFunctionContext.ts +++ b/javascript/json-transform/src/functions/common/ObjectFunctionContext.ts @@ -1,22 +1,29 @@ import FunctionContext from "./FunctionContext"; -import { isMap, isNullOrUndefined } from "../../JsonHelpers"; +import { isMap, isNullOrUndefined, toObjectFieldPath } from "../../JsonHelpers"; class ObjectFunctionContext extends FunctionContext { private definition: any; - private constructor(definition: any, functionKey: string, func: any, resolver: any, extractor: any) { - super(functionKey, func, resolver, extractor); + private constructor(path: string, definition: any, functionKey: string, func: any, resolver: any, extractor: any) { + super(path, functionKey, func, resolver, extractor); this.definition = definition; } - public static async createAsync(definition: any, functionKey: string, func: any, resolver: any, extractor: any) { + public static async createAsync( + path: string, + definition: any, + functionKey: string, + func: any, + resolver: any, + extractor: any, + ) { let objResolver = resolver; if (definition?.[FunctionContext.CONTEXT_KEY]) { const contextElement = definition[FunctionContext.CONTEXT_KEY]; if (isMap(contextElement)) { - objResolver = await FunctionContext.recalcResolver(contextElement, resolver, extractor); + objResolver = await FunctionContext.recalcResolver(path, contextElement, resolver, extractor); } } - return new ObjectFunctionContext(definition, functionKey, func, objResolver, extractor); + return new ObjectFunctionContext(path, definition, functionKey, func, objResolver, extractor); } override has(name: string): boolean { @@ -28,7 +35,11 @@ class ObjectFunctionContext extends FunctionContext { if (isNullOrUndefined(el)) { return this.function.getDefaultValue(name); } - return !transform ? el : await this.extractor.transform(el, this.resolver, true); + return !transform ? el : await this.extractor.transform(this.getPathFor(name), el, this.resolver, true); + } + + override getPathFor(key: number | string | null): string { + return this.path + (typeof key === "number" ? `[${key}]` : toObjectFieldPath(!key ? this.getAlias() : key)); } } diff --git a/javascript/json-transform/src/transformerFunctions.ts b/javascript/json-transform/src/transformerFunctions.ts index 21be601..f16e89f 100644 --- a/javascript/json-transform/src/transformerFunctions.ts +++ b/javascript/json-transform/src/transformerFunctions.ts @@ -6,19 +6,42 @@ import ObjectFunctionContext from "./functions/common/ObjectFunctionContext"; import InlineFunctionContext from "./functions/common/InlineFunctionContext"; import embeddedFunctions from "./functions"; -class FunctionMatchResult { - private readonly result; +/** + * The purpose of this class is to differentiate between null and null result + */ +export class FunctionMatchResult { + private readonly result: any; + private readonly resultPath: string; - constructor(result: any) { + constructor(result: any, resultPath: string) { this.result = result; + this.resultPath = resultPath; } getResult() { return this.result; } + getResultPath() { + return this.resultPath; + } +} + +export interface TransformerFunctionsAdapter { + matchObject( + path: string, + definition: any, + resolver: ParameterResolver, + transformer: JsonTransformerFunction, + ): Promise; + matchInline( + path: string, + value: string, + resolver: ParameterResolver, + transformer: JsonTransformerFunction, + ): Promise; } -export class TransformerFunctions { +export class TransformerFunctions implements TransformerFunctionsAdapter { private static readonly inlineFunctionRegex = /^\$\$(\w+)(\((.*?)\))?(:|$)/; private static readonly inlineFunctionArgsRegex = /('(\\'|[^'])*'|[^,]*)(?:,|$)/g; public static readonly FUNCTION_KEY_PREFIX = "$$"; @@ -48,7 +71,7 @@ export class TransformerFunctions { } } - async matchObject(definition: any, resolver: ParameterResolver, transformer: JsonTransformerFunction) { + async matchObject(path: string, definition: any, resolver: ParameterResolver, transformer: JsonTransformerFunction) { if (isNullOrUndefined(definition)) { return null; } @@ -58,18 +81,20 @@ export class TransformerFunctions { if (Object.prototype.hasOwnProperty.call(definition, TransformerFunctions.FUNCTION_KEY_PREFIX + key)) { const func = this.functions[key]; const context = await ObjectFunctionContext.createAsync( + path, definition, TransformerFunctions.FUNCTION_KEY_PREFIX + key, func, resolver, transformer, ); + const resolvedPath = path + "." + TransformerFunctions.FUNCTION_KEY_PREFIX + key; try { const result = await func.apply(context); - return new FunctionMatchResult(result); + return new FunctionMatchResult(result, resolvedPath); } catch (ex) { - console.warn("Failed running object function ", ex); - return new FunctionMatchResult(null); + console.warn(`Failed running object function (at ${resolvedPath})`, ex); + return new FunctionMatchResult(null, resolvedPath); } } } @@ -77,7 +102,12 @@ export class TransformerFunctions { return null; } - tryParseInlineFunction(value: string, resolver: ParameterResolver, transformer: JsonTransformerFunction) { + tryParseInlineFunction( + path: string, + value: string, + resolver: ParameterResolver, + transformer: JsonTransformerFunction, + ) { const match = value.match(TransformerFunctions.inlineFunctionRegex); if (match) { const functionKey = match[1]; @@ -117,27 +147,36 @@ export class TransformerFunctions { } else { input = value.substring(matchEndIndex); } - return InlineFunctionContext.create(input, args, functionKey, _function, resolver, transformer); + return InlineFunctionContext.create( + path + "/" + TransformerFunctions.FUNCTION_KEY_PREFIX + functionKey, + input, + args, + functionKey, + _function, + resolver, + transformer, + ); } } return null; } - async matchInline(value: string, resolver: ParameterResolver, transformer: JsonTransformerFunction) { + async matchInline(path: string, value: string, resolver: ParameterResolver, transformer: JsonTransformerFunction) { if (value == null) return null; - const context = this.tryParseInlineFunction(value, resolver, transformer); + const context = this.tryParseInlineFunction(path, value, resolver, transformer); if (context == null) { return null; } // at this point we detected an inline function, we must return a match result + const resolvedPath = context.getPathFor(null); try { const func = this.functions[context.getAlias()]; const result = await func.apply(context); - return new FunctionMatchResult(result); + return new FunctionMatchResult(result, resolvedPath); } catch (ex) { - console.warn("Failed running inline function ", ex); + console.warn(`Failed running inline function (at ${resolvedPath})`, ex); } - return new FunctionMatchResult(null); + return new FunctionMatchResult(null, resolvedPath); } } diff --git a/javascript/json-transform/test/server.ts b/javascript/json-transform/test/server.ts index d25b5a1..18708ae 100644 --- a/javascript/json-transform/test/server.ts +++ b/javascript/json-transform/test/server.ts @@ -21,11 +21,12 @@ const api: Record { const body = await parseBody(req); console.log("called with " + JSONBig.stringify(body)); - const t = new JsonTransformer(body.definition); + let functionsAdapter = body.debug ? JsonTransformer.getDebuggableAdapter() : undefined; + const t = new JsonTransformer(body.definition, functionsAdapter); try { const result = await t.transform(body.input, body.additionalContext); console.log("returning as result <" + JSONBig.stringify(result) + ">"); - send(res, 200, JSONHeaders, { result }); + send(res, 200, JSONHeaders, { result, debugResults: functionsAdapter?.getDebugResults() }); } catch (e: any) { send(res, 500, JSONHeaders, { error: e.message ?? e }); } diff --git a/ml/training/_merge.js b/ml/training/_merge.js new file mode 100644 index 0000000..153e594 --- /dev/null +++ b/ml/training/_merge.js @@ -0,0 +1,21 @@ +const fs = require('fs'); +const path = require('path'); +const files = fs.readdirSync(__dirname); +console.log(`found ${files.length} files`) +let result = "[\n"; +for (const file of files) { + if (!file.endsWith(".json")) { // just json files + console.log("skipping " + file); + continue; + } + const filepath = path.join(__dirname, file); + console.log("reading " + filepath); + const contents = fs.readFileSync(filepath, 'utf8'); + const json = JSON.parse(contents); + for (const input in json) { + result += ` { "inputs": ${JSON.stringify([input])}, "outputs": ${[JSON.stringify([JSON.stringify(json[input])])]} },\n`; + } +} +result = result.slice(0, -2) + "\n]" +console.log(result); +fs.writeFileSync(path.join(__dirname, "output", "merged.json"), result); diff --git a/ml/training/date.json b/ml/training/date.json index 58c67e2..9d9f08e 100644 --- a/ml/training/date.json +++ b/ml/training/date.json @@ -1,47 +1,53 @@ { - "converts the input to a date (object form)": [ + "converts the input to a date (inline form)": [ "$$date('date'):$" ], - "converts the input to an ISO formatted date (object form)": [ + "converts the input to an ISO formatted date (inline form)": [ "$$date(iso):$" ], - "converts the input to an ISO formatted date with 0 fractional seconds (object form)": [ + "converts the input to an ISO formatted date with 0 fractional seconds (inline form)": [ "$$date(ISO,0):$" ], - "converts the input to an ISO formatted date with 3 fractional seconds (object form)": [ + "converts the input to an ISO formatted date with 3 fractional seconds (inline form)": [ "$$date(iso,3):$" ], - "converts the input to an ISO formatted date with 6 fractional seconds (object form)": [ + "converts the input to an ISO formatted date with 6 fractional seconds (inline form)": [ "$$date(iso,6):$" ], - "converts the input to a date in the specified timezone (object form)": [ + "converts the input to a date in the specified timezone (inline form)": [ "$$date(ZONE,America/New_York):$" ], - "converts the input to a date in EST timezone (object form)": [ + "converts the input to a date in EST timezone (inline form)": [ "$$date(ZONE,EST):$" ], - "converts the input to a date in GMT timezone (object form)": [ + "converts the input to a date in GMT timezone (inline form)": [ "$$date(GMT):$" ], - "converts the input to a date with a specified format (object form)": [ + "converts the input to a date with a specified format (inline form)": [ "$$date(format,'yyyy-MM-dd HH:mm'):$" ], - "converts the input to a date with a specified format and timezone (object form)": [ + "converts the input to a date with a specified format and timezone (inline form)": [ "$$date(format,'yyyy-MM-dd HH:mm','America/New_York'):$" ], - "converts the input to a unix timestamp (object form)": [ + "converts the input to a unix timestamp (inline form)": [ "$$date(epoch):$" ], - "converts the input to a unix timestamp in milliseconds (object form)": [ + "converts the input to a unix timestamp in milliseconds (inline form)": [ "$$date(epoch,MS):$" ], - "converts the input to a date with the specified timezone (object form)": [ + "calculate the days difference between date in specified object under field 'a' to the date under field 'b' (inline form)": [ + "$$date(DIFF,DAYS,$.b):$.a" + ], + "calculate the seconds since 1970-01-01 to a specified date (inline form)": [ + "$$date(DIFF,SECONDS,$):1970-01-01" + ], + "converts the input to a date with the specified timezone (inline form)": [ "$$date(ZONE,$):2023-01-01T00:00:00Z" ], - "converts the input to a date with the specified unit and amount added (object form)": [ + "converts the input to a date with the specified unit and amount added (inline form)": [ "$$date(add,$.unit,'$.amount'):$.date" ], - "converts the input to a date with the specified unit and amount subtracted (object form)": [ + "converts the input to a date with the specified unit and amount subtracted (inline form)": [ "$$date(sub,YEARS,1):$" ] } \ No newline at end of file diff --git a/test/tests/functions/date.json b/test/tests/functions/date.json index 10d1fe8..2240e62 100644 --- a/test/tests/functions/date.json +++ b/test/tests/functions/date.json @@ -9,6 +9,16 @@ "equal": "2020-12-31T12:34:56.780Z" } }, + { + "name": "parse date only", + "given": { + "input": "2020-12-31", + "definition": "$$date:$" + }, + "expect": { + "equal": "2020-12-31T00:00:00Z" + } + }, { "name": "basic now with empty argument", "given": { @@ -352,5 +362,28 @@ "expect": { "equal": "2020-12-31T02:00:00Z" } + }, + { + "name": "difference between date in specified object under field 'a' to the date under field 'b'", + "given": { + "input": { + "a": "2024-01-01", + "b": "2025-01-01" + }, + "definition": "$$date(DIFF,DAYS,$.b):$.a" + }, + "expect": { + "equal": 366 + } + }, + { + "name": "seconds since 1970-01-01 to a specified date", + "given": { + "input": "2024-01-01", + "definition": "$$date(DIFF,SECONDS,$):1970-01-01" + }, + "expect": { + "equal": 1704067200 + } } ] \ No newline at end of file