From 7e4e4bd9b8089f7bae7d68eb3a381790d48f44ca Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Fri, 30 Aug 2024 15:14:13 +0200 Subject: [PATCH 01/55] Upgrade to Kafka 3.8.0 --- ksml-kafka-clients/NOTICE.txt | 12 +++---- .../ksml/client/admin/ResolvingAdmin.java | 35 ++++++++++--------- .../client/consumer/ForwardingConsumer.java | 10 +++--- .../client/producer/ForwardingProducer.java | 12 ++++--- .../client/producer/ResolvingProducer.java | 2 +- ksml/NOTICE.txt | 6 ++-- pom.xml | 2 +- 7 files changed, 43 insertions(+), 36 deletions(-) diff --git a/ksml-kafka-clients/NOTICE.txt b/ksml-kafka-clients/NOTICE.txt index 9776d03c..3c4a6dae 100644 --- a/ksml-kafka-clients/NOTICE.txt +++ b/ksml-kafka-clients/NOTICE.txt @@ -1,9 +1,9 @@ Lists of 20 third-party dependencies. - (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.13.5 - http://github.com/FasterXML/jackson) - (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.13.5 - https://github.com/FasterXML/jackson-core) - (The Apache Software License, Version 2.0) jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.13.5 - http://github.com/FasterXML/jackson) - (BSD 2-Clause License) zstd-jni (com.github.luben:zstd-jni:1.5.5-1 - https://github.com/luben/zstd-jni) + (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.16.2 - https://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.16.2 - https://github.com/FasterXML/jackson-core) + (The Apache Software License, Version 2.0) jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.16.2 - https://github.com/FasterXML/jackson) + (BSD 2-Clause License) zstd-jni (com.github.luben:zstd-jni:1.5.6-3 - https://github.com/luben/zstd-jni) (The Apache Software License, Version 2.0) FindBugs-jsr305 (com.google.code.findbugs:jsr305:3.0.2 - http://findbugs.sourceforge.net/) (Apache 2.0) error-prone annotations (com.google.errorprone:error_prone_annotations:2.26.1 - https://errorprone.info/error_prone_annotations) (The Apache Software License, Version 2.0) Guava InternalFutureFailureAccess and InternalFutures (com.google.guava:failureaccess:1.0.2 - https://github.com/google/guava/failureaccess) @@ -12,8 +12,8 @@ Lists of 20 third-party dependencies. (Apache License, Version 2.0) J2ObjC Annotations (com.google.j2objc:j2objc-annotations:3.0.0 - https://github.com/google/j2objc/) (Apache-2.0) Apache Commons Lang (org.apache.commons:commons-lang3:3.14.0 - https://commons.apache.org/proper/commons-lang/) (Apache-2.0) Apache Commons Text (org.apache.commons:commons-text:1.12.0 - https://commons.apache.org/proper/commons-text) - (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-clients:3.6.2 - https://kafka.apache.org) - (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-streams:3.6.2 - https://kafka.apache.org) + (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-clients:3.8.0 - https://kafka.apache.org) + (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-streams:3.8.0 - https://kafka.apache.org) (The MIT License) Checker Qual (org.checkerframework:checker-qual:3.42.0 - https://checkerframework.org/) (The Apache Software License, Version 2.0) LZ4 and xxHash (org.lz4:lz4-java:1.8.0 - https://github.com/lz4/lz4-java) (The MIT License) Project Lombok (org.projectlombok:lombok:1.18.32 - https://projectlombok.org) diff --git a/ksml-kafka-clients/src/main/java/io/axual/ksml/client/admin/ResolvingAdmin.java b/ksml-kafka-clients/src/main/java/io/axual/ksml/client/admin/ResolvingAdmin.java index 57be8e6f..e7739e86 100644 --- a/ksml-kafka-clients/src/main/java/io/axual/ksml/client/admin/ResolvingAdmin.java +++ b/ksml-kafka-clients/src/main/java/io/axual/ksml/client/admin/ResolvingAdmin.java @@ -42,7 +42,7 @@ public class ResolvingAdmin extends ForwardingAdmin { public ResolvingAdmin(Map configs) { super(configs); - var config = new ResolvingClientConfig(configs); + final var config = new ResolvingClientConfig(configs); topicResolver = config.topicResolver; groupResolver = config.groupResolver; } @@ -58,7 +58,7 @@ public CreateTopicsResult createTopics(Collection newTopics, CreateTop @Override public DeleteTopicsResult deleteTopics(TopicCollection topicCollection, DeleteTopicsOptions options) { - var result = super.deleteTopics(topicResolver.resolve(topicCollection), options); + final var result = super.deleteTopics(topicResolver.resolve(topicCollection), options); return new ResolvingDeleteTopicsResult(result.topicIdValues(), result.topicNameValues(), topicResolver); } @@ -69,13 +69,13 @@ public ListTopicsResult listTopics(ListTopicsOptions options) { @Override public DescribeTopicsResult describeTopics(TopicCollection topicCollection, DescribeTopicsOptions options) { - var result = super.describeTopics(topicResolver.resolve(topicCollection), options); + final var result = super.describeTopics(topicResolver.resolve(topicCollection), options); return new ResolvingDescribeTopicsResult(result.topicIdValues(), result.topicNameValues(), topicResolver); } @Override public DescribeAclsResult describeAcls(AclBindingFilter filter, DescribeAclsOptions options) { - var result = super.describeAcls(ResolverUtil.resolve(filter, topicResolver, groupResolver), options); + final var result = super.describeAcls(ResolverUtil.resolve(filter, topicResolver, groupResolver), options); if (result == null) return null; return new ResolvingDescribeAclsResult(result.values(), topicResolver, groupResolver); } @@ -156,28 +156,31 @@ public ListConsumerGroupsResult listConsumerGroups(ListConsumerGroupsOptions opt @Override public ListConsumerGroupOffsetsResult listConsumerGroupOffsets(Map groupSpecs, ListConsumerGroupOffsetsOptions options) { // Resolve the groupSpecs - var newGroupSpecs = new HashMap(); + final var newGroupSpecs = new HashMap(); groupSpecs.forEach((groupId, spec) -> newGroupSpecs.put(groupResolver.resolve(groupId), new ListConsumerGroupOffsetsSpec().topicPartitions(topicResolver.resolveTopicPartitions(spec.topicPartitions())))); // Resolve the options if (options != null) { - var newOptions = new ListConsumerGroupOffsetsOptions().requireStable(options.requireStable()); - if (options.topicPartitions() != null) { - var newTopicPartitions = new ArrayList(options.topicPartitions().size()); - options.topicPartitions().forEach(tp -> newTopicPartitions.add(topicResolver.resolve(tp))); + final var newOptions = new ListConsumerGroupOffsetsOptions().requireStable(options.requireStable()); + final var topicPartitions = options.topicPartitions(); + if (topicPartitions != null) { + final var newTopicPartitions = new ArrayList(topicPartitions.size()); + topicPartitions.forEach(tp -> newTopicPartitions.add(topicResolver.resolve(tp))); newOptions.topicPartitions(newTopicPartitions); } options = newOptions; } // Call the original API - var result = super.listConsumerGroupOffsets(newGroupSpecs, options); - // Convert the result to an unresolving result - var newResult = new HashMap>>(); + final var result = super.listConsumerGroupOffsets(newGroupSpecs, options); + + // Convert the result to an unresolved result + final var newResult = new HashMap>>(); newGroupSpecs.keySet().forEach(groupId -> { - var future = result.partitionsToOffsetAndMetadata(groupId); + final var future = result.partitionsToOffsetAndMetadata(groupId); newResult.put(CoordinatorKey.byGroupId(groupId), future); }); + return new ResolvingListConsumerGroupOffsetsResult(newResult, topicResolver, groupResolver); } @@ -262,9 +265,9 @@ public AlterClientQuotasResult alterClientQuotas(Collection resolveNewTopics(Collection newTopics) { // Resolve all new topics into a new collection - var resolvedTopics = new ArrayList(); - for (var newTopic : newTopics) { - var resolvedTopic = newTopic.replicasAssignments() == null + final var resolvedTopics = new ArrayList(); + for (final var newTopic : newTopics) { + final var resolvedTopic = newTopic.replicasAssignments() == null ? new NewTopic(topicResolver.resolve(newTopic.name()), newTopic.numPartitions(), newTopic.replicationFactor()) : new NewTopic(topicResolver.resolve(newTopic.name()), newTopic.replicasAssignments()); // Make sure that the config is added properly. Cleanup properties and timestamps are typical properties set in Streams diff --git a/ksml-kafka-clients/src/main/java/io/axual/ksml/client/consumer/ForwardingConsumer.java b/ksml-kafka-clients/src/main/java/io/axual/ksml/client/consumer/ForwardingConsumer.java index d6c7faa0..c37a5d23 100644 --- a/ksml-kafka-clients/src/main/java/io/axual/ksml/client/consumer/ForwardingConsumer.java +++ b/ksml-kafka-clients/src/main/java/io/axual/ksml/client/consumer/ForwardingConsumer.java @@ -21,10 +21,7 @@ */ import org.apache.kafka.clients.consumer.*; -import org.apache.kafka.common.Metric; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.*; import java.time.Duration; import java.util.*; @@ -181,6 +178,11 @@ public Map committed(Set part return delegate.committed(partitions, timeout); } + @Override + public Uuid clientInstanceId(Duration duration) { + return delegate.clientInstanceId(duration); + } + @Override public Map metrics() { return delegate.metrics(); diff --git a/ksml-kafka-clients/src/main/java/io/axual/ksml/client/producer/ForwardingProducer.java b/ksml-kafka-clients/src/main/java/io/axual/ksml/client/producer/ForwardingProducer.java index a9eebe27..b23133bc 100644 --- a/ksml-kafka-clients/src/main/java/io/axual/ksml/client/producer/ForwardingProducer.java +++ b/ksml-kafka-clients/src/main/java/io/axual/ksml/client/producer/ForwardingProducer.java @@ -4,7 +4,7 @@ * ========================LICENSE_START================================= * Extended Kafka clients for KSML * %% - * Copyright (C) 2021 - 2023 Axual B.V. + * Copyright (C) 2021 - 2024 Axual B.V. * %% * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,10 +26,7 @@ import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.Metric; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.*; import org.apache.kafka.common.errors.ProducerFencedException; import java.time.Duration; @@ -106,6 +103,11 @@ public List partitionsFor(String topic) { return delegate.metrics(); } + @Override + public Uuid clientInstanceId(Duration duration) { + return delegate.clientInstanceId(duration); + } + @Override public void close() { delegate.close(); diff --git a/ksml-kafka-clients/src/main/java/io/axual/ksml/client/producer/ResolvingProducer.java b/ksml-kafka-clients/src/main/java/io/axual/ksml/client/producer/ResolvingProducer.java index 817780ad..7c6716a9 100644 --- a/ksml-kafka-clients/src/main/java/io/axual/ksml/client/producer/ResolvingProducer.java +++ b/ksml-kafka-clients/src/main/java/io/axual/ksml/client/producer/ResolvingProducer.java @@ -54,11 +54,11 @@ private static RecordMetadata convertRecordMetadata(RecordMetadata input, String input.offset(), 0, input.timestamp(), - null, input.serializedKeySize(), input.serializedValueSize()); } + @Deprecated @Override public void sendOffsetsToTransaction(Map offsets, String consumerGroupId) throws ProducerFencedException { Map newOffsets = new HashMap<>(); diff --git a/ksml/NOTICE.txt b/ksml/NOTICE.txt index 885f149f..ae2728d2 100644 --- a/ksml/NOTICE.txt +++ b/ksml/NOTICE.txt @@ -8,7 +8,7 @@ Lists of 72 third-party dependencies. (The Apache Software License, Version 2.0) Jackson-dataformat-XML (com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.1 - https://github.com/FasterXML/jackson-dataformat-xml) (The Apache Software License, Version 2.0) Jackson-dataformat-YAML (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.17.1 - https://github.com/FasterXML/jackson-dataformats-text) (The Apache License, Version 2.0) Woodstox (com.fasterxml.woodstox:woodstox-core:6.6.2 - https://github.com/FasterXML/woodstox) - (BSD 2-Clause License) zstd-jni (com.github.luben:zstd-jni:1.5.5-1 - https://github.com/luben/zstd-jni) + (BSD 2-Clause License) zstd-jni (com.github.luben:zstd-jni:1.5.6-3 - https://github.com/luben/zstd-jni) (The Apache Software License, Version 2.0) FindBugs-jsr305 (com.google.code.findbugs:jsr305:3.0.2 - http://findbugs.sourceforge.net/) (Apache 2.0) error-prone annotations (com.google.errorprone:error_prone_annotations:2.26.1 - https://errorprone.info/error_prone_annotations) (The Apache Software License, Version 2.0) Guava InternalFutureFailureAccess and InternalFutures (com.google.guava:failureaccess:1.0.2 - https://github.com/google/guava/failureaccess) @@ -39,8 +39,8 @@ Lists of 72 third-party dependencies. (Apache-2.0) Apache Avro (org.apache.avro:avro:1.11.3 - https://avro.apache.org) (Apache-2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.26.2 - https://commons.apache.org/proper/commons-compress/) (Apache-2.0) Apache Commons Lang (org.apache.commons:commons-lang3:3.14.0 - https://commons.apache.org/proper/commons-lang/) - (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-clients:3.6.2 - https://kafka.apache.org) - (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-streams:3.6.2 - https://kafka.apache.org) + (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-clients:3.8.0 - https://kafka.apache.org) + (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-streams:3.8.0 - https://kafka.apache.org) (Bouncy Castle Licence) Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (org.bouncycastle:bcpkix-jdk18on:1.76 - https://www.bouncycastle.org/java.html) (Bouncy Castle Licence) Bouncy Castle Provider (org.bouncycastle:bcprov-jdk18on:1.76 - https://www.bouncycastle.org/java.html) (Bouncy Castle Licence) Bouncy Castle ASN.1 Extension and Utility APIs (org.bouncycastle:bcutil-jdk18on:1.76 - https://www.bouncycastle.org/java.html) diff --git a/pom.xml b/pom.xml index 6f3223ea..a05465a0 100644 --- a/pom.xml +++ b/pom.xml @@ -64,7 +64,7 @@ 1.11.3 1.12.0 1.26.2 - 3.6.2 + 3.8.0 7.6.1 23.1.2 23.1.2 From f853d25787115d4915e4aa4b6fdffd651fae5690 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Mon, 23 Sep 2024 08:33:59 +0200 Subject: [PATCH 02/55] Update byte manipulation example - Generate example data for byte manipulation example - Have byte manipulation example read from alternative topic --- examples/00-example-generate-sensordata.yaml | 10 ++++++++++ examples/12-example-byte-manipulation.yaml | 10 +++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/examples/00-example-generate-sensordata.yaml b/examples/00-example-generate-sensordata.yaml index c21822a1..46b73278 100644 --- a/examples/00-example-generate-sensordata.yaml +++ b/examples/00-example-generate-sensordata.yaml @@ -44,3 +44,13 @@ producers: topic: ksml_sensordata_avro keyType: string valueType: avro:SensorData + + # The following producer generates messages for 12-example-byte-manipulation.yaml. Enable this producer if you + # want to run that specific example. + sensordata_avro_producer_binary: + generator: generate_sensordata_message + interval: 3s + to: + topic: ksml_sensordata_avro_binary + keyType: string + valueType: avro:SensorData diff --git a/examples/12-example-byte-manipulation.yaml b/examples/12-example-byte-manipulation.yaml index efa7c011..9973b6fa 100644 --- a/examples/12-example-byte-manipulation.yaml +++ b/examples/12-example-byte-manipulation.yaml @@ -8,12 +8,12 @@ # message contains a schema id that is not locally recognized. By changing bytes 1-4 in the value, # one can override the schema id for further downstream processing. -# Yes, this is hacky, but it may serve a purpose for cases where binary copies are made from -# remote Kafka clusters that contain conflicting schema ids. +# Yes, this is hacky, but it may serve a purpose for cases where binary message copies are made from +# remote Kafka clusters with their own (possibly conflicting) schema ids. streams: - sensor_source: - topic: ksml_sensordata_avro + sensor_binary_source: + topic: ksml_sensordata_avro_binary keyType: string valueType: bytes offsetResetPolicy: latest @@ -44,7 +44,7 @@ functions: pipelines: main: - from: sensor_source + from: sensor_binary_source via: - type: peek forEach: From 2823e9f43e64512e58f572e72d0879a446960705 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Thu, 10 Oct 2024 17:16:06 +0200 Subject: [PATCH 03/55] Introduce notation factories to flexibly choose between notation serdes --- ksml-data-avro/NOTICE.txt | 40 +++++- ksml-data-avro/pom.xml | 4 + .../ksml/data/notation/avro/AvroNotation.java | 82 ++++++++---- .../data/notation/avro/MockAvroNotation.java | 15 ++- .../notation/csv/CsvDataObjectConverter.java | 1 - .../ksml/data/notation/csv/CsvNotation.java | 17 ++- .../ksml/data/notation/soap/SOAPNotation.java | 14 +- .../ksml/data/notation/xml/XmlNotation.java | 17 ++- .../io/axual/ksml/data/notation/Notation.java | 7 +- .../ksml/data/notation/NotationLibrary.java | 21 +-- .../data/notation/binary/BinaryNotation.java | 13 +- .../ksml/data/notation/json/JsonNotation.java | 17 ++- .../data/notation/string/StringNotation.java | 4 +- .../axual/ksml/data/schema/SchemaLibrary.java | 12 +- .../io/axual/ksml/client/util/MapUtil.java | 6 + ksml-query/NOTICE.txt | 37 +++++- ksml-reporting/NOTICE.txt | 38 +++++- ksml-runner/NOTICE.txt | 37 +++++- .../java/io/axual/ksml/runner/KSMLRunner.java | 124 ++++++++---------- .../java/io/axual/ksml/runner/KsmlInfo.java | 3 - .../runner/backend/ExecutableProducer.java | 4 +- .../runner/backend/KafkaStreamsRunner.java | 2 +- .../axual/ksml/runner/config/KSMLConfig.java | 8 ++ .../config/KSMLErrorHandlingConfig.java | 14 +- .../ksml/runner/config/KSMLRunnerConfig.java | 18 +-- .../ksml/runner/config/NotationConfig.java | 40 ++++++ .../java/io/axual/ksml/runner/lombok.config | 3 + .../runner/notation/NotationFactories.java | 97 ++++++++++++++ .../runner/prometheus/PrometheusExport.java | 2 +- .../backend/KafkaProducerRunnerTest.java | 7 +- .../runner/config/KSMLRunnerConfigTest.java | 4 +- ksml/NOTICE.txt | 39 +++++- .../ksml/data/mapper/DataObjectConverter.java | 4 +- .../axual/ksml/generator/StreamDataType.java | 2 +- .../io/axual/ksml/parser/UserTypeParser.java | 105 +++++---------- .../io/axual/ksml/BasicStreamRunTest.java | 21 ++- .../ksml/TopologyGeneratorBasicTest.java | 7 +- .../ksml/testutil/KSMLTestExtension.java | 23 ++-- pom.xml | 6 + 39 files changed, 633 insertions(+), 282 deletions(-) create mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/config/NotationConfig.java create mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config create mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/notation/NotationFactories.java diff --git a/ksml-data-avro/NOTICE.txt b/ksml-data-avro/NOTICE.txt index d6530302..5c8b2038 100644 --- a/ksml-data-avro/NOTICE.txt +++ b/ksml-data-avro/NOTICE.txt @@ -1,5 +1,5 @@ -Lists of 34 third-party dependencies. +Lists of 72 third-party dependencies. (MIT License) minimal-json (com.eclipsesource.minimal-json:minimal-json:0.9.5 - https://github.com/ralfstx/minimal-json) (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.17.1 - https://github.com/FasterXML/jackson) (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.17.1 - https://github.com/FasterXML/jackson-core) @@ -14,8 +14,17 @@ Lists of 34 third-party dependencies. (The Apache Software License, Version 2.0) Guava ListenableFuture only (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava - https://github.com/google/guava/listenablefuture) (Apache License, Version 2.0) J2ObjC Annotations (com.google.j2objc:j2objc-annotations:3.0.0 - https://github.com/google/j2objc/) (Go License) RE2/J (com.google.re2j:re2j:1.6 - http://github.com/google/re2j) + (MIT License) Microsoft Kiota-Java Abstractions (com.microsoft.kiota:microsoft-kiota-abstractions:1.1.14 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Form (com.microsoft.kiota:microsoft-kiota-serialization-form:1.1.14 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Multipart (com.microsoft.kiota:microsoft-kiota-serialization-multipart:1.1.14 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Text (com.microsoft.kiota:microsoft-kiota-serialization-text:1.1.14 - https://github.com/microsoft/kiota-java) (Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.17.0 - https://commons.apache.org/proper/commons-codec/) (Apache-2.0) Apache Commons IO (commons-io:commons-io:2.16.1 - https://commons.apache.org/proper/commons-io/) + (Apache License Version 2.0) apicurio-registry-common (io.apicurio:apicurio-registry-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-common/) + (Apache License Version 2.0) apicurio-registry-java-sdk (io.apicurio:apicurio-registry-java-sdk:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-java-sdk/) + (Apache License Version 2.0) apicurio-registry-schema-resolver (io.apicurio:apicurio-registry-schema-resolver:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-schema-resolver/) + (Apache License Version 2.0) apicurio-registry-serde-common (io.apicurio:apicurio-registry-serde-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serde-common/) + (Apache License Version 2.0) apicurio-registry-serdes-avro-serde (io.apicurio:apicurio-registry-serdes-avro-serde:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serdes-avro-serde/) (Apache 2.0) KSML Data Library (io.axual.ksml:ksml-data:1.0.3-SNAPSHOT - https://github.com/Axual/ksml/ksml-data) (Apache License 2.0) utils (io.confluent:common-utils:7.6.1 - https://confluent.io/common-utils) (Apache License 2.0) kafka-avro-serializer (io.confluent:kafka-avro-serializer:7.6.1 - http://confluent.io/kafka-avro-serializer) @@ -23,12 +32,41 @@ Lists of 34 third-party dependencies. (Apache License 2.0) kafka-schema-serializer (io.confluent:kafka-schema-serializer:7.6.1 - http://confluent.io/kafka-schema-serializer) (The Apache Software License, Version 2.0) Log Redactor (io.confluent:logredactor:1.0.12 - https://github.com/confluentinc/logredactor) (The Apache Software License, Version 2.0) Log Redactor Metrics (io.confluent:logredactor-metrics:1.0.12 - https://github.com/confluentinc/logredactor) + (Apache License, Version 2.0) Standard Uri Template (io.github.std-uritemplate:std-uritemplate:0.0.59 - https://std-uritemplate.github.io/) + (Apache-2.0) Kiota - Http - Vert.X (io.kiota:kiota-http-vertx:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-http-vertx) + (Apache-2.0) Kiota - Serialization - Jackson (io.kiota:kiota-serialization-jackson:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-serialization-jackson-parent/kiota-serialization-jackson) + (Apache License, Version 2.0) Netty/Buffer (io.netty:netty-buffer:4.1.108.Final - https://netty.io/netty-buffer/) + (Apache License, Version 2.0) Netty/Codec (io.netty:netty-codec:4.1.108.Final - https://netty.io/netty-codec/) + (Apache License, Version 2.0) Netty/Codec/DNS (io.netty:netty-codec-dns:4.1.108.Final - https://netty.io/netty-codec-dns/) + (Apache License, Version 2.0) Netty/Codec/HTTP (io.netty:netty-codec-http:4.1.108.Final - https://netty.io/netty-codec-http/) + (Apache License, Version 2.0) Netty/Codec/HTTP2 (io.netty:netty-codec-http2:4.1.108.Final - https://netty.io/netty-codec-http2/) + (Apache License, Version 2.0) Netty/Codec/Socks (io.netty:netty-codec-socks:4.1.108.Final - https://netty.io/netty-codec-socks/) + (Apache License, Version 2.0) Netty/Common (io.netty:netty-common:4.1.108.Final - https://netty.io/netty-common/) + (Apache License, Version 2.0) Netty/Handler (io.netty:netty-handler:4.1.108.Final - https://netty.io/netty-handler/) + (Apache License, Version 2.0) Netty/Handler/Proxy (io.netty:netty-handler-proxy:4.1.108.Final - https://netty.io/netty-handler-proxy/) + (Apache License, Version 2.0) Netty/Resolver (io.netty:netty-resolver:4.1.108.Final - https://netty.io/netty-resolver/) + (Apache License, Version 2.0) Netty/Resolver/DNS (io.netty:netty-resolver-dns:4.1.108.Final - https://netty.io/netty-resolver-dns/) + (Apache License, Version 2.0) Netty/Transport (io.netty:netty-transport:4.1.108.Final - https://netty.io/netty-transport/) + (Apache License, Version 2.0) Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.108.Final - https://netty.io/netty-transport-native-unix-common/) + (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-api:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) + (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-context:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) (Apache License 2.0) swagger-annotations (io.swagger.core.v3:swagger-annotations:2.1.10 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-common (io.vertx:vertx-auth-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-common) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-oauth2 (io.vertx:vertx-auth-oauth2:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-oauth2) + (Eclipse Public License - v 2.0) (The Apache Software License, Version 2.0) Vert.x Core (io.vertx:vertx-core:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-core) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) Vert.x URI Template (io.vertx:vertx-uri-template:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-uri-template) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-client (io.vertx:vertx-web-client:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-client) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-common (io.vertx:vertx-web-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-common) + (EPL 2.0) (GPL2 w/ CPE) Jakarta Annotations API (jakarta.annotation:jakarta.annotation-api:3.0.0 - https://projects.eclipse.org/projects/ee4j.ca) + (Apache License 2.0) Jakarta Bean Validation API (jakarta.validation:jakarta.validation-api:3.0.2 - https://beanvalidation.org) + (EPL-2.0) (GPL-2.0-with-classpath-exception) Jakarta RESTful WS API (jakarta.ws.rs:jakarta.ws.rs-api:3.1.0 - https://github.com/eclipse-ee4j/jaxrs-api) (Apache-2.0) Apache Avro (org.apache.avro:avro:1.11.3 - https://avro.apache.org) (Apache-2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.26.2 - https://commons.apache.org/proper/commons-compress/) (Apache-2.0) Apache Commons Lang (org.apache.commons:commons-lang3:3.14.0 - https://commons.apache.org/proper/commons-lang/) (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-clients:3.8.0 - https://kafka.apache.org) (The MIT License) Checker Qual (org.checkerframework:checker-qual:3.42.0 - https://checkerframework.org/) + (Apache License, version 2.0) JBoss Logging 3 (org.jboss.logging:jboss-logging:3.4.1.Final - http://www.jboss.org) + (Apache License 2.0) slf4j to JBoss Logging Adapter (org.jboss.slf4j:slf4j-jboss-logging:1.2.1.Final - http://www.jboss.org) (The Apache Software License, Version 2.0) LZ4 and xxHash (org.lz4:lz4-java:1.8.0 - https://github.com/lz4/lz4-java) (The MIT License) Project Lombok (org.projectlombok:lombok:1.18.32 - https://projectlombok.org) (MIT License) SLF4J API Module (org.slf4j:slf4j-api:2.0.13 - http://www.slf4j.org) diff --git a/ksml-data-avro/pom.xml b/ksml-data-avro/pom.xml index 00016b2d..01fdc9d4 100644 --- a/ksml-data-avro/pom.xml +++ b/ksml-data-avro/pom.xml @@ -43,6 +43,10 @@ + + io.apicurio + apicurio-registry-serdes-avro-serde + io.confluent kafka-avro-serializer diff --git a/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/AvroNotation.java b/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/AvroNotation.java index ecd46821..93a9da6f 100644 --- a/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/AvroNotation.java +++ b/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/AvroNotation.java @@ -21,15 +21,20 @@ */ import com.google.common.collect.ImmutableMap; +import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; +import io.apicurio.registry.serde.avro.AvroKafkaSerializer; import io.axual.ksml.data.exception.DataException; import io.axual.ksml.data.exception.ExecutionException; +import io.axual.ksml.data.loader.SchemaLoader; import io.axual.ksml.data.mapper.NativeDataObjectMapper; import io.axual.ksml.data.notation.Notation; +import io.axual.ksml.data.notation.NotationConverter; import io.axual.ksml.data.type.DataType; import io.axual.ksml.data.type.MapType; import io.axual.ksml.data.type.StructType; import io.confluent.kafka.serializers.KafkaAvroDeserializer; import io.confluent.kafka.serializers.KafkaAvroSerializer; +import lombok.Getter; import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serializer; @@ -37,22 +42,32 @@ import java.util.Map; public class AvroNotation implements Notation { - public static final String NOTATION_NAME = "AVRO"; public static final DataType DEFAULT_TYPE = new StructType(); + + public enum SerdeType { + APICURIO, + CONFLUENT + } + private static final AvroDataObjectMapper mapper = new AvroDataObjectMapper(); + private final SerdeType serdeType; private final NativeDataObjectMapper nativeMapper; private final Map serdeConfigs; + @Getter + private final SchemaLoader loader; private Serde keySerde; private Serde valueSerde; - public AvroNotation(NativeDataObjectMapper nativeMapper, Map configs) { + public AvroNotation(SerdeType type, NativeDataObjectMapper nativeMapper, Map configs, SchemaLoader loader) { + this.serdeType = type; this.nativeMapper = nativeMapper; this.serdeConfigs = ImmutableMap.copyOf(configs); + this.loader = loader; } @Override - public String name() { - return NOTATION_NAME; + public DataType defaultType() { + return DEFAULT_TYPE; } @Override @@ -61,37 +76,52 @@ public Serde serde(DataType type, boolean isKey) { // Create the serdes only upon request to prevent error messages on missing SR url configs if AVRO is not used if (keySerde == null) { - keySerde = new AvroSerde(); + keySerde = new AvroSerde(serdeType); keySerde.configure(serdeConfigs, true); } if (valueSerde == null) { - valueSerde = new AvroSerde(); + valueSerde = new AvroSerde(serdeType); valueSerde.configure(serdeConfigs, false); } return isKey ? keySerde : valueSerde; } + public NotationConverter converter() { + return null; + } + private class AvroSerde implements Serde { - private final Serializer serializer = new KafkaAvroSerializer(); - private final Deserializer deserializer = new KafkaAvroDeserializer(); - - private final Serializer wrapSerializer = - (topic, data) -> { - try { - return serializer.serialize(topic, mapper.fromDataObject(nativeMapper.toDataObject(data))); - } catch (Exception e) { - throw new ExecutionException("Error serializing AVRO message to topic " + topic, e); - } - }; - - private final Deserializer wrapDeserializer = - (topic, data) -> { - try { - return mapper.toDataObject(deserializer.deserialize(topic, data)); - } catch (Exception e) { - throw new ExecutionException("Error deserializing AVRO message from topic " + topic, e); - } - }; + private final Serializer serializer; + private final Deserializer deserializer; + private final Serializer wrapSerializer; + private final Deserializer wrapDeserializer; + + public AvroSerde(SerdeType type) { + serializer = switch (type) { + case APICURIO -> new AvroKafkaSerializer<>(); + case CONFLUENT -> new KafkaAvroSerializer(); + }; + deserializer = switch (type) { + case APICURIO -> new AvroKafkaDeserializer<>(); + case CONFLUENT -> new KafkaAvroDeserializer(); + }; + + wrapSerializer = (topic, data) -> { + try { + return serializer.serialize(topic, mapper.fromDataObject(nativeMapper.toDataObject(data))); + } catch (Exception e) { + throw new ExecutionException("Error serializing AVRO message to topic " + topic, e); + } + }; + + wrapDeserializer = (topic, data) -> { + try { + return mapper.toDataObject(deserializer.deserialize(topic, data)); + } catch (Exception e) { + throw new ExecutionException("Error deserializing AVRO message from topic " + topic, e); + } + }; + } @Override public void configure(Map configs, boolean isKey) { diff --git a/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/MockAvroNotation.java b/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/MockAvroNotation.java index 21ca2ef5..5800f892 100644 --- a/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/MockAvroNotation.java +++ b/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/MockAvroNotation.java @@ -22,8 +22,10 @@ import io.axual.ksml.data.exception.DataException; import io.axual.ksml.data.exception.ExecutionException; +import io.axual.ksml.data.loader.SchemaLoader; import io.axual.ksml.data.mapper.NativeDataObjectMapper; import io.axual.ksml.data.notation.Notation; +import io.axual.ksml.data.notation.NotationConverter; import io.axual.ksml.data.type.DataType; import io.axual.ksml.data.type.MapType; import io.axual.ksml.data.type.StructType; @@ -51,17 +53,22 @@ * instance of {@link MockSchemaRegistry} to be used. */ public class MockAvroNotation implements Notation { - public static final String NOTATION_NAME = "AVRO"; + public static final String NOTATION_NAME = "avro"; public static final DataType DEFAULT_TYPE = new StructType(); private static final AvroDataObjectMapper mapper = new AvroDataObjectMapper(); private final Map configs = new HashMap<>(); @Getter private final SyncMockSchemaRegistryClient mockSchemaRegistryClient = new SyncMockSchemaRegistryClient(); + @Getter + private final NotationConverter converter = null; + @Getter + private final SchemaLoader loader; - public MockAvroNotation(Map configs) { + public MockAvroNotation(Map configs, SchemaLoader loader) { this.configs.putAll(configs); this.configs.put("schema.registry.url", "mock://mock-scope"); this.configs.put(KafkaAvroDeserializerConfig.AUTO_REGISTER_SCHEMAS, true); + this.loader = loader; } public Map getSchemaRegistryConfigs() { @@ -76,8 +83,8 @@ public void registerSubjectSchema(String subject, Schema schema) throws RestClie } @Override - public String name() { - return NOTATION_NAME; + public DataType defaultType() { + return DEFAULT_TYPE; } @Override diff --git a/ksml-data-csv/src/main/java/io/axual/ksml/data/notation/csv/CsvDataObjectConverter.java b/ksml-data-csv/src/main/java/io/axual/ksml/data/notation/csv/CsvDataObjectConverter.java index 4833cb95..ecb4c7bb 100644 --- a/ksml-data-csv/src/main/java/io/axual/ksml/data/notation/csv/CsvDataObjectConverter.java +++ b/ksml-data-csv/src/main/java/io/axual/ksml/data/notation/csv/CsvDataObjectConverter.java @@ -20,7 +20,6 @@ * =========================LICENSE_END================================== */ -import io.axual.ksml.data.mapper.NativeDataObjectMapper; import io.axual.ksml.data.notation.NotationConverter; import io.axual.ksml.data.notation.UserType; import io.axual.ksml.data.object.DataList; diff --git a/ksml-data-csv/src/main/java/io/axual/ksml/data/notation/csv/CsvNotation.java b/ksml-data-csv/src/main/java/io/axual/ksml/data/notation/csv/CsvNotation.java index 4a6eb04b..777b4396 100644 --- a/ksml-data-csv/src/main/java/io/axual/ksml/data/notation/csv/CsvNotation.java +++ b/ksml-data-csv/src/main/java/io/axual/ksml/data/notation/csv/CsvNotation.java @@ -20,25 +20,32 @@ * =========================LICENSE_END================================== */ +import io.axual.ksml.data.loader.SchemaLoader; import io.axual.ksml.data.mapper.NativeDataObjectMapper; +import io.axual.ksml.data.notation.NotationConverter; import io.axual.ksml.data.notation.string.StringNotation; import io.axual.ksml.data.type.DataType; import io.axual.ksml.data.type.ListType; import io.axual.ksml.data.type.StructType; import io.axual.ksml.data.type.UnionType; +import lombok.Getter; import org.apache.kafka.common.serialization.Serde; public class CsvNotation extends StringNotation { - public static final String NOTATION_NAME = "CSV"; public static final DataType DEFAULT_TYPE = new UnionType(new StructType(), new ListType()); + @Getter + private final NotationConverter converter = new CsvDataObjectConverter(); + @Getter + private final SchemaLoader loader; - public CsvNotation(NativeDataObjectMapper nativeMapper) { + public CsvNotation(NativeDataObjectMapper nativeMapper, SchemaLoader loader) { super(nativeMapper, new CsvDataObjectMapper()); + this.loader = loader; } @Override - public String name() { - return NOTATION_NAME; + public DataType defaultType() { + return DEFAULT_TYPE; } @Override @@ -47,6 +54,6 @@ public Serde serde(DataType type, boolean isKey) { if (type instanceof ListType || type instanceof StructType || DEFAULT_TYPE.equals(type)) return super.serde(type, isKey); // Other types can not be serialized as XML - throw noSerdeFor(type); + throw noSerdeFor("CSV", type); } } diff --git a/ksml-data-soap/src/main/java/io/axual/ksml/data/notation/soap/SOAPNotation.java b/ksml-data-soap/src/main/java/io/axual/ksml/data/notation/soap/SOAPNotation.java index d881db3b..874aab30 100644 --- a/ksml-data-soap/src/main/java/io/axual/ksml/data/notation/soap/SOAPNotation.java +++ b/ksml-data-soap/src/main/java/io/axual/ksml/data/notation/soap/SOAPNotation.java @@ -20,23 +20,29 @@ * =========================LICENSE_END================================== */ +import io.axual.ksml.data.loader.SchemaLoader; import io.axual.ksml.data.mapper.DataObjectMapper; import io.axual.ksml.data.mapper.NativeDataObjectMapper; +import io.axual.ksml.data.notation.NotationConverter; import io.axual.ksml.data.notation.string.StringNotation; import io.axual.ksml.data.object.DataObject; import io.axual.ksml.data.schema.AnySchema; import io.axual.ksml.data.type.DataType; import io.axual.ksml.data.type.MapType; import io.axual.ksml.data.type.StructType; +import lombok.Getter; import org.apache.kafka.common.serialization.Serde; import static io.axual.ksml.data.notation.soap.SOAPSchema.generateSOAPSchema; public class SOAPNotation extends StringNotation { - public static final String NOTATION_NAME = "SOAP"; public static final DataType DEFAULT_TYPE = new StructType(generateSOAPSchema(AnySchema.INSTANCE)); private static final SOAPDataObjectMapper DATA_OBJECT_MAPPER = new SOAPDataObjectMapper(); private static final SOAPStringMapper STRING_MAPPER = new SOAPStringMapper(); + @Getter + private final NotationConverter converter = new SOAPDataObjectConverter(); + @Getter + private final SchemaLoader loader = null; public SOAPNotation(NativeDataObjectMapper nativeMapper) { super(nativeMapper, new DataObjectMapper<>() { @@ -53,8 +59,8 @@ public String fromDataObject(DataObject value) { } @Override - public String name() { - return NOTATION_NAME; + public DataType defaultType() { + return DEFAULT_TYPE; } @Override @@ -62,6 +68,6 @@ public Serde serde(DataType type, boolean isKey) { // SOAP types should ways be Maps (or Structs) if (type instanceof MapType) return super.serde(type, isKey); // Other types can not be serialized as SOAP - throw noSerdeFor(type); + throw noSerdeFor("SOAP", type); } } diff --git a/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlNotation.java b/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlNotation.java index 31980be9..b712ffcf 100644 --- a/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlNotation.java +++ b/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlNotation.java @@ -20,26 +20,33 @@ * =========================LICENSE_END================================== */ +import io.axual.ksml.data.loader.SchemaLoader; import io.axual.ksml.data.mapper.DataObjectMapper; import io.axual.ksml.data.mapper.NativeDataObjectMapper; +import io.axual.ksml.data.notation.NotationConverter; import io.axual.ksml.data.notation.string.StringNotation; import io.axual.ksml.data.type.DataType; import io.axual.ksml.data.type.MapType; import io.axual.ksml.data.type.StructType; +import lombok.Getter; import org.apache.kafka.common.serialization.Serde; public class XmlNotation extends StringNotation { - public static final String NOTATION_NAME = "XML"; public static final DataType DEFAULT_TYPE = new StructType(); private static final DataObjectMapper MAPPER = new XmlDataObjectMapper(); + @Getter + private final NotationConverter converter = new XmlDataObjectConverter(); + @Getter + private final SchemaLoader loader; - public XmlNotation(NativeDataObjectMapper nativeMapper) { + public XmlNotation(NativeDataObjectMapper nativeMapper, SchemaLoader loader) { super(nativeMapper, MAPPER); + this.loader = loader; } @Override - public String name() { - return NOTATION_NAME; + public DataType defaultType() { + return DEFAULT_TYPE; } @Override @@ -47,6 +54,6 @@ public Serde serde(DataType type, boolean isKey) { // XML types should ways be Maps (or Structs) if (type instanceof MapType) return super.serde(type, isKey); // Other types can not be serialized as XML - throw noSerdeFor(type); + throw noSerdeFor("XML", type); } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/Notation.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/Notation.java index 79b920ff..935f6249 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/Notation.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/Notation.java @@ -20,11 +20,16 @@ * =========================LICENSE_END================================== */ +import io.axual.ksml.data.loader.SchemaLoader; import io.axual.ksml.data.type.DataType; import org.apache.kafka.common.serialization.Serde; public interface Notation { - String name(); + DataType defaultType(); Serde serde(DataType type, boolean isKey); + + NotationConverter converter(); + + SchemaLoader loader(); } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/NotationLibrary.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/NotationLibrary.java index a3923f95..a3a63c60 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/NotationLibrary.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/NotationLibrary.java @@ -29,28 +29,19 @@ public class NotationLibrary { private NotationLibrary() { } - private record NotationEntry(Notation notation, NotationConverter converter) { - } - - private static final Map notationEntries = new HashMap<>(); + private static final Map notationEntries = new HashMap<>(); public static void register(String name, Notation notation) { - register(name, notation, null); + notationEntries.put(name, notation); } - public static void register(String name, Notation notation, NotationConverter converter) { - notationEntries.put(name, new NotationEntry(notation, converter)); + public static boolean exists(String notation) { + return notationEntries.containsKey(notation); } - public static Notation get(String notation) { + public static Notation notation(String notation) { var result = notation != null ? notationEntries.get(notation) : null; - if (result != null) return result.notation; + if (result != null) return result; throw new DataException("Data notation is not registered in the NotationLibrary: " + (notation != null ? notation : "null")); } - - public static NotationConverter converter(String notation) { - var result = notation != null ? notationEntries.get(notation) : null; - if (result != null) return result.converter; - throw new DataException("Data type notation not found: " + notation); - } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/binary/BinaryNotation.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/binary/BinaryNotation.java index 32a1d1ce..a36cf9f2 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/binary/BinaryNotation.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/binary/BinaryNotation.java @@ -20,8 +20,10 @@ * =========================LICENSE_END================================== */ +import io.axual.ksml.data.loader.SchemaLoader; import io.axual.ksml.data.mapper.NativeDataObjectMapper; import io.axual.ksml.data.notation.Notation; +import io.axual.ksml.data.notation.NotationConverter; import io.axual.ksml.data.serde.ByteSerde; import io.axual.ksml.data.serde.NullSerde; import io.axual.ksml.data.serde.SerdeSupplier; @@ -35,9 +37,14 @@ import org.apache.kafka.common.serialization.Serializer; public class BinaryNotation implements Notation { - public static final String NOTATION_NAME = "BINARY"; + public static final String NOTATION_NAME = "binary"; + public static final DataType DEFAULT_TYPE = DataType.UNKNOWN; private final NativeDataObjectMapper nativeMapper; private final SerdeSupplier complexTypeSerdeSupplier; + @Getter + private final NotationConverter converter = null; + @Getter + private final SchemaLoader loader = null; public BinaryNotation(NativeDataObjectMapper nativeMapper, SerdeSupplier complexTypeSerdeSupplier) { this.nativeMapper = nativeMapper; @@ -45,8 +52,8 @@ public BinaryNotation(NativeDataObjectMapper nativeMapper, SerdeSupplier complex } @Override - public String name() { - return NOTATION_NAME; + public DataType defaultType() { + return DEFAULT_TYPE; } @Override diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonNotation.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonNotation.java index c181851c..9f7a142d 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonNotation.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonNotation.java @@ -21,24 +21,31 @@ */ import io.axual.ksml.data.exception.ExecutionException; +import io.axual.ksml.data.loader.SchemaLoader; import io.axual.ksml.data.mapper.NativeDataObjectMapper; import io.axual.ksml.data.notation.Notation; +import io.axual.ksml.data.notation.NotationConverter; import io.axual.ksml.data.serde.JsonSerde; import io.axual.ksml.data.type.*; +import lombok.Getter; import org.apache.kafka.common.serialization.Serde; public class JsonNotation implements Notation { - public static final String NOTATION_NAME = "JSON"; public static final DataType DEFAULT_TYPE = new UnionType(new StructType(), new ListType()); private final NativeDataObjectMapper nativeMapper; + @Getter + private final NotationConverter converter = new JsonDataObjectConverter(); + @Getter + private final SchemaLoader loader; - public JsonNotation(NativeDataObjectMapper nativeMapper) { + public JsonNotation(NativeDataObjectMapper nativeMapper, SchemaLoader loader) { this.nativeMapper = nativeMapper; + this.loader = loader; } @Override - public String name() { - return NOTATION_NAME; + public DataType defaultType() { + return DEFAULT_TYPE; } @Override @@ -47,6 +54,6 @@ public Serde serde(DataType type, boolean isKey) { if (type instanceof MapType || type instanceof ListType || JsonNotation.DEFAULT_TYPE.isAssignableFrom(type)) return new JsonSerde(nativeMapper, type); // Other types can not be serialized as JSON - throw new ExecutionException(name() + " serde not found for data type: " + type); + throw new ExecutionException("JSON serde not found for data type: " + type); } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/string/StringNotation.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/string/StringNotation.java index d7bf5a60..7e461142 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/string/StringNotation.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/string/StringNotation.java @@ -42,7 +42,7 @@ public Serde serde(DataType type, boolean isKey) { return new StringSerde(nativeMapper, stringMapper, type); } - protected RuntimeException noSerdeFor(DataType type) { - return new ExecutionException(name() + " serde not found for data type: " + type); + protected RuntimeException noSerdeFor(String notation, DataType type) { + return new ExecutionException(notation + " serde not found for data type: " + type); } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/schema/SchemaLibrary.java b/ksml-data/src/main/java/io/axual/ksml/data/schema/SchemaLibrary.java index bf25d0c9..13053948 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/schema/SchemaLibrary.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/schema/SchemaLibrary.java @@ -21,13 +21,13 @@ */ import io.axual.ksml.data.exception.ExecutionException; +import io.axual.ksml.data.notation.NotationLibrary; import java.util.HashMap; import java.util.Map; import java.util.TreeMap; public class SchemaLibrary { - private static final Map loaders = new TreeMap<>(); private static final Map> schemas = new HashMap<>(); public interface Loader { @@ -60,10 +60,10 @@ public static DataSchema getSchema(String notationName, String schemaName, boole if (schema != null) return schema; } - var loader = loaders.get(notationName); - if (loader == null) return null; + final var notation = NotationLibrary.notation(notationName); + if (notation.loader() == null) return null; - var schema = loader.load(schemaName); + var schema = notation.loader().load(schemaName); if (schema instanceof NamedSchema ns) { if (notationSchemas == null) { notationSchemas = new TreeMap<>(); @@ -77,8 +77,4 @@ public static DataSchema getSchema(String notationName, String schemaName, boole } return schema; } - - public static void registerLoader(String notationName, Loader loader) { - loaders.put(notationName, loader); - } } diff --git a/ksml-kafka-clients/src/main/java/io/axual/ksml/client/util/MapUtil.java b/ksml-kafka-clients/src/main/java/io/axual/ksml/client/util/MapUtil.java index b12de149..3e82f605 100644 --- a/ksml-kafka-clients/src/main/java/io/axual/ksml/client/util/MapUtil.java +++ b/ksml-kafka-clients/src/main/java/io/axual/ksml/client/util/MapUtil.java @@ -29,4 +29,10 @@ public static Map toStringValues(Map map) { map.forEach((key, value) -> result.put(key, value != null ? value.toString() : null)); return result; } + + public static Map merge(Map map1, Map map2) { + final var result = new HashMap<>(map1); + if (map2 != null) result.putAll(map2); + return result; + } } diff --git a/ksml-query/NOTICE.txt b/ksml-query/NOTICE.txt index 1cbed6dc..d60d8b60 100644 --- a/ksml-query/NOTICE.txt +++ b/ksml-query/NOTICE.txt @@ -1,5 +1,5 @@ -Lists of 101 third-party dependencies. +Lists of 136 third-party dependencies. (MIT License) minimal-json (com.eclipsesource.minimal-json:minimal-json:0.9.5 - https://github.com/ralfstx/minimal-json) (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.17.1 - https://github.com/FasterXML/jackson) (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.17.1 - https://github.com/FasterXML/jackson-core) @@ -19,9 +19,18 @@ Lists of 101 third-party dependencies. (The Apache Software License, Version 2.0) Guava ListenableFuture only (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava - https://github.com/google/guava/listenablefuture) (Apache License, Version 2.0) J2ObjC Annotations (com.google.j2objc:j2objc-annotations:3.0.0 - https://github.com/google/j2objc/) (Go License) RE2/J (com.google.re2j:re2j:1.6 - http://github.com/google/re2j) + (MIT License) Microsoft Kiota-Java Abstractions (com.microsoft.kiota:microsoft-kiota-abstractions:1.1.14 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Form (com.microsoft.kiota:microsoft-kiota-serialization-form:1.1.14 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Multipart (com.microsoft.kiota:microsoft-kiota-serialization-multipart:1.1.14 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Text (com.microsoft.kiota:microsoft-kiota-serialization-text:1.1.14 - https://github.com/microsoft/kiota-java) (Eclipse Distribution License - v 1.0) Jakarta SOAP Implementation (com.sun.xml.messaging.saaj:saaj-impl:3.0.4 - https://projects.eclipse.org/projects/ee4j/metro-saaj/saaj-impl) (Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.17.0 - https://commons.apache.org/proper/commons-codec/) (Apache-2.0) Apache Commons IO (commons-io:commons-io:2.16.1 - https://commons.apache.org/proper/commons-io/) + (Apache License Version 2.0) apicurio-registry-common (io.apicurio:apicurio-registry-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-common/) + (Apache License Version 2.0) apicurio-registry-java-sdk (io.apicurio:apicurio-registry-java-sdk:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-java-sdk/) + (Apache License Version 2.0) apicurio-registry-schema-resolver (io.apicurio:apicurio-registry-schema-resolver:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-schema-resolver/) + (Apache License Version 2.0) apicurio-registry-serde-common (io.apicurio:apicurio-registry-serde-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serde-common/) + (Apache License Version 2.0) apicurio-registry-serdes-avro-serde (io.apicurio:apicurio-registry-serdes-avro-serde:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serdes-avro-serde/) (Apache 2.0) KSML (io.axual.ksml:ksml:1.0.3-SNAPSHOT - https://github.com/Axual/ksml/ksml) (Apache 2.0) KSML Data Library (io.axual.ksml:ksml-data:1.0.3-SNAPSHOT - https://github.com/Axual/ksml/ksml-data) (Apache 2.0) KSML Data Library - AVRO (io.axual.ksml:ksml-data-avro:1.0.3-SNAPSHOT - https://github.com/Axual/ksml/ksml-data-avro) @@ -36,7 +45,31 @@ Lists of 101 third-party dependencies. (The Apache Software License, Version 2.0) Log Redactor Metrics (io.confluent:logredactor-metrics:1.0.12 - https://github.com/confluentinc/logredactor) (Apache License 2.0) Metrics Core (io.dropwizard.metrics:metrics-core:4.2.25 - https://metrics.dropwizard.io/metrics-core) (Apache License 2.0) Metrics Integration with JMX (io.dropwizard.metrics:metrics-jmx:4.2.25 - https://metrics.dropwizard.io/metrics-jmx) + (Apache License, Version 2.0) Standard Uri Template (io.github.std-uritemplate:std-uritemplate:0.0.59 - https://std-uritemplate.github.io/) + (Apache-2.0) Kiota - Http - Vert.X (io.kiota:kiota-http-vertx:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-http-vertx) + (Apache-2.0) Kiota - Serialization - Jackson (io.kiota:kiota-serialization-jackson:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-serialization-jackson-parent/kiota-serialization-jackson) + (Apache License, Version 2.0) Netty/Buffer (io.netty:netty-buffer:4.1.108.Final - https://netty.io/netty-buffer/) + (Apache License, Version 2.0) Netty/Codec (io.netty:netty-codec:4.1.108.Final - https://netty.io/netty-codec/) + (Apache License, Version 2.0) Netty/Codec/DNS (io.netty:netty-codec-dns:4.1.108.Final - https://netty.io/netty-codec-dns/) + (Apache License, Version 2.0) Netty/Codec/HTTP (io.netty:netty-codec-http:4.1.108.Final - https://netty.io/netty-codec-http/) + (Apache License, Version 2.0) Netty/Codec/HTTP2 (io.netty:netty-codec-http2:4.1.108.Final - https://netty.io/netty-codec-http2/) + (Apache License, Version 2.0) Netty/Codec/Socks (io.netty:netty-codec-socks:4.1.108.Final - https://netty.io/netty-codec-socks/) + (Apache License, Version 2.0) Netty/Common (io.netty:netty-common:4.1.108.Final - https://netty.io/netty-common/) + (Apache License, Version 2.0) Netty/Handler (io.netty:netty-handler:4.1.108.Final - https://netty.io/netty-handler/) + (Apache License, Version 2.0) Netty/Handler/Proxy (io.netty:netty-handler-proxy:4.1.108.Final - https://netty.io/netty-handler-proxy/) + (Apache License, Version 2.0) Netty/Resolver (io.netty:netty-resolver:4.1.108.Final - https://netty.io/netty-resolver/) + (Apache License, Version 2.0) Netty/Resolver/DNS (io.netty:netty-resolver-dns:4.1.108.Final - https://netty.io/netty-resolver-dns/) + (Apache License, Version 2.0) Netty/Transport (io.netty:netty-transport:4.1.108.Final - https://netty.io/netty-transport/) + (Apache License, Version 2.0) Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.108.Final - https://netty.io/netty-transport-native-unix-common/) + (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-api:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) + (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-context:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) (Apache License 2.0) swagger-annotations (io.swagger.core.v3:swagger-annotations:2.1.10 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-common (io.vertx:vertx-auth-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-common) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-oauth2 (io.vertx:vertx-auth-oauth2:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-oauth2) + (Eclipse Public License - v 2.0) (The Apache Software License, Version 2.0) Vert.x Core (io.vertx:vertx-core:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-core) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) Vert.x URI Template (io.vertx:vertx-uri-template:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-uri-template) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-client (io.vertx:vertx-web-client:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-client) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-common (io.vertx:vertx-web-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-common) (EDL 1.0) Jakarta Activation API (jakarta.activation:jakarta.activation-api:2.1.0 - https://github.com/eclipse-ee4j/jaf) (EPL 2.0) (GPL2 w/ CPE) Jakarta Annotations API (jakarta.annotation:jakarta.annotation-api:2.1.1 - https://projects.eclipse.org/projects/ee4j.ca) (The Apache Software License, Version 2.0) Jakarta Dependency Injection (jakarta.inject:jakarta.inject-api:2.0.1 - https://github.com/eclipse-ee4j/injection-api) @@ -93,6 +126,8 @@ Lists of 101 third-party dependencies. (Universal Permissive License, Version 1.0) Truffle Nfi Libffi (org.graalvm.truffle:truffle-nfi-libffi:23.1.2 - http://openjdk.java.net/projects/graal) (Universal Permissive License, Version 1.0) Truffle Runtime (org.graalvm.truffle:truffle-runtime:23.1.2 - http://openjdk.java.net/projects/graal) (Apache License 2.0) (LGPL 2.1) (MPL 1.1) Javassist (org.javassist:javassist:3.30.2-GA - https://www.javassist.org/) + (Apache License, version 2.0) JBoss Logging 3 (org.jboss.logging:jboss-logging:3.4.1.Final - http://www.jboss.org) + (Apache License 2.0) slf4j to JBoss Logging Adapter (org.jboss.slf4j:slf4j-jboss-logging:1.2.1.Final - http://www.jboss.org) (Eclipse Distribution License - v 1.0) Extended StAX API (org.jvnet.staxex:stax-ex:2.1.0 - https://projects.eclipse.org/projects/ee4j/stax-ex) (The Apache Software License, Version 2.0) LZ4 and xxHash (org.lz4:lz4-java:1.8.0 - https://github.com/lz4/lz4-java) (The MIT License) Project Lombok (org.projectlombok:lombok:1.18.32 - https://projectlombok.org) diff --git a/ksml-reporting/NOTICE.txt b/ksml-reporting/NOTICE.txt index dae56604..766bb63f 100644 --- a/ksml-reporting/NOTICE.txt +++ b/ksml-reporting/NOTICE.txt @@ -1,5 +1,5 @@ -Lists of 123 third-party dependencies. +Lists of 159 third-party dependencies. (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Classic Module (ch.qos.logback:logback-classic:1.5.6 - http://logback.qos.ch/logback-classic) (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Core Module (ch.qos.logback:logback-core:1.5.6 - http://logback.qos.ch/logback-core) (MIT License) minimal-json (com.eclipsesource.minimal-json:minimal-json:0.9.5 - https://github.com/ralfstx/minimal-json) @@ -21,9 +21,18 @@ Lists of 123 third-party dependencies. (The Apache Software License, Version 2.0) Guava ListenableFuture only (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava - https://github.com/google/guava/listenablefuture) (Apache License, Version 2.0) J2ObjC Annotations (com.google.j2objc:j2objc-annotations:3.0.0 - https://github.com/google/j2objc/) (Go License) RE2/J (com.google.re2j:re2j:1.6 - http://github.com/google/re2j) + (MIT License) Microsoft Kiota-Java Abstractions (com.microsoft.kiota:microsoft-kiota-abstractions:1.1.14 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Form (com.microsoft.kiota:microsoft-kiota-serialization-form:1.1.14 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Multipart (com.microsoft.kiota:microsoft-kiota-serialization-multipart:1.1.14 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Text (com.microsoft.kiota:microsoft-kiota-serialization-text:1.1.14 - https://github.com/microsoft/kiota-java) (Eclipse Distribution License - v 1.0) Jakarta SOAP Implementation (com.sun.xml.messaging.saaj:saaj-impl:3.0.4 - https://projects.eclipse.org/projects/ee4j/metro-saaj/saaj-impl) (Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.17.0 - https://commons.apache.org/proper/commons-codec/) (Apache-2.0) Apache Commons IO (commons-io:commons-io:2.16.1 - https://commons.apache.org/proper/commons-io/) + (Apache License Version 2.0) apicurio-registry-common (io.apicurio:apicurio-registry-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-common/) + (Apache License Version 2.0) apicurio-registry-java-sdk (io.apicurio:apicurio-registry-java-sdk:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-java-sdk/) + (Apache License Version 2.0) apicurio-registry-schema-resolver (io.apicurio:apicurio-registry-schema-resolver:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-schema-resolver/) + (Apache License Version 2.0) apicurio-registry-serde-common (io.apicurio:apicurio-registry-serde-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serde-common/) + (Apache License Version 2.0) apicurio-registry-serdes-avro-serde (io.apicurio:apicurio-registry-serdes-avro-serde:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serdes-avro-serde/) (Apache 2.0) KSML (io.axual.ksml:ksml:1.0.3-SNAPSHOT - https://github.com/Axual/ksml/ksml) (Apache 2.0) KSML Data Library (io.axual.ksml:ksml-data:1.0.3-SNAPSHOT - https://github.com/Axual/ksml/ksml-data) (Apache 2.0) KSML Data Library - AVRO (io.axual.ksml:ksml-data-avro:1.0.3-SNAPSHOT - https://github.com/Axual/ksml/ksml-data-avro) @@ -41,6 +50,24 @@ Lists of 123 third-party dependencies. (The Apache Software License, Version 2.0) Log Redactor Metrics (io.confluent:logredactor-metrics:1.0.12 - https://github.com/confluentinc/logredactor) (Apache License 2.0) Metrics Core (io.dropwizard.metrics:metrics-core:4.2.25 - https://metrics.dropwizard.io/metrics-core) (Apache License 2.0) Metrics Integration with JMX (io.dropwizard.metrics:metrics-jmx:4.2.25 - https://metrics.dropwizard.io/metrics-jmx) + (Apache License, Version 2.0) Standard Uri Template (io.github.std-uritemplate:std-uritemplate:0.0.59 - https://std-uritemplate.github.io/) + (Apache-2.0) Kiota - Http - Vert.X (io.kiota:kiota-http-vertx:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-http-vertx) + (Apache-2.0) Kiota - Serialization - Jackson (io.kiota:kiota-serialization-jackson:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-serialization-jackson-parent/kiota-serialization-jackson) + (Apache License, Version 2.0) Netty/Buffer (io.netty:netty-buffer:4.1.108.Final - https://netty.io/netty-buffer/) + (Apache License, Version 2.0) Netty/Codec (io.netty:netty-codec:4.1.108.Final - https://netty.io/netty-codec/) + (Apache License, Version 2.0) Netty/Codec/DNS (io.netty:netty-codec-dns:4.1.108.Final - https://netty.io/netty-codec-dns/) + (Apache License, Version 2.0) Netty/Codec/HTTP (io.netty:netty-codec-http:4.1.108.Final - https://netty.io/netty-codec-http/) + (Apache License, Version 2.0) Netty/Codec/HTTP2 (io.netty:netty-codec-http2:4.1.108.Final - https://netty.io/netty-codec-http2/) + (Apache License, Version 2.0) Netty/Codec/Socks (io.netty:netty-codec-socks:4.1.108.Final - https://netty.io/netty-codec-socks/) + (Apache License, Version 2.0) Netty/Common (io.netty:netty-common:4.1.108.Final - https://netty.io/netty-common/) + (Apache License, Version 2.0) Netty/Handler (io.netty:netty-handler:4.1.108.Final - https://netty.io/netty-handler/) + (Apache License, Version 2.0) Netty/Handler/Proxy (io.netty:netty-handler-proxy:4.1.108.Final - https://netty.io/netty-handler-proxy/) + (Apache License, Version 2.0) Netty/Resolver (io.netty:netty-resolver:4.1.108.Final - https://netty.io/netty-resolver/) + (Apache License, Version 2.0) Netty/Resolver/DNS (io.netty:netty-resolver-dns:4.1.108.Final - https://netty.io/netty-resolver-dns/) + (Apache License, Version 2.0) Netty/Transport (io.netty:netty-transport:4.1.108.Final - https://netty.io/netty-transport/) + (Apache License, Version 2.0) Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.108.Final - https://netty.io/netty-transport-native-unix-common/) + (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-api:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) + (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-context:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) (The Apache Software License, Version 2.0) Prometheus Metrics Config (io.prometheus:prometheus-metrics-config:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-config) (The Apache Software License, Version 2.0) Prometheus Metrics Core (io.prometheus:prometheus-metrics-core:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-core) (The Apache Software License, Version 2.0) Prometheus Metrics Exporter - Common (io.prometheus:prometheus-metrics-exporter-common:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-exporter-common) @@ -56,6 +83,12 @@ Lists of 123 third-party dependencies. (The Apache Software License, Version 2.0) Prometheus JMX Exporter - Collector (io.prometheus.jmx:collector:1.0.1 - http://github.com/prometheus/jmx_exporter) (The Apache Software License, Version 2.0) Prometheus JMX Exporter - Common (io.prometheus.jmx:jmx_prometheus_common:1.0.1 - http://github.com/prometheus/jmx_exporter) (Apache License 2.0) swagger-annotations (io.swagger.core.v3:swagger-annotations:2.1.10 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-common (io.vertx:vertx-auth-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-common) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-oauth2 (io.vertx:vertx-auth-oauth2:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-oauth2) + (Eclipse Public License - v 2.0) (The Apache Software License, Version 2.0) Vert.x Core (io.vertx:vertx-core:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-core) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) Vert.x URI Template (io.vertx:vertx-uri-template:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-uri-template) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-client (io.vertx:vertx-web-client:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-client) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-common (io.vertx:vertx-web-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-common) (EDL 1.0) Jakarta Activation API (jakarta.activation:jakarta.activation-api:2.1.3 - https://github.com/jakartaee/jaf-api) (EPL 2.0) (GPL2 w/ CPE) Jakarta Annotations API (jakarta.annotation:jakarta.annotation-api:2.1.1 - https://projects.eclipse.org/projects/ee4j.ca) (The Apache Software License, Version 2.0) Jakarta Dependency Injection (jakarta.inject:jakarta.inject-api:2.0.1 - https://github.com/eclipse-ee4j/injection-api) @@ -116,8 +149,11 @@ Lists of 123 third-party dependencies. (Universal Permissive License, Version 1.0) Truffle Nfi Libffi (org.graalvm.truffle:truffle-nfi-libffi:23.1.2 - http://openjdk.java.net/projects/graal) (Universal Permissive License, Version 1.0) Truffle Runtime (org.graalvm.truffle:truffle-runtime:23.1.2 - http://openjdk.java.net/projects/graal) (Apache License 2.0) (LGPL 2.1) (MPL 1.1) Javassist (org.javassist:javassist:3.30.2-GA - https://www.javassist.org/) + (Apache License, version 2.0) JBoss Logging 3 (org.jboss.logging:jboss-logging:3.4.1.Final - http://www.jboss.org) + (Apache License 2.0) slf4j to JBoss Logging Adapter (org.jboss.slf4j:slf4j-jboss-logging:1.2.1.Final - http://www.jboss.org) (Eclipse Distribution License - v 1.0) Extended StAX API (org.jvnet.staxex:stax-ex:2.1.0 - https://projects.eclipse.org/projects/ee4j/stax-ex) (The Apache Software License, Version 2.0) LZ4 and xxHash (org.lz4:lz4-java:1.8.0 - https://github.com/lz4/lz4-java) + (The MIT License) Project Lombok (org.projectlombok:lombok:1.18.32 - https://projectlombok.org) (Apache License 2.0) (GNU General Public License, version 2) RocksDB JNI (org.rocksdb:rocksdbjni:7.9.2 - https://rocksdb.org) (MIT License) SLF4J API Module (org.slf4j:slf4j-api:2.0.13 - http://www.slf4j.org) (Public Domain) XZ for Java (org.tukaani:xz:1.9 - https://tukaani.org/xz/java.html) diff --git a/ksml-runner/NOTICE.txt b/ksml-runner/NOTICE.txt index f8ba9684..8eeb4bd0 100644 --- a/ksml-runner/NOTICE.txt +++ b/ksml-runner/NOTICE.txt @@ -1,5 +1,5 @@ -Lists of 123 third-party dependencies. +Lists of 158 third-party dependencies. (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Classic Module (ch.qos.logback:logback-classic:1.5.6 - http://logback.qos.ch/logback-classic) (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Core Module (ch.qos.logback:logback-core:1.5.6 - http://logback.qos.ch/logback-core) (MIT License) minimal-json (com.eclipsesource.minimal-json:minimal-json:0.9.5 - https://github.com/ralfstx/minimal-json) @@ -21,9 +21,18 @@ Lists of 123 third-party dependencies. (The Apache Software License, Version 2.0) Guava ListenableFuture only (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava - https://github.com/google/guava/listenablefuture) (Apache License, Version 2.0) J2ObjC Annotations (com.google.j2objc:j2objc-annotations:3.0.0 - https://github.com/google/j2objc/) (Go License) RE2/J (com.google.re2j:re2j:1.6 - http://github.com/google/re2j) + (MIT License) Microsoft Kiota-Java Abstractions (com.microsoft.kiota:microsoft-kiota-abstractions:1.1.14 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Form (com.microsoft.kiota:microsoft-kiota-serialization-form:1.1.14 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Multipart (com.microsoft.kiota:microsoft-kiota-serialization-multipart:1.1.14 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Text (com.microsoft.kiota:microsoft-kiota-serialization-text:1.1.14 - https://github.com/microsoft/kiota-java) (Eclipse Distribution License - v 1.0) Jakarta SOAP Implementation (com.sun.xml.messaging.saaj:saaj-impl:3.0.4 - https://projects.eclipse.org/projects/ee4j/metro-saaj/saaj-impl) (Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.17.0 - https://commons.apache.org/proper/commons-codec/) (Apache-2.0) Apache Commons IO (commons-io:commons-io:2.16.1 - https://commons.apache.org/proper/commons-io/) + (Apache License Version 2.0) apicurio-registry-common (io.apicurio:apicurio-registry-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-common/) + (Apache License Version 2.0) apicurio-registry-java-sdk (io.apicurio:apicurio-registry-java-sdk:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-java-sdk/) + (Apache License Version 2.0) apicurio-registry-schema-resolver (io.apicurio:apicurio-registry-schema-resolver:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-schema-resolver/) + (Apache License Version 2.0) apicurio-registry-serde-common (io.apicurio:apicurio-registry-serde-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serde-common/) + (Apache License Version 2.0) apicurio-registry-serdes-avro-serde (io.apicurio:apicurio-registry-serdes-avro-serde:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serdes-avro-serde/) (Apache 2.0) KSML (io.axual.ksml:ksml:1.0.3-SNAPSHOT - https://github.com/Axual/ksml/ksml) (Apache 2.0) KSML Data Library (io.axual.ksml:ksml-data:1.0.3-SNAPSHOT - https://github.com/Axual/ksml/ksml-data) (Apache 2.0) KSML Data Library - AVRO (io.axual.ksml:ksml-data-avro:1.0.3-SNAPSHOT - https://github.com/Axual/ksml/ksml-data-avro) @@ -40,6 +49,24 @@ Lists of 123 third-party dependencies. (The Apache Software License, Version 2.0) Log Redactor Metrics (io.confluent:logredactor-metrics:1.0.12 - https://github.com/confluentinc/logredactor) (Apache License 2.0) Metrics Core (io.dropwizard.metrics:metrics-core:4.2.25 - https://metrics.dropwizard.io/metrics-core) (Apache License 2.0) Metrics Integration with JMX (io.dropwizard.metrics:metrics-jmx:4.2.25 - https://metrics.dropwizard.io/metrics-jmx) + (Apache License, Version 2.0) Standard Uri Template (io.github.std-uritemplate:std-uritemplate:0.0.59 - https://std-uritemplate.github.io/) + (Apache-2.0) Kiota - Http - Vert.X (io.kiota:kiota-http-vertx:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-http-vertx) + (Apache-2.0) Kiota - Serialization - Jackson (io.kiota:kiota-serialization-jackson:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-serialization-jackson-parent/kiota-serialization-jackson) + (Apache License, Version 2.0) Netty/Buffer (io.netty:netty-buffer:4.1.108.Final - https://netty.io/netty-buffer/) + (Apache License, Version 2.0) Netty/Codec (io.netty:netty-codec:4.1.108.Final - https://netty.io/netty-codec/) + (Apache License, Version 2.0) Netty/Codec/DNS (io.netty:netty-codec-dns:4.1.108.Final - https://netty.io/netty-codec-dns/) + (Apache License, Version 2.0) Netty/Codec/HTTP (io.netty:netty-codec-http:4.1.108.Final - https://netty.io/netty-codec-http/) + (Apache License, Version 2.0) Netty/Codec/HTTP2 (io.netty:netty-codec-http2:4.1.108.Final - https://netty.io/netty-codec-http2/) + (Apache License, Version 2.0) Netty/Codec/Socks (io.netty:netty-codec-socks:4.1.108.Final - https://netty.io/netty-codec-socks/) + (Apache License, Version 2.0) Netty/Common (io.netty:netty-common:4.1.108.Final - https://netty.io/netty-common/) + (Apache License, Version 2.0) Netty/Handler (io.netty:netty-handler:4.1.108.Final - https://netty.io/netty-handler/) + (Apache License, Version 2.0) Netty/Handler/Proxy (io.netty:netty-handler-proxy:4.1.108.Final - https://netty.io/netty-handler-proxy/) + (Apache License, Version 2.0) Netty/Resolver (io.netty:netty-resolver:4.1.108.Final - https://netty.io/netty-resolver/) + (Apache License, Version 2.0) Netty/Resolver/DNS (io.netty:netty-resolver-dns:4.1.108.Final - https://netty.io/netty-resolver-dns/) + (Apache License, Version 2.0) Netty/Transport (io.netty:netty-transport:4.1.108.Final - https://netty.io/netty-transport/) + (Apache License, Version 2.0) Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.108.Final - https://netty.io/netty-transport-native-unix-common/) + (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-api:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) + (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-context:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) (The Apache Software License, Version 2.0) Prometheus Metrics Config (io.prometheus:prometheus-metrics-config:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-config) (The Apache Software License, Version 2.0) Prometheus Metrics Core (io.prometheus:prometheus-metrics-core:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-core) (The Apache Software License, Version 2.0) Prometheus Metrics Exporter - Common (io.prometheus:prometheus-metrics-exporter-common:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-exporter-common) @@ -55,6 +82,12 @@ Lists of 123 third-party dependencies. (The Apache Software License, Version 2.0) Prometheus JMX Exporter - Collector (io.prometheus.jmx:collector:1.0.1 - http://github.com/prometheus/jmx_exporter) (The Apache Software License, Version 2.0) Prometheus JMX Exporter - Common (io.prometheus.jmx:jmx_prometheus_common:1.0.1 - http://github.com/prometheus/jmx_exporter) (Apache License 2.0) swagger-annotations (io.swagger.core.v3:swagger-annotations:2.1.10 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-common (io.vertx:vertx-auth-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-common) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-oauth2 (io.vertx:vertx-auth-oauth2:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-oauth2) + (Eclipse Public License - v 2.0) (The Apache Software License, Version 2.0) Vert.x Core (io.vertx:vertx-core:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-core) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) Vert.x URI Template (io.vertx:vertx-uri-template:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-uri-template) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-client (io.vertx:vertx-web-client:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-client) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-common (io.vertx:vertx-web-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-common) (EDL 1.0) Jakarta Activation API (jakarta.activation:jakarta.activation-api:2.1.3 - https://github.com/jakartaee/jaf-api) (EPL 2.0) (GPL2 w/ CPE) Jakarta Annotations API (jakarta.annotation:jakarta.annotation-api:2.1.1 - https://projects.eclipse.org/projects/ee4j.ca) (The Apache Software License, Version 2.0) Jakarta Dependency Injection (jakarta.inject:jakarta.inject-api:2.0.1 - https://github.com/eclipse-ee4j/injection-api) @@ -115,6 +148,8 @@ Lists of 123 third-party dependencies. (Universal Permissive License, Version 1.0) Truffle Nfi Libffi (org.graalvm.truffle:truffle-nfi-libffi:23.1.2 - http://openjdk.java.net/projects/graal) (Universal Permissive License, Version 1.0) Truffle Runtime (org.graalvm.truffle:truffle-runtime:23.1.2 - http://openjdk.java.net/projects/graal) (Apache License 2.0) (LGPL 2.1) (MPL 1.1) Javassist (org.javassist:javassist:3.30.2-GA - https://www.javassist.org/) + (Apache License, version 2.0) JBoss Logging 3 (org.jboss.logging:jboss-logging:3.4.1.Final - http://www.jboss.org) + (Apache License 2.0) slf4j to JBoss Logging Adapter (org.jboss.slf4j:slf4j-jboss-logging:1.2.1.Final - http://www.jboss.org) (Eclipse Distribution License - v 1.0) Extended StAX API (org.jvnet.staxex:stax-ex:2.1.0 - https://projects.eclipse.org/projects/ee4j/stax-ex) (The Apache Software License, Version 2.0) LZ4 and xxHash (org.lz4:lz4-java:1.8.0 - https://github.com/lz4/lz4-java) (The MIT License) Project Lombok (org.projectlombok:lombok:1.18.32 - https://projectlombok.org) diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java index ed18c7ff..cf530d45 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java @@ -23,49 +23,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; - -import org.apache.kafka.common.serialization.Serdes; -import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.common.utils.Utils; -import org.apache.kafka.streams.KeyQueryMetadata; -import org.apache.kafka.streams.StoreQueryParameters; -import org.apache.kafka.streams.StreamsMetadata; -import org.apache.kafka.streams.state.HostInfo; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - import io.axual.ksml.client.serde.ResolvingDeserializer; import io.axual.ksml.client.serde.ResolvingSerializer; -import io.axual.ksml.data.mapper.DataObjectFlattener; import io.axual.ksml.data.notation.NotationLibrary; -import io.axual.ksml.data.notation.avro.AvroNotation; -import io.axual.ksml.data.notation.avro.AvroSchemaLoader; -import io.axual.ksml.data.notation.binary.BinaryNotation; -import io.axual.ksml.data.notation.csv.CsvDataObjectConverter; -import io.axual.ksml.data.notation.csv.CsvNotation; -import io.axual.ksml.data.notation.csv.CsvSchemaLoader; -import io.axual.ksml.data.notation.json.JsonDataObjectConverter; -import io.axual.ksml.data.notation.json.JsonNotation; -import io.axual.ksml.data.notation.json.JsonSchemaLoader; import io.axual.ksml.data.notation.json.JsonSchemaMapper; -import io.axual.ksml.data.notation.soap.SOAPDataObjectConverter; -import io.axual.ksml.data.notation.soap.SOAPNotation; -import io.axual.ksml.data.notation.xml.XmlDataObjectConverter; -import io.axual.ksml.data.notation.xml.XmlNotation; -import io.axual.ksml.data.notation.xml.XmlSchemaLoader; import io.axual.ksml.data.parser.ParseNode; -import io.axual.ksml.data.schema.SchemaLibrary; import io.axual.ksml.definition.parser.TopologyDefinitionParser; import io.axual.ksml.execution.ErrorHandler; import io.axual.ksml.execution.ExecutionContext; @@ -80,8 +42,25 @@ import io.axual.ksml.runner.config.KSMLErrorHandlingConfig; import io.axual.ksml.runner.config.KSMLRunnerConfig; import io.axual.ksml.runner.exception.ConfigException; +import io.axual.ksml.runner.notation.NotationFactories; import io.axual.ksml.runner.prometheus.PrometheusExport; import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.streams.KeyQueryMetadata; +import org.apache.kafka.streams.StoreQueryParameters; +import org.apache.kafka.streams.StreamsMetadata; +import org.apache.kafka.streams.state.HostInfo; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; @Slf4j public class KSMLRunner { @@ -106,7 +85,7 @@ public static void main(String[] args) { } final var config = readConfiguration(configFile); - final var ksmlConfig = config.getKsmlConfig(); + final var ksmlConfig = config.ksmlConfig(); log.info("Using directories: config: {}, schema: {}, storage: {}", ksmlConfig.getConfigDirectory(), ksmlConfig.getSchemaDirectory(), ksmlConfig.getStorageDirectory()); final var definitions = ksmlConfig.getDefinitions(); if (definitions == null || definitions.isEmpty()) { @@ -119,28 +98,39 @@ public static void main(String[] args) { final var appServer = ksmlConfig.getApplicationServerConfig(); RestServer restServer = null; // Start rest server to provide service endpoints - if (appServer.isEnabled()) { + if (appServer.enabled()) { HostInfo hostInfo = new HostInfo(appServer.getHost(), appServer.getPort()); restServer = new RestServer(hostInfo); restServer.start(); } - // Set up the notation library with all known notations and type override classes - final var nativeMapper = new DataObjectFlattener(true); - final var jsonNotation = new JsonNotation(nativeMapper); - NotationLibrary.register(AvroNotation.NOTATION_NAME, new AvroNotation(nativeMapper, config.getKafkaConfig()), null); - NotationLibrary.register(BinaryNotation.NOTATION_NAME, new BinaryNotation(nativeMapper, jsonNotation::serde), null); - NotationLibrary.register(CsvNotation.NOTATION_NAME, new CsvNotation(nativeMapper), new CsvDataObjectConverter()); - NotationLibrary.register(JsonNotation.NOTATION_NAME, jsonNotation, new JsonDataObjectConverter()); - NotationLibrary.register(SOAPNotation.NOTATION_NAME, new SOAPNotation(nativeMapper), new SOAPDataObjectConverter()); - NotationLibrary.register(XmlNotation.NOTATION_NAME, new XmlNotation(nativeMapper), new XmlDataObjectConverter()); - - // Register schema loaders - final var schemaDirectory = ksmlConfig.getSchemaDirectory(); - SchemaLibrary.registerLoader(AvroNotation.NOTATION_NAME, new AvroSchemaLoader(schemaDirectory)); - SchemaLibrary.registerLoader(CsvNotation.NOTATION_NAME, new CsvSchemaLoader(schemaDirectory)); - SchemaLibrary.registerLoader(JsonNotation.NOTATION_NAME, new JsonSchemaLoader(schemaDirectory)); - SchemaLibrary.registerLoader(XmlNotation.NOTATION_NAME, new XmlSchemaLoader(schemaDirectory)); + // Set up all default notations and register them in the NotationLibrary + final var notationFactories = new NotationFactories(config.getKafkaConfig(), ksmlConfig.getSchemaDirectory()); + for (final var notation : notationFactories.notations().entrySet()) { + NotationLibrary.register(notation.getKey(), notation.getValue().create(null)); + } + + // Set up all notation overrides from the KSML config + for (final var notationEntry : ksmlConfig.notations().entrySet()) { + final var notation = notationEntry.getKey(); + final var notationConfig = notationEntry.getValue(); + if (notation != null && notationConfig != null) { + final var n = notationFactories.notations().get(notationConfig.type()); + if (n == null) { + throw FatalError.reportAndExit(new ConfigException("Notation type '" + notationConfig.type() + "' not found")); + } + NotationLibrary.register(notation, n.create(notationConfig.config())); + } + } + + // Ensure typical defaults are used + // WARNING: Defaults for notations will be deprecated in the future. Make sure you explicitly define + // notations that have multiple implementations in your ksml-runner.yaml. + if (!NotationLibrary.exists(NotationFactories.AVRO)) { + final var defaultAvro = notationFactories.confluentAvro(); + NotationLibrary.register(NotationFactories.AVRO, defaultAvro.create(null)); + log.warn("No implementation specified for AVRO notation. If you plan to use AVRO, add the required implementation to the ksml-runner.yaml"); + } final var errorHandling = ksmlConfig.getErrorHandlingConfig(); if (errorHandling != null) { @@ -163,11 +153,11 @@ public static void main(String[] args) { }); - if (!ksmlConfig.isEnableProducers() && !producerSpecs.isEmpty()) { + if (!ksmlConfig.enableProducers() && !producerSpecs.isEmpty()) { log.warn("Producers are disabled for this runner. The supplied producer specifications will be ignored."); producerSpecs.clear(); } - if (!ksmlConfig.isEnablePipelines() && !pipelineSpecs.isEmpty()) { + if (!ksmlConfig.enablePipelines() && !pipelineSpecs.isEmpty()) { log.warn("Pipelines are disabled for this runner. The supplied pipeline specifications will be ignored."); pipelineSpecs.clear(); } @@ -196,7 +186,7 @@ public static void main(String[] args) { Runtime.getRuntime().addShutdownHook(shutdownHook); - try (var prometheusExport = new PrometheusExport(config.getKsmlConfig().getPrometheusConfig())) { + try (var prometheusExport = new PrometheusExport(config.ksmlConfig().prometheusConfig())) { prometheusExport.start(); final var executorService = Executors.newFixedThreadPool(2); final var producerFuture = producer == null ? CompletableFuture.completedFuture(null) : executorService.submit(producer); @@ -267,7 +257,7 @@ public Collection allMetadataForStore(String storeName) { if (streamsRunner == null) { return List.of(); } - return streamsRunner.getKafkaStreams().streamsMetadataForStore(storeName); + return streamsRunner.kafkaStreams().streamsMetadataForStore(storeName); } @Override @@ -275,7 +265,7 @@ public KeyQueryMetadata queryMetadataForKey(String storeName, K key, Seriali if (streamsRunner == null) { return null; } - return streamsRunner.getKafkaStreams().queryMetadataForKey(storeName, key, keySerializer); + return streamsRunner.kafkaStreams().queryMetadataForKey(storeName, key, keySerializer); } @Override @@ -283,7 +273,7 @@ public T store(StoreQueryParameters storeQueryParameters) { if (streamsRunner == null) { return null; } - return streamsRunner.getKafkaStreams().store(storeQueryParameters); + return streamsRunner.kafkaStreams().store(storeQueryParameters); } @Override @@ -366,7 +356,7 @@ private static KSMLRunnerConfig readConfiguration(File configFile) { try { final var config = mapper.readValue(configFile, KSMLRunnerConfig.class); if (config != null) { - if (config.getKsmlConfig() == null) { + if (config.ksmlConfig() == null) { throw new ConfigException("Section \"ksml\" is missing in configuration"); } if (config.getKafkaConfig() == null) { @@ -382,14 +372,14 @@ private static KSMLRunnerConfig readConfiguration(File configFile) { private static ErrorHandler getErrorHandler(KSMLErrorHandlingConfig.ErrorHandlingConfig config) { - final var handlerType = switch (config.getHandler()) { + final var handlerType = switch (config.handler()) { case CONTINUE -> ErrorHandler.HandlerType.CONTINUE_ON_FAIL; case STOP -> ErrorHandler.HandlerType.STOP_ON_FAIL; }; return new ErrorHandler( - config.isLog(), - config.isLogPayload(), - config.getLoggerName(), + config.log(), + config.logPayload(), + config.loggerName(), handlerType); } } diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/KsmlInfo.java b/ksml-runner/src/main/java/io/axual/ksml/runner/KsmlInfo.java index cd573607..13beeab0 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/KsmlInfo.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/KsmlInfo.java @@ -31,7 +31,6 @@ @Slf4j public class KsmlInfo { - private static final String DEFAULT_APP_NAME = "KSML"; private static final String DEFAULT_APP_VERSION = ""; private static final String DEFAULT_BUILD_TIME = ""; @@ -42,7 +41,6 @@ public class KsmlInfo { public static final KsmlInfoMBean BEAN_CONTENT = new KsmlInfoMBean(); static { - String appName, appVersion, buildTime; try { ClassLoader cl = KSMLRunner.class.getClassLoader(); @@ -89,7 +87,6 @@ public interface IKsmlInfoMXBean { } public static class KsmlInfoMBean implements IKsmlInfoMXBean { - @Override public int getValue() { return 1; diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/ExecutableProducer.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/ExecutableProducer.java index 8ff4da8b..9bcd056d 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/ExecutableProducer.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/ExecutableProducer.java @@ -110,9 +110,9 @@ public static ExecutableProducer forProducer(PythonContext context, String names ? PythonFunction.forPredicate(context, namespace, cond.name(), cond) : PythonFunction.forPredicate(context, namespace, name, cond) : null; - final var keySerde = NotationLibrary.get(target.keyType().notation()).serde(target.keyType().dataType(), true); + final var keySerde = NotationLibrary.notation(target.keyType().notation()).serde(target.keyType().dataType(), true); final var keySerializer = new ResolvingSerializer<>(keySerde.serializer(), kafkaConfig); - final var valueSerde = NotationLibrary.get(target.valueType().notation()).serde(target.valueType().dataType(), false); + final var valueSerde = NotationLibrary.notation(target.valueType().notation()).serde(target.valueType().dataType(), false); final var valueSerializer = new ResolvingSerializer<>(valueSerde.serializer(), kafkaConfig); final var reschedulingStrategy = setupRescheduling(producerDefinition, context, namespace, name); diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaStreamsRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaStreamsRunner.java index 15d9badb..ce5f0399 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaStreamsRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaStreamsRunner.java @@ -98,7 +98,7 @@ private Map getStreamsConfig(Map initialConfigs, } result.put(StreamsConfig.STATE_DIR_CONFIG, storageDirectory); - if (appServer != null && appServer.isEnabled()) { + if (appServer != null && appServer.enabled()) { result.put(StreamsConfig.APPLICATION_SERVER_CONFIG, appServer.getApplicationServer()); } diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java index 736dae2a..cb29ea08 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; import io.axual.ksml.data.notation.binary.JsonNodeNativeMapper; import io.axual.ksml.generator.YAMLObjectMapper; import io.axual.ksml.runner.exception.ConfigException; @@ -69,6 +70,8 @@ public class KSMLConfig { @JsonProperty("errorHandling") private KSMLErrorHandlingConfig errorHandling; + @JsonProperty("notations") + private Map notations; @JsonProperty("definitions") private Map definitions; @JsonProperty("schemas") @@ -103,6 +106,11 @@ public KSMLErrorHandlingConfig getErrorHandlingConfig() { return errorHandling; } + public Map notations() { + if (notations != null) return ImmutableMap.copyOf(notations); + return ImmutableMap.of(); + } + public Map getDefinitions() { final var result = new HashMap(); if (definitions != null) { diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java index cacdad28..79ab9f01 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java @@ -40,8 +40,8 @@ public ErrorHandlingConfig getConsumerErrorHandlingConfig() { if (consume == null) { return getDefaultErrorHandlingConfig("ConsumeError"); } - if (consume.getLoggerName() == null) { - consume.setLoggerName("ConsumeError"); + if (consume.loggerName() == null) { + consume.loggerName("ConsumeError"); } return consume; } @@ -50,8 +50,8 @@ public ErrorHandlingConfig getProducerErrorHandlingConfig() { if (produce == null) { return getDefaultErrorHandlingConfig("ProduceError"); } - if (produce.getLoggerName() == null) { - produce.setLoggerName("ProduceError"); + if (produce.loggerName() == null) { + produce.loggerName("ProduceError"); } return produce; } @@ -60,15 +60,15 @@ public ErrorHandlingConfig getProcessErrorHandlingConfig() { if (process == null) { return getDefaultErrorHandlingConfig("ProcessError"); } - if (process.getLoggerName() == null) { - process.setLoggerName("ProcessError"); + if (process.loggerName() == null) { + process.loggerName("ProcessError"); } return process; } private ErrorHandlingConfig getDefaultErrorHandlingConfig(String logger) { var errorHandlingConfig = new ErrorHandlingConfig(); - errorHandlingConfig.setLoggerName(logger); + errorHandlingConfig.loggerName(logger); return errorHandlingConfig; } diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java index 0a8a5cee..3492fabb 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java @@ -44,26 +44,26 @@ public class KSMLRunnerConfig { private KSMLConfig ksmlConfig; @JsonProperty("kafka") - private KafkaConfig kafka; + private KafkaConfig kafkaConfig; - public Map getKafkaConfig(){ - var newConfig = new HashMap<>(kafka.kafkaConfig()); - newConfig.put("application.id", kafka.getApplicationId()); + public Map getKafkaConfig() { + var newConfig = new HashMap<>(kafkaConfig.kafkaConfig()); + newConfig.put("application.id", kafkaConfig.applicationId()); return newConfig; } - public String getApplicationId(){ - return kafka.getApplicationId(); + public String getApplicationId() { + return kafkaConfig.applicationId(); } @Data - public static class KafkaConfig{ + public static class KafkaConfig { @JsonProperty("app.id") - @JsonAlias({"applicationId","application.id"}) + @JsonAlias({"applicationId", "application.id"}) public String applicationId; @JsonIgnore - private Map kafkaConfig = new HashMap<>(); + private Map kafkaConfig = new HashMap<>(); // Capture all other fields that Jackson do not match other members @JsonAnyGetter diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/NotationConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/NotationConfig.java new file mode 100644 index 00000000..10ac3516 --- /dev/null +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/NotationConfig.java @@ -0,0 +1,40 @@ +package io.axual.ksml.runner.config; + +/*- + * ========================LICENSE_START================================= + * KSML Runner + * %% + * Copyright (C) 2021 - 2023 Axual B.V. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * =========================LICENSE_END================================== + */ + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Builder; +import lombok.Data; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.jackson.Jacksonized; + +import java.util.Map; + +@JsonIgnoreProperties(ignoreUnknown = true) +@Builder +@Jacksonized +@Data +public class NotationConfig { + private String type; + private Map config; +} diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config b/ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config new file mode 100644 index 00000000..881a8aa5 --- /dev/null +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config @@ -0,0 +1,3 @@ +lombok.accessors.fluent=true +lombok.accessors.chain=false +lombok.equalsAndHashCode.callSuper=call \ No newline at end of file diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/notation/NotationFactories.java b/ksml-runner/src/main/java/io/axual/ksml/runner/notation/NotationFactories.java new file mode 100644 index 00000000..50bab6f5 --- /dev/null +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/notation/NotationFactories.java @@ -0,0 +1,97 @@ +package io.axual.ksml.runner.notation; + +/*- + * ========================LICENSE_START================================= + * KSML Runner + * %% + * Copyright (C) 2021 - 2024 Axual B.V. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * =========================LICENSE_END================================== + */ + +import io.axual.ksml.client.util.MapUtil; +import io.axual.ksml.data.mapper.DataObjectFlattener; +import io.axual.ksml.data.notation.Notation; +import io.axual.ksml.data.notation.avro.AvroNotation; +import io.axual.ksml.data.notation.avro.AvroSchemaLoader; +import io.axual.ksml.data.notation.binary.BinaryNotation; +import io.axual.ksml.data.notation.csv.CsvNotation; +import io.axual.ksml.data.notation.csv.CsvSchemaLoader; +import io.axual.ksml.data.notation.json.JsonNotation; +import io.axual.ksml.data.notation.json.JsonSchemaLoader; +import io.axual.ksml.data.notation.soap.SOAPNotation; +import io.axual.ksml.data.notation.xml.XmlNotation; +import io.axual.ksml.data.notation.xml.XmlSchemaLoader; +import lombok.Getter; + +import java.util.HashMap; +import java.util.Map; + +@Getter +public class NotationFactories { + public interface NotationFactory { + Notation create(Map notationConfig); + } + + // Notation constants + public static final String AVRO = "avro"; + public static final String APICURIO_AVRO = "apicurioAvro"; + public static final String CONFLUENT_AVRO = "confluentAvro"; + public static final String BINARY = "binary"; + public static final String CSV = "csv"; + public static final String JSON = "json"; + public static final String SOAP = "soap"; + public static final String XML = "xml"; + + // Notation variables + private final NotationFactory apicurioAvro; + private final NotationFactory confluentAvro; + private final NotationFactory binary; + private final NotationFactory csv; + private final NotationFactory json; + private final NotationFactory soap; + private final NotationFactory xml; + + private final Map notations = new HashMap<>(); + + public NotationFactories(Map kafkaConfig, String schemaDirectory) { + // AVRO + final var avroLoader = new AvroSchemaLoader(schemaDirectory); + final var nativeMapper = new DataObjectFlattener(true); + apicurioAvro = (notationConfig) -> new AvroNotation(AvroNotation.SerdeType.APICURIO, nativeMapper, MapUtil.merge(kafkaConfig, notationConfig), avroLoader); + notations.put(APICURIO_AVRO, apicurioAvro); + confluentAvro = (notationConfig) -> new AvroNotation(AvroNotation.SerdeType.CONFLUENT, nativeMapper, MapUtil.merge(kafkaConfig, notationConfig), avroLoader); + notations.put(CONFLUENT_AVRO, confluentAvro); + + // CSV + csv = (config) -> new CsvNotation(nativeMapper, new CsvSchemaLoader(schemaDirectory)); + notations.put(CSV, csv); + + // JSON + json = (config) -> new JsonNotation(nativeMapper, new JsonSchemaLoader(schemaDirectory)); + notations.put(JSON, json); + + // SOAP + soap = (config) -> new SOAPNotation(nativeMapper); + notations.put(SOAP, soap); + + // XML + xml = (config) -> new XmlNotation(nativeMapper, new XmlSchemaLoader(schemaDirectory)); + notations.put(XML, xml); + + // Binary + binary = (config) -> new BinaryNotation(nativeMapper, json.create(null)::serde); + notations.put(BINARY, binary); + } +} diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/prometheus/PrometheusExport.java b/ksml-runner/src/main/java/io/axual/ksml/runner/prometheus/PrometheusExport.java index 8963d569..ff09c629 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/prometheus/PrometheusExport.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/prometheus/PrometheusExport.java @@ -54,7 +54,7 @@ public PrometheusExport(PrometheusConfig config) { @Synchronized public void start() throws Exception { Metrics.init(); - if (!config.isEnabled()) { + if (!config.enabled()) { log.info("Prometheus export is disabled"); return; } diff --git a/ksml-runner/src/test/java/io/axual/ksml/runner/backend/KafkaProducerRunnerTest.java b/ksml-runner/src/test/java/io/axual/ksml/runner/backend/KafkaProducerRunnerTest.java index 535b0dcd..5753e135 100644 --- a/ksml-runner/src/test/java/io/axual/ksml/runner/backend/KafkaProducerRunnerTest.java +++ b/ksml-runner/src/test/java/io/axual/ksml/runner/backend/KafkaProducerRunnerTest.java @@ -25,7 +25,6 @@ import io.axual.ksml.data.mapper.NativeDataObjectMapper; import io.axual.ksml.data.notation.NotationLibrary; import io.axual.ksml.data.notation.binary.BinaryNotation; -import io.axual.ksml.data.notation.json.JsonDataObjectConverter; import io.axual.ksml.data.notation.json.JsonNotation; import io.axual.ksml.data.parser.ParseNode; import io.axual.ksml.definition.parser.TopologyDefinitionParser; @@ -128,9 +127,9 @@ void verifyCondition() throws Exception { */ private Map loadDefinitions(String filename) throws IOException, URISyntaxException { final var mapper = new NativeDataObjectMapper(); - final var jsonNotation = new JsonNotation(mapper); - NotationLibrary.register(BinaryNotation.NOTATION_NAME, new BinaryNotation(mapper, jsonNotation::serde), null); - NotationLibrary.register(JsonNotation.NOTATION_NAME, jsonNotation, new JsonDataObjectConverter()); + final var jsonNotation = new JsonNotation(mapper, null); + NotationLibrary.register(BinaryNotation.NOTATION_NAME, new BinaryNotation(mapper, jsonNotation::serde)); + NotationLibrary.register("json", jsonNotation); final var uri = ClassLoader.getSystemResource(filename).toURI(); final var path = Paths.get(uri); diff --git a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java index 1130d858..90da3a51 100644 --- a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java +++ b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java @@ -45,7 +45,7 @@ void shouldLoadWithoutExceptions() throws IOException { final var yaml = getClass().getClassLoader().getResourceAsStream("ksml-runner-config.yaml"); final var ksmlRunnerConfig = objectMapper.readValue(yaml, KSMLRunnerConfig.class); - assertNotNull(ksmlRunnerConfig.getKsmlConfig()); - assertNotNull(ksmlRunnerConfig.getKafkaConfig()); + assertNotNull(ksmlRunnerConfig.ksmlConfig()); + assertNotNull(ksmlRunnerConfig.kafkaConfig()); } } diff --git a/ksml/NOTICE.txt b/ksml/NOTICE.txt index 540472bc..a55bcf09 100644 --- a/ksml/NOTICE.txt +++ b/ksml/NOTICE.txt @@ -1,5 +1,5 @@ -Lists of 72 third-party dependencies. +Lists of 109 third-party dependencies. (MIT License) minimal-json (com.eclipsesource.minimal-json:minimal-json:0.9.5 - https://github.com/ralfstx/minimal-json) (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.17.1 - https://github.com/FasterXML/jackson) (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.17.1 - https://github.com/FasterXML/jackson-core) @@ -16,9 +16,18 @@ Lists of 72 third-party dependencies. (The Apache Software License, Version 2.0) Guava ListenableFuture only (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava - https://github.com/google/guava/listenablefuture) (Apache License, Version 2.0) J2ObjC Annotations (com.google.j2objc:j2objc-annotations:3.0.0 - https://github.com/google/j2objc/) (Go License) RE2/J (com.google.re2j:re2j:1.6 - http://github.com/google/re2j) + (MIT License) Microsoft Kiota-Java Abstractions (com.microsoft.kiota:microsoft-kiota-abstractions:1.1.14 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Form (com.microsoft.kiota:microsoft-kiota-serialization-form:1.1.14 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Multipart (com.microsoft.kiota:microsoft-kiota-serialization-multipart:1.1.14 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Text (com.microsoft.kiota:microsoft-kiota-serialization-text:1.1.14 - https://github.com/microsoft/kiota-java) (Eclipse Distribution License - v 1.0) Jakarta SOAP Implementation (com.sun.xml.messaging.saaj:saaj-impl:3.0.4 - https://projects.eclipse.org/projects/ee4j/metro-saaj/saaj-impl) (Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.17.0 - https://commons.apache.org/proper/commons-codec/) (Apache-2.0) Apache Commons IO (commons-io:commons-io:2.16.1 - https://commons.apache.org/proper/commons-io/) + (Apache License Version 2.0) apicurio-registry-common (io.apicurio:apicurio-registry-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-common/) + (Apache License Version 2.0) apicurio-registry-java-sdk (io.apicurio:apicurio-registry-java-sdk:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-java-sdk/) + (Apache License Version 2.0) apicurio-registry-schema-resolver (io.apicurio:apicurio-registry-schema-resolver:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-schema-resolver/) + (Apache License Version 2.0) apicurio-registry-serde-common (io.apicurio:apicurio-registry-serde-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serde-common/) + (Apache License Version 2.0) apicurio-registry-serdes-avro-serde (io.apicurio:apicurio-registry-serdes-avro-serde:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serdes-avro-serde/) (Apache 2.0) KSML Data Library (io.axual.ksml:ksml-data:1.0.3-SNAPSHOT - https://github.com/Axual/ksml/ksml-data) (Apache 2.0) KSML Data Library - AVRO (io.axual.ksml:ksml-data-avro:1.0.3-SNAPSHOT - https://github.com/Axual/ksml/ksml-data-avro) (Apache 2.0) KSML Data Library - CSV (io.axual.ksml:ksml-data-csv:1.0.3-SNAPSHOT - https://github.com/Axual/ksml/ksml-data-csv) @@ -32,9 +41,35 @@ Lists of 72 third-party dependencies. (The Apache Software License, Version 2.0) Log Redactor Metrics (io.confluent:logredactor-metrics:1.0.12 - https://github.com/confluentinc/logredactor) (Apache License 2.0) Metrics Core (io.dropwizard.metrics:metrics-core:4.2.25 - https://metrics.dropwizard.io/metrics-core) (Apache License 2.0) Metrics Integration with JMX (io.dropwizard.metrics:metrics-jmx:4.2.25 - https://metrics.dropwizard.io/metrics-jmx) + (Apache License, Version 2.0) Standard Uri Template (io.github.std-uritemplate:std-uritemplate:0.0.59 - https://std-uritemplate.github.io/) + (Apache-2.0) Kiota - Http - Vert.X (io.kiota:kiota-http-vertx:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-http-vertx) + (Apache-2.0) Kiota - Serialization - Jackson (io.kiota:kiota-serialization-jackson:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-serialization-jackson-parent/kiota-serialization-jackson) + (Apache License, Version 2.0) Netty/Buffer (io.netty:netty-buffer:4.1.108.Final - https://netty.io/netty-buffer/) + (Apache License, Version 2.0) Netty/Codec (io.netty:netty-codec:4.1.108.Final - https://netty.io/netty-codec/) + (Apache License, Version 2.0) Netty/Codec/DNS (io.netty:netty-codec-dns:4.1.108.Final - https://netty.io/netty-codec-dns/) + (Apache License, Version 2.0) Netty/Codec/HTTP (io.netty:netty-codec-http:4.1.108.Final - https://netty.io/netty-codec-http/) + (Apache License, Version 2.0) Netty/Codec/HTTP2 (io.netty:netty-codec-http2:4.1.108.Final - https://netty.io/netty-codec-http2/) + (Apache License, Version 2.0) Netty/Codec/Socks (io.netty:netty-codec-socks:4.1.108.Final - https://netty.io/netty-codec-socks/) + (Apache License, Version 2.0) Netty/Common (io.netty:netty-common:4.1.108.Final - https://netty.io/netty-common/) + (Apache License, Version 2.0) Netty/Handler (io.netty:netty-handler:4.1.108.Final - https://netty.io/netty-handler/) + (Apache License, Version 2.0) Netty/Handler/Proxy (io.netty:netty-handler-proxy:4.1.108.Final - https://netty.io/netty-handler-proxy/) + (Apache License, Version 2.0) Netty/Resolver (io.netty:netty-resolver:4.1.108.Final - https://netty.io/netty-resolver/) + (Apache License, Version 2.0) Netty/Resolver/DNS (io.netty:netty-resolver-dns:4.1.108.Final - https://netty.io/netty-resolver-dns/) + (Apache License, Version 2.0) Netty/Transport (io.netty:netty-transport:4.1.108.Final - https://netty.io/netty-transport/) + (Apache License, Version 2.0) Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.108.Final - https://netty.io/netty-transport-native-unix-common/) + (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-api:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) + (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-context:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) (Apache License 2.0) swagger-annotations (io.swagger.core.v3:swagger-annotations:2.1.10 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-common (io.vertx:vertx-auth-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-common) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-oauth2 (io.vertx:vertx-auth-oauth2:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-oauth2) + (Eclipse Public License - v 2.0) (The Apache Software License, Version 2.0) Vert.x Core (io.vertx:vertx-core:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-core) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) Vert.x URI Template (io.vertx:vertx-uri-template:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-uri-template) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-client (io.vertx:vertx-web-client:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-client) + (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-common (io.vertx:vertx-web-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-common) (EDL 1.0) Jakarta Activation API (jakarta.activation:jakarta.activation-api:2.1.3 - https://github.com/jakartaee/jaf-api) + (EPL 2.0) (GPL2 w/ CPE) Jakarta Annotations API (jakarta.annotation:jakarta.annotation-api:3.0.0 - https://projects.eclipse.org/projects/ee4j.ca) (Apache License 2.0) Jakarta Bean Validation API (jakarta.validation:jakarta.validation-api:3.0.2 - https://beanvalidation.org) + (EPL-2.0) (GPL-2.0-with-classpath-exception) Jakarta RESTful WS API (jakarta.ws.rs:jakarta.ws.rs-api:3.1.0 - https://github.com/eclipse-ee4j/jaxrs-api) (Eclipse Distribution License - v 1.0) Jakarta SOAP with Attachments API (jakarta.xml.soap:jakarta.xml.soap-api:3.0.2 - https://github.com/jakartaee/saaj-api) (Apache-2.0) Apache Avro (org.apache.avro:avro:1.11.3 - https://avro.apache.org) (Apache-2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.26.2 - https://commons.apache.org/proper/commons-compress/) @@ -64,6 +99,8 @@ Lists of 72 third-party dependencies. (Universal Permissive License, Version 1.0) Truffle Nfi (org.graalvm.truffle:truffle-nfi:23.1.2 - http://openjdk.java.net/projects/graal) (Universal Permissive License, Version 1.0) Truffle Nfi Libffi (org.graalvm.truffle:truffle-nfi-libffi:23.1.2 - http://openjdk.java.net/projects/graal) (Universal Permissive License, Version 1.0) Truffle Runtime (org.graalvm.truffle:truffle-runtime:23.1.2 - http://openjdk.java.net/projects/graal) + (Apache License, version 2.0) JBoss Logging 3 (org.jboss.logging:jboss-logging:3.4.1.Final - http://www.jboss.org) + (Apache License 2.0) slf4j to JBoss Logging Adapter (org.jboss.slf4j:slf4j-jboss-logging:1.2.1.Final - http://www.jboss.org) (Eclipse Distribution License - v 1.0) Extended StAX API (org.jvnet.staxex:stax-ex:2.1.0 - https://projects.eclipse.org/projects/ee4j/stax-ex) (The Apache Software License, Version 2.0) LZ4 and xxHash (org.lz4:lz4-java:1.8.0 - https://github.com/lz4/lz4-java) (The MIT License) Project Lombok (org.projectlombok:lombok:1.18.32 - https://projectlombok.org) diff --git a/ksml/src/main/java/io/axual/ksml/data/mapper/DataObjectConverter.java b/ksml/src/main/java/io/axual/ksml/data/mapper/DataObjectConverter.java index 8cf6c0e9..9b60e792 100644 --- a/ksml/src/main/java/io/axual/ksml/data/mapper/DataObjectConverter.java +++ b/ksml/src/main/java/io/axual/ksml/data/mapper/DataObjectConverter.java @@ -101,14 +101,14 @@ private DataObject applyNotationConverters(String sourceNotation, DataObject val } // First we see if the target notation is able to interpret the source value - var targetConverter = NotationLibrary.converter(targetType.notation()); + var targetConverter = NotationLibrary.notation(targetType.notation()).converter(); if (targetConverter != null) { final var target = targetConverter.convert(value, targetType); if (target != null && targetType.dataType().isAssignableFrom(target.type())) return target; } // If the target notation was not able to convert, then try the source notation - var sourceConverter = NotationLibrary.converter(sourceNotation); + var sourceConverter = NotationLibrary.notation(sourceNotation).converter(); if (sourceConverter != null) { final var target = sourceConverter.convert(value, targetType); if (target != null && targetType.dataType().isAssignableFrom(target.type())) return target; diff --git a/ksml/src/main/java/io/axual/ksml/generator/StreamDataType.java b/ksml/src/main/java/io/axual/ksml/generator/StreamDataType.java index 48bf81da..7c40f2a0 100644 --- a/ksml/src/main/java/io/axual/ksml/generator/StreamDataType.java +++ b/ksml/src/main/java/io/axual/ksml/generator/StreamDataType.java @@ -47,7 +47,7 @@ public String toString() { } public Serde serde() { - final var notation = NotationLibrary.get(userType.notation()); + final var notation = NotationLibrary.notation(userType.notation()); if (userType.dataType() instanceof UnionType unionType) return new UnionSerde(FLATTENER.flatten(unionType), isKey, notation::serde); var serde = notation.serde(FLATTENER.flatten(userType.dataType()), isKey); diff --git a/ksml/src/main/java/io/axual/ksml/parser/UserTypeParser.java b/ksml/src/main/java/io/axual/ksml/parser/UserTypeParser.java index 093ddf09..2d82ad69 100644 --- a/ksml/src/main/java/io/axual/ksml/parser/UserTypeParser.java +++ b/ksml/src/main/java/io/axual/ksml/parser/UserTypeParser.java @@ -20,20 +20,17 @@ * =========================LICENSE_END================================== */ +import io.axual.ksml.data.exception.DataException; import io.axual.ksml.data.exception.ParseException; +import io.axual.ksml.data.mapper.DataTypeSchemaMapper; +import io.axual.ksml.data.notation.NotationLibrary; import io.axual.ksml.data.notation.UserTupleType; import io.axual.ksml.data.notation.UserType; -import io.axual.ksml.data.notation.avro.AvroNotation; -import io.axual.ksml.data.notation.csv.CsvNotation; -import io.axual.ksml.data.notation.json.JsonNotation; -import io.axual.ksml.data.notation.soap.SOAPNotation; -import io.axual.ksml.data.notation.xml.XmlNotation; import io.axual.ksml.data.object.*; import io.axual.ksml.data.parser.schema.DataSchemaDSL; import io.axual.ksml.data.schema.DataSchema; import io.axual.ksml.data.schema.EnumSchema; import io.axual.ksml.data.schema.SchemaLibrary; -import io.axual.ksml.data.schema.StructSchema; import io.axual.ksml.data.type.*; import io.axual.ksml.exception.TopologyException; @@ -95,8 +92,18 @@ private record DecomposedType(String notation, String datatype) { private static UserType parseTypeAndNotation(String composedType, String defaultNotation) { final var decomposedType = decompose(composedType, defaultNotation); - final var datatype = decomposedType.datatype(); final var notation = decomposedType.notation(); + final var datatype = decomposedType.datatype(); + + // Internal types + final var internalType = parseType(datatype); + if (internalType != null) { + final var n = NotationLibrary.notation(notation); + if (n.defaultType() != null && !n.defaultType().isAssignableFrom(internalType)) { + throw new DataException("Notation " + notation + " does not allow for data type " + datatype); + } + return new UserType(notation, internalType); + } // List type if (datatype.startsWith(SQUARE_BRACKET_OPEN)) { @@ -104,9 +111,9 @@ private static UserType parseTypeAndNotation(String composedType, String default throw new ParseException("List type not properly closed: " + datatype); } // Parse the type in brackets separately as the type of list elements. Notation overrides are not allowed - // for list elements. If specified (eg. "[avro:SomeSchema]") then the value notation is ignored. + // for list elements. If specified (eg. "[avro:SomeSchema]") then the notation is ignored. var valueType = parseTypeAndNotation(datatype.substring(1, datatype.length() - 1), notation); - // Return the parsed value type with the above parsed notation + // Return as list of parsed value type using notation specified by the above parsed parent list type return new UserType(notation, new ListType(valueType.dataType())); } @@ -137,70 +144,26 @@ private static UserType parseTypeAndNotation(String composedType, String default return new UserType(notation, new WindowedType(parseType(windowedType))); } - // AVRO with schema - if (notation.equalsIgnoreCase(AvroNotation.NOTATION_NAME)) { - var schema = SchemaLibrary.getSchema(AvroNotation.NOTATION_NAME, datatype, false); - if (!(schema instanceof StructSchema structSchema)) - throw new ParseException("AVRO schema is not a STRUCT: " + datatype); - return new UserType(AvroNotation.NOTATION_NAME, new StructType(structSchema)); - } - - // AVRO without schema - if (datatype.equalsIgnoreCase(AvroNotation.NOTATION_NAME)) { - return new UserType(AvroNotation.NOTATION_NAME, AvroNotation.DEFAULT_TYPE); - } - - // CSV with schema - if (notation.equalsIgnoreCase(CsvNotation.NOTATION_NAME)) { - var schema = SchemaLibrary.getSchema(CsvNotation.NOTATION_NAME, datatype, false); - if (!(schema instanceof StructSchema structSchema)) - throw new ParseException("CSV schema is not a STRUCT: " + datatype); - return new UserType(CsvNotation.NOTATION_NAME, new StructType(structSchema)); - } - - // CSV without schema - if (datatype.equalsIgnoreCase(CsvNotation.NOTATION_NAME)) { - return new UserType(CsvNotation.NOTATION_NAME, CsvNotation.DEFAULT_TYPE); - } - - // JSON with schema - if (notation.equalsIgnoreCase(JsonNotation.NOTATION_NAME)) { - var schema = SchemaLibrary.getSchema(JsonNotation.NOTATION_NAME, datatype, false); - if (!(schema instanceof StructSchema structSchema)) - throw new ParseException("JSON schema is not a STRUCT: " + datatype); - return new UserType(JsonNotation.NOTATION_NAME, new StructType(structSchema)); - } - - // JSON without schema - if (datatype.equalsIgnoreCase(JsonNotation.NOTATION_NAME)) { - return new UserType(JsonNotation.NOTATION_NAME, JsonNotation.DEFAULT_TYPE); - } - - // SOAP with schema (not implemented yet) - if (notation.equalsIgnoreCase(SOAPNotation.NOTATION_NAME)) { - return new UserType(SOAPNotation.NOTATION_NAME, SOAPNotation.DEFAULT_TYPE); - } - - // SOAP without schema - if (datatype.equalsIgnoreCase(SOAPNotation.NOTATION_NAME)) { - return new UserType(SOAPNotation.NOTATION_NAME, SOAPNotation.DEFAULT_TYPE); - } - - // XML with schema - if (notation.equalsIgnoreCase(XmlNotation.NOTATION_NAME)) { - var schema = SchemaLibrary.getSchema(XmlNotation.NOTATION_NAME, datatype, false); - if (!(schema instanceof StructSchema structSchema)) - throw new ParseException("XML schema is not a STRUCT: " + datatype); - return new UserType(XmlNotation.NOTATION_NAME, new StructType(structSchema)); + // Notation type with specific schema + if (NotationLibrary.exists(notation)) { + final var loader = NotationLibrary.notation(notation).loader(); + if (loader != null) { + // If we reach this point, then we assume that the datatype refers to a schema to load + final var schema = SchemaLibrary.getSchema(notation, datatype, false); + final var schemaType = new DataTypeSchemaMapper().fromDataSchema(schema); + if (!(NotationLibrary.notation(notation).defaultType().isAssignableFrom(schemaType))) { + throw new DataException("Notation " + notation + " does not allow for data type " + datatype); + } + return new UserType(notation, schemaType); + } } - // XML without schema - if (datatype.equalsIgnoreCase(XmlNotation.NOTATION_NAME)) { - return new UserType(XmlNotation.NOTATION_NAME, XmlNotation.DEFAULT_TYPE); + // Notation without specific schema + if (NotationLibrary.exists(datatype)) { + return new UserType(notation, NotationLibrary.notation(datatype).defaultType()); } - // Parse the type as an in-built primary data type and return it - return new UserType(notation, parseType(datatype)); + throw new TopologyException("Unknown data type: " + datatype); } // This method decomposes a user type into its components. User types are always of the form "notation:datatype". @@ -217,7 +180,7 @@ private static DecomposedType decompose(String composedType, String defaultNotat final var parsedType = composedType.substring(composedType.indexOf(NOTATION_SEPARATOR) + 1); // Return the parsed notation and datatype - return new DecomposedType(parsedNotation.toUpperCase(), parsedType); + return new DecomposedType(parsedNotation, parsedType); } // Return the whole type string to be processed further, along with default notation @@ -252,7 +215,7 @@ private static DataType parseType(String type) { case DataSchemaDSL.STRING_TYPE, DataSchemaDSL.STRING_TYPE_ALTERNATIVE -> DataString.DATATYPE; case DataSchemaDSL.STRUCT_TYPE -> new StructType(); case DataSchemaDSL.ANY_TYPE, DataSchemaDSL.UNKNOWN_TYPE -> DataType.UNKNOWN; - default -> throw new TopologyException("Can not derive dataType: " + type); + default -> null; }; } diff --git a/ksml/src/test/java/io/axual/ksml/BasicStreamRunTest.java b/ksml/src/test/java/io/axual/ksml/BasicStreamRunTest.java index f343b9ec..ca8cc07b 100644 --- a/ksml/src/test/java/io/axual/ksml/BasicStreamRunTest.java +++ b/ksml/src/test/java/io/axual/ksml/BasicStreamRunTest.java @@ -27,10 +27,9 @@ import io.axual.ksml.data.notation.avro.AvroSchemaLoader; import io.axual.ksml.data.notation.avro.MockAvroNotation; import io.axual.ksml.data.notation.binary.BinaryNotation; -import io.axual.ksml.data.notation.json.JsonDataObjectConverter; import io.axual.ksml.data.notation.json.JsonNotation; +import io.axual.ksml.data.notation.json.JsonSchemaLoader; import io.axual.ksml.data.parser.ParseNode; -import io.axual.ksml.data.schema.SchemaLibrary; import io.axual.ksml.definition.parser.TopologyDefinitionParser; import io.axual.ksml.generator.YAMLObjectMapper; import io.confluent.kafka.serializers.KafkaAvroDeserializer; @@ -64,9 +63,9 @@ public class BasicStreamRunTest { @Test void parseAndCheckOuput() throws Exception { final var mapper = new NativeDataObjectMapper(); - final var jsonNotation = new JsonNotation(mapper); - NotationLibrary.register(BinaryNotation.NOTATION_NAME, new BinaryNotation(mapper, jsonNotation::serde), null); - NotationLibrary.register(JsonNotation.NOTATION_NAME, jsonNotation, new JsonDataObjectConverter()); + final var jsonNotation = new JsonNotation(mapper, new JsonSchemaLoader(".")); + NotationLibrary.register(BinaryNotation.NOTATION_NAME, new BinaryNotation(mapper, jsonNotation::serde)); + NotationLibrary.register("json", jsonNotation); final var uri = ClassLoader.getSystemResource("pipelines/test-copying.yaml").toURI(); final var path = Paths.get(uri); @@ -91,17 +90,15 @@ void parseAndCheckOuput() throws Exception { @Test void testFilterAvroRecords() throws Exception { final var mapper = new NativeDataObjectMapper(); - final var jsonNotation = new JsonNotation(mapper); - NotationLibrary.register(BinaryNotation.NOTATION_NAME, new BinaryNotation(mapper, jsonNotation::serde), null); - NotationLibrary.register(JsonNotation.NOTATION_NAME, jsonNotation, new JsonDataObjectConverter()); - - final var avroNotation = new MockAvroNotation(new HashMap<>()); - NotationLibrary.register(MockAvroNotation.NOTATION_NAME, avroNotation, null); + final var jsonNotation = new JsonNotation(mapper, new JsonSchemaLoader(".")); + NotationLibrary.register(BinaryNotation.NOTATION_NAME, new BinaryNotation(mapper, jsonNotation::serde)); + NotationLibrary.register("json", jsonNotation); URI testDirectory = ClassLoader.getSystemResource("pipelines").toURI(); String schemaPath = testDirectory.getPath(); System.out.println("schemaPath = " + schemaPath); - SchemaLibrary.registerLoader(MockAvroNotation.NOTATION_NAME, new AvroSchemaLoader(schemaPath)); + final var avroNotation = new MockAvroNotation(new HashMap<>(), new AvroSchemaLoader(schemaPath)); + NotationLibrary.register(MockAvroNotation.NOTATION_NAME, avroNotation); final var uri = ClassLoader.getSystemResource("pipelines/test-filtering.yaml").toURI(); final var path = Paths.get(uri); diff --git a/ksml/src/test/java/io/axual/ksml/TopologyGeneratorBasicTest.java b/ksml/src/test/java/io/axual/ksml/TopologyGeneratorBasicTest.java index 6036f5fd..b8512471 100644 --- a/ksml/src/test/java/io/axual/ksml/TopologyGeneratorBasicTest.java +++ b/ksml/src/test/java/io/axual/ksml/TopologyGeneratorBasicTest.java @@ -25,7 +25,6 @@ import io.axual.ksml.data.mapper.NativeDataObjectMapper; import io.axual.ksml.data.notation.NotationLibrary; import io.axual.ksml.data.notation.binary.BinaryNotation; -import io.axual.ksml.data.notation.json.JsonDataObjectConverter; import io.axual.ksml.data.notation.json.JsonNotation; import io.axual.ksml.data.parser.ParseNode; import io.axual.ksml.definition.parser.TopologyDefinitionParser; @@ -56,9 +55,9 @@ public class TopologyGeneratorBasicTest { @ValueSource(ints = {1, 2, 3, 4, 5}) void parseAndCheckOuput(int nr) throws Exception { final var mapper = new NativeDataObjectMapper(); - final var jsonNotation = new JsonNotation(mapper); - NotationLibrary.register(BinaryNotation.NOTATION_NAME, new BinaryNotation(mapper, jsonNotation::serde), null); - NotationLibrary.register(JsonNotation.NOTATION_NAME, jsonNotation, new JsonDataObjectConverter()); + final var jsonNotation = new JsonNotation(mapper, null); + NotationLibrary.register(BinaryNotation.NOTATION_NAME, new BinaryNotation(mapper, jsonNotation::serde)); + NotationLibrary.register("json", jsonNotation); final var uri = ClassLoader.getSystemResource("pipelines/" + nr + "-demo.yaml").toURI(); final var path = Paths.get(uri); diff --git a/ksml/src/test/java/io/axual/ksml/testutil/KSMLTestExtension.java b/ksml/src/test/java/io/axual/ksml/testutil/KSMLTestExtension.java index 11ae2fc5..ea4f304e 100644 --- a/ksml/src/test/java/io/axual/ksml/testutil/KSMLTestExtension.java +++ b/ksml/src/test/java/io/axual/ksml/testutil/KSMLTestExtension.java @@ -23,15 +23,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.axual.ksml.TopologyGenerator; +import io.axual.ksml.data.loader.SchemaLoader; import io.axual.ksml.data.mapper.NativeDataObjectMapper; import io.axual.ksml.data.notation.NotationLibrary; import io.axual.ksml.data.notation.avro.AvroSchemaLoader; import io.axual.ksml.data.notation.avro.MockAvroNotation; import io.axual.ksml.data.notation.binary.BinaryNotation; -import io.axual.ksml.data.notation.json.JsonDataObjectConverter; import io.axual.ksml.data.notation.json.JsonNotation; import io.axual.ksml.data.parser.ParseNode; -import io.axual.ksml.data.schema.SchemaLibrary; import io.axual.ksml.definition.parser.TopologyDefinitionParser; import io.axual.ksml.generator.YAMLObjectMapper; import io.confluent.kafka.serializers.KafkaAvroDeserializer; @@ -64,8 +63,6 @@ public class KSMLTestExtension implements ExecutionCondition, BeforeAllCallback, private TopologyTestDriver topologyTestDriver; - final MockAvroNotation avroNotation = new MockAvroNotation(new HashMap<>()); - /** * Check if the test(s) are running on GraalVM. * @@ -94,10 +91,9 @@ public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext ext public void beforeAll(ExtensionContext extensionContext) { log.debug("registering notations"); final var mapper = new NativeDataObjectMapper(); - final var jsonNotation = new JsonNotation(mapper); - NotationLibrary.register(BinaryNotation.NOTATION_NAME, new BinaryNotation(mapper, jsonNotation::serde), null); - NotationLibrary.register(JsonNotation.NOTATION_NAME, jsonNotation, new JsonDataObjectConverter()); - NotationLibrary.register(MockAvroNotation.NOTATION_NAME, avroNotation, null); + final var jsonNotation = new JsonNotation(mapper, null); + NotationLibrary.register(BinaryNotation.NOTATION_NAME, new BinaryNotation(mapper, jsonNotation::serde)); + NotationLibrary.register("json", jsonNotation); } @Override @@ -118,14 +114,17 @@ public void beforeEach(ExtensionContext extensionContext) throws Exception { } log.debug("Setting up KSML test"); + // set up AVRO loader + SchemaLoader avroLoader = null; // if a schema path is specified, set up AVRO if (!ksmlTest.schemapath().equals(ksmlTest.NO_SCHEMAS)) { log.debug("register schema path: `{}`", ksmlTest.schemapath()); URI schemaDirectory = ClassLoader.getSystemResource(ksmlTest.schemapath()).toURI(); String schemaPath = schemaDirectory.getPath(); - SchemaLibrary.registerLoader(MockAvroNotation.NOTATION_NAME, new AvroSchemaLoader(schemaPath)); + avroLoader = new AvroSchemaLoader(schemaPath); log.debug("registered schema path: {}", schemaPath); } + NotationLibrary.register(MockAvroNotation.NOTATION_NAME, new MockAvroNotation(new HashMap<>(), avroLoader)); // get the KSML definition classpath relative path and load the topology into the test driver String topologyName = ksmlTest.topology(); @@ -182,8 +181,6 @@ public void afterEach(ExtensionContext context) throws Exception { // method was not annotated, nothing to do return; } - // to decide: unregister schema directory? But not like this probably - SchemaLibrary.registerLoader(MockAvroNotation.NOTATION_NAME, null); // clean up if (topologyTestDriver != null) { @@ -211,6 +208,7 @@ private Serializer getValueSerializer(KSMLTopic ksmlTopic) { private Serializer getSerializer(KSMLTopic ksmlTopic, boolean isKey) { return switch (isKey ? ksmlTopic.keySerde() : ksmlTopic.valueSerde()) { case AVRO -> { + final var avroNotation = (MockAvroNotation) NotationLibrary.notation(MockAvroNotation.NOTATION_NAME); var result = new KafkaAvroSerializer(avroNotation.mockSchemaRegistryClient()); result.configure(avroNotation.getSchemaRegistryConfigs(), isKey); yield result; @@ -230,7 +228,8 @@ private Deserializer getValueDeserializer(KSMLTopic kamlTopic) { private Deserializer getDeserializer(KSMLTopic kamlTopic, boolean isKey) { return switch (isKey ? kamlTopic.keySerde() : kamlTopic.valueSerde()) { case AVRO -> { - var result = new KafkaAvroDeserializer(avroNotation.mockSchemaRegistryClient()); + final var avroNotation = (MockAvroNotation) NotationLibrary.notation(MockAvroNotation.NOTATION_NAME); + final var result = new KafkaAvroDeserializer(avroNotation.mockSchemaRegistryClient()); result.configure(avroNotation.getSchemaRegistryConfigs(), isKey); yield result; } diff --git a/pom.xml b/pom.xml index c54678d9..3581e077 100644 --- a/pom.xml +++ b/pom.xml @@ -65,6 +65,7 @@ 1.12.0 1.26.2 3.8.0 + 3.0.0-SNAPSHOT 7.6.1 23.1.2 23.1.2 @@ -228,6 +229,11 @@ ${apache.kafka.version} + + io.apicurio + apicurio-registry-serdes-avro-serde + ${apicurio.avro.version} + io.confluent kafka-avro-serializer From 3e90a15b30428e8998014149c1ba2acbd8d490ae Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Fri, 11 Oct 2024 11:20:42 +0200 Subject: [PATCH 04/55] Updates to the code --- .../data/notation/avro/MockAvroNotation.java | 2 +- .../ksml/data/notation/NotationLibrary.java | 8 ++++---- .../io/axual/ksml/data/notation/UserType.java | 2 +- .../data/notation/binary/BinaryNotation.java | 2 +- .../ksml/data/notation/json/JsonNotation.java | 1 + .../java/io/axual/ksml/runner/KSMLRunner.java | 20 +++++++++---------- .../ksml/runner/config/KSMLRunnerConfig.java | 20 +++++++++---------- .../backend/KafkaProducerRunnerTest.java | 4 ++-- .../runner/config/KSMLRunnerConfigTest.java | 4 ++-- .../io/axual/ksml/BasicStreamRunTest.java | 10 +++++----- .../ksml/TopologyGeneratorBasicTest.java | 4 ++-- .../ksml/dsl/GlobalTableDefinitionTest.java | 2 +- .../axual/ksml/dsl/StreamDefinitionTest.java | 2 +- .../axual/ksml/dsl/TableDefinitionTest.java | 2 +- .../axual/ksml/python/PythonFunctionTest.java | 4 ++-- .../ksml/testutil/KSMLTestExtension.java | 10 +++++----- 16 files changed, 49 insertions(+), 48 deletions(-) diff --git a/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/MockAvroNotation.java b/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/MockAvroNotation.java index 5800f892..23a3ef19 100644 --- a/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/MockAvroNotation.java +++ b/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/MockAvroNotation.java @@ -53,7 +53,7 @@ * instance of {@link MockSchemaRegistry} to be used. */ public class MockAvroNotation implements Notation { - public static final String NOTATION_NAME = "avro"; + public static final String NAME = "avro"; public static final DataType DEFAULT_TYPE = new StructType(); private static final AvroDataObjectMapper mapper = new AvroDataObjectMapper(); private final Map configs = new HashMap<>(); diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/NotationLibrary.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/NotationLibrary.java index a3a63c60..35a4e45b 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/NotationLibrary.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/NotationLibrary.java @@ -29,18 +29,18 @@ public class NotationLibrary { private NotationLibrary() { } - private static final Map notationEntries = new HashMap<>(); + private static final Map notations = new HashMap<>(); public static void register(String name, Notation notation) { - notationEntries.put(name, notation); + notations.put(name, notation); } public static boolean exists(String notation) { - return notationEntries.containsKey(notation); + return notations.containsKey(notation); } public static Notation notation(String notation) { - var result = notation != null ? notationEntries.get(notation) : null; + var result = notation != null ? notations.get(notation) : null; if (result != null) return result; throw new DataException("Data notation is not registered in the NotationLibrary: " + (notation != null ? notation : "null")); } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/UserType.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/UserType.java index 27268c2f..3ee280a5 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/UserType.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/UserType.java @@ -27,7 +27,7 @@ import java.util.Arrays; public record UserType(String notation, DataType dataType) { - public static final String DEFAULT_NOTATION = BinaryNotation.NOTATION_NAME; + public static final String DEFAULT_NOTATION = BinaryNotation.NAME; public static final UserType UNKNOWN = new UserType(DataType.UNKNOWN); public UserType(DataType dataType) { diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/binary/BinaryNotation.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/binary/BinaryNotation.java index a36cf9f2..a416d241 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/binary/BinaryNotation.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/binary/BinaryNotation.java @@ -37,7 +37,7 @@ import org.apache.kafka.common.serialization.Serializer; public class BinaryNotation implements Notation { - public static final String NOTATION_NAME = "binary"; + public static final String NAME = "binary"; public static final DataType DEFAULT_TYPE = DataType.UNKNOWN; private final NativeDataObjectMapper nativeMapper; private final SerdeSupplier complexTypeSerdeSupplier; diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonNotation.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonNotation.java index 9f7a142d..acd0d354 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonNotation.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonNotation.java @@ -31,6 +31,7 @@ import org.apache.kafka.common.serialization.Serde; public class JsonNotation implements Notation { + public static final String NAME = "json"; public static final DataType DEFAULT_TYPE = new UnionType(new StructType(), new ListType()); private final NativeDataObjectMapper nativeMapper; @Getter diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java index cf530d45..2e836572 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java @@ -85,14 +85,14 @@ public static void main(String[] args) { } final var config = readConfiguration(configFile); - final var ksmlConfig = config.ksmlConfig(); + final var ksmlConfig = config.ksml(); log.info("Using directories: config: {}, schema: {}, storage: {}", ksmlConfig.getConfigDirectory(), ksmlConfig.getSchemaDirectory(), ksmlConfig.getStorageDirectory()); final var definitions = ksmlConfig.getDefinitions(); if (definitions == null || definitions.isEmpty()) { throw new ConfigException("definitions", definitions, "No KSML definitions found in configuration"); } - KsmlInfo.registerKsmlAppInfo(config.getApplicationId()); + KsmlInfo.registerKsmlAppInfo(config.applicationId()); // Start the appserver if needed final var appServer = ksmlConfig.getApplicationServerConfig(); @@ -105,7 +105,7 @@ public static void main(String[] args) { } // Set up all default notations and register them in the NotationLibrary - final var notationFactories = new NotationFactories(config.getKafkaConfig(), ksmlConfig.getSchemaDirectory()); + final var notationFactories = new NotationFactories(config.kafka().kafkaConfig(), ksmlConfig.getSchemaDirectory()); for (final var notation : notationFactories.notations().entrySet()) { NotationLibrary.register(notation.getKey(), notation.getValue().create(null)); } @@ -140,8 +140,8 @@ public static void main(String[] args) { } ExecutionContext.INSTANCE.serdeWrapper( serde -> new Serdes.WrapperSerde<>( - new ResolvingSerializer<>(serde.serializer(), config.getKafkaConfig()), - new ResolvingDeserializer<>(serde.deserializer(), config.getKafkaConfig()))); + new ResolvingSerializer<>(serde.serializer(), config.kafka().kafkaConfig()), + new ResolvingDeserializer<>(serde.deserializer(), config.kafka().kafkaConfig()))); final Map producerSpecs = new HashMap<>(); final Map pipelineSpecs = new HashMap<>(); @@ -164,13 +164,13 @@ public static void main(String[] args) { final var producer = producerSpecs.isEmpty() ? null : new KafkaProducerRunner(KafkaProducerRunner.Config.builder() .definitions(producerSpecs) - .kafkaConfig(config.getKafkaConfig()) + .kafkaConfig(config.kafka().kafkaConfig()) .build()); final var streams = pipelineSpecs.isEmpty() ? null : new KafkaStreamsRunner(KafkaStreamsRunner.Config.builder() .storageDirectory(ksmlConfig.getStorageDirectory()) .appServer(ksmlConfig.getApplicationServerConfig()) .definitions(pipelineSpecs) - .kafkaConfig(config.getKafkaConfig()) + .kafkaConfig(config.kafka().kafkaConfig()) .build()); if (producer != null || streams != null) { @@ -186,7 +186,7 @@ public static void main(String[] args) { Runtime.getRuntime().addShutdownHook(shutdownHook); - try (var prometheusExport = new PrometheusExport(config.ksmlConfig().prometheusConfig())) { + try (var prometheusExport = new PrometheusExport(config.ksml().prometheusConfig())) { prometheusExport.start(); final var executorService = Executors.newFixedThreadPool(2); final var producerFuture = producer == null ? CompletableFuture.completedFuture(null) : executorService.submit(producer); @@ -356,10 +356,10 @@ private static KSMLRunnerConfig readConfiguration(File configFile) { try { final var config = mapper.readValue(configFile, KSMLRunnerConfig.class); if (config != null) { - if (config.ksmlConfig() == null) { + if (config.ksml() == null) { throw new ConfigException("Section \"ksml\" is missing in configuration"); } - if (config.getKafkaConfig() == null) { + if (config.kafka() == null) { throw new ConfigException("Section \"kafka\" is missing in configuration"); } return config; diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java index 3492fabb..f5bac5e5 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java @@ -41,26 +41,26 @@ public class KSMLRunnerConfig { private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); @JsonProperty("ksml") - private KSMLConfig ksmlConfig; + private KSMLConfig ksml; @JsonProperty("kafka") - private KafkaConfig kafkaConfig; + private KafkaConfig kafka; - public Map getKafkaConfig() { - var newConfig = new HashMap<>(kafkaConfig.kafkaConfig()); - newConfig.put("application.id", kafkaConfig.applicationId()); - return newConfig; - } +// public Map kafkaConfig() { +// var newConfig = new HashMap<>(kafka.kafkaConfig()); +// newConfig.put("application.id", kafka.applicationId); +// return newConfig; +// } - public String getApplicationId() { - return kafkaConfig.applicationId(); + public String applicationId() { + return kafka.applicationId; } @Data public static class KafkaConfig { @JsonProperty("app.id") @JsonAlias({"applicationId", "application.id"}) - public String applicationId; + private String applicationId; @JsonIgnore private Map kafkaConfig = new HashMap<>(); diff --git a/ksml-runner/src/test/java/io/axual/ksml/runner/backend/KafkaProducerRunnerTest.java b/ksml-runner/src/test/java/io/axual/ksml/runner/backend/KafkaProducerRunnerTest.java index 5753e135..3f921fbb 100644 --- a/ksml-runner/src/test/java/io/axual/ksml/runner/backend/KafkaProducerRunnerTest.java +++ b/ksml-runner/src/test/java/io/axual/ksml/runner/backend/KafkaProducerRunnerTest.java @@ -128,8 +128,8 @@ void verifyCondition() throws Exception { private Map loadDefinitions(String filename) throws IOException, URISyntaxException { final var mapper = new NativeDataObjectMapper(); final var jsonNotation = new JsonNotation(mapper, null); - NotationLibrary.register(BinaryNotation.NOTATION_NAME, new BinaryNotation(mapper, jsonNotation::serde)); - NotationLibrary.register("json", jsonNotation); + NotationLibrary.register(BinaryNotation.NAME, new BinaryNotation(mapper, jsonNotation::serde)); + NotationLibrary.register(JsonNotation.NAME, jsonNotation); final var uri = ClassLoader.getSystemResource(filename).toURI(); final var path = Paths.get(uri); diff --git a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java index 90da3a51..b536e067 100644 --- a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java +++ b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java @@ -45,7 +45,7 @@ void shouldLoadWithoutExceptions() throws IOException { final var yaml = getClass().getClassLoader().getResourceAsStream("ksml-runner-config.yaml"); final var ksmlRunnerConfig = objectMapper.readValue(yaml, KSMLRunnerConfig.class); - assertNotNull(ksmlRunnerConfig.ksmlConfig()); - assertNotNull(ksmlRunnerConfig.kafkaConfig()); + assertNotNull(ksmlRunnerConfig.ksml()); + assertNotNull(ksmlRunnerConfig.kafka()); } } diff --git a/ksml/src/test/java/io/axual/ksml/BasicStreamRunTest.java b/ksml/src/test/java/io/axual/ksml/BasicStreamRunTest.java index ca8cc07b..09f2f646 100644 --- a/ksml/src/test/java/io/axual/ksml/BasicStreamRunTest.java +++ b/ksml/src/test/java/io/axual/ksml/BasicStreamRunTest.java @@ -64,8 +64,8 @@ public class BasicStreamRunTest { void parseAndCheckOuput() throws Exception { final var mapper = new NativeDataObjectMapper(); final var jsonNotation = new JsonNotation(mapper, new JsonSchemaLoader(".")); - NotationLibrary.register(BinaryNotation.NOTATION_NAME, new BinaryNotation(mapper, jsonNotation::serde)); - NotationLibrary.register("json", jsonNotation); + NotationLibrary.register(BinaryNotation.NAME, new BinaryNotation(mapper, jsonNotation::serde)); + NotationLibrary.register(JsonNotation.NAME, jsonNotation); final var uri = ClassLoader.getSystemResource("pipelines/test-copying.yaml").toURI(); final var path = Paths.get(uri); @@ -91,14 +91,14 @@ void parseAndCheckOuput() throws Exception { void testFilterAvroRecords() throws Exception { final var mapper = new NativeDataObjectMapper(); final var jsonNotation = new JsonNotation(mapper, new JsonSchemaLoader(".")); - NotationLibrary.register(BinaryNotation.NOTATION_NAME, new BinaryNotation(mapper, jsonNotation::serde)); - NotationLibrary.register("json", jsonNotation); + NotationLibrary.register(BinaryNotation.NAME, new BinaryNotation(mapper, jsonNotation::serde)); + NotationLibrary.register(JsonNotation.NAME, jsonNotation); URI testDirectory = ClassLoader.getSystemResource("pipelines").toURI(); String schemaPath = testDirectory.getPath(); System.out.println("schemaPath = " + schemaPath); final var avroNotation = new MockAvroNotation(new HashMap<>(), new AvroSchemaLoader(schemaPath)); - NotationLibrary.register(MockAvroNotation.NOTATION_NAME, avroNotation); + NotationLibrary.register(MockAvroNotation.NAME, avroNotation); final var uri = ClassLoader.getSystemResource("pipelines/test-filtering.yaml").toURI(); final var path = Paths.get(uri); diff --git a/ksml/src/test/java/io/axual/ksml/TopologyGeneratorBasicTest.java b/ksml/src/test/java/io/axual/ksml/TopologyGeneratorBasicTest.java index b8512471..59791709 100644 --- a/ksml/src/test/java/io/axual/ksml/TopologyGeneratorBasicTest.java +++ b/ksml/src/test/java/io/axual/ksml/TopologyGeneratorBasicTest.java @@ -56,8 +56,8 @@ public class TopologyGeneratorBasicTest { void parseAndCheckOuput(int nr) throws Exception { final var mapper = new NativeDataObjectMapper(); final var jsonNotation = new JsonNotation(mapper, null); - NotationLibrary.register(BinaryNotation.NOTATION_NAME, new BinaryNotation(mapper, jsonNotation::serde)); - NotationLibrary.register("json", jsonNotation); + NotationLibrary.register(BinaryNotation.NAME, new BinaryNotation(mapper, jsonNotation::serde)); + NotationLibrary.register(JsonNotation.NAME, jsonNotation); final var uri = ClassLoader.getSystemResource("pipelines/" + nr + "-demo.yaml").toURI(); final var path = Paths.get(uri); diff --git a/ksml/src/test/java/io/axual/ksml/dsl/GlobalTableDefinitionTest.java b/ksml/src/test/java/io/axual/ksml/dsl/GlobalTableDefinitionTest.java index 8b561b5f..9157051e 100644 --- a/ksml/src/test/java/io/axual/ksml/dsl/GlobalTableDefinitionTest.java +++ b/ksml/src/test/java/io/axual/ksml/dsl/GlobalTableDefinitionTest.java @@ -54,7 +54,7 @@ class GlobalTableDefinitionTest { @Test void testGlobalTableDefinition() { - NotationLibrary.register(BinaryNotation.NOTATION_NAME, mockNotation); + NotationLibrary.register(BinaryNotation.NAME, mockNotation); var stringType = UserTypeParser.parse("string"); // given a TableDefinition diff --git a/ksml/src/test/java/io/axual/ksml/dsl/StreamDefinitionTest.java b/ksml/src/test/java/io/axual/ksml/dsl/StreamDefinitionTest.java index a3a447ca..b0e312bc 100644 --- a/ksml/src/test/java/io/axual/ksml/dsl/StreamDefinitionTest.java +++ b/ksml/src/test/java/io/axual/ksml/dsl/StreamDefinitionTest.java @@ -52,7 +52,7 @@ class StreamDefinitionTest { @Test void testStreamDefinition() { - NotationLibrary.register(BinaryNotation.NOTATION_NAME, mockNotation); + NotationLibrary.register(BinaryNotation.NAME, mockNotation); // given a StreamDefinition var streamDefinition = new StreamDefinition("topic", "string", "string", null, null); diff --git a/ksml/src/test/java/io/axual/ksml/dsl/TableDefinitionTest.java b/ksml/src/test/java/io/axual/ksml/dsl/TableDefinitionTest.java index 1cd5bae3..c4a226ad 100644 --- a/ksml/src/test/java/io/axual/ksml/dsl/TableDefinitionTest.java +++ b/ksml/src/test/java/io/axual/ksml/dsl/TableDefinitionTest.java @@ -54,7 +54,7 @@ class TableDefinitionTest { @Test void testTableDefinition() { - NotationLibrary.register(BinaryNotation.NOTATION_NAME, mockNotation); + NotationLibrary.register(BinaryNotation.NAME, mockNotation); var stringType = UserTypeParser.parse("string"); // given a TableDefinition diff --git a/ksml/src/test/java/io/axual/ksml/python/PythonFunctionTest.java b/ksml/src/test/java/io/axual/ksml/python/PythonFunctionTest.java index fc3dc898..5969dcef 100644 --- a/ksml/src/test/java/io/axual/ksml/python/PythonFunctionTest.java +++ b/ksml/src/test/java/io/axual/ksml/python/PythonFunctionTest.java @@ -39,7 +39,7 @@ public class PythonFunctionTest { final ParameterDefinition one = new ParameterDefinition("one", DataInteger.DATATYPE); final ParameterDefinition two = new ParameterDefinition("two", DataInteger.DATATYPE); final ParameterDefinition[] params = new ParameterDefinition[]{one, two}; - final UserType resultType = new UserType(BinaryNotation.NOTATION_NAME, DataInteger.DATATYPE); + final UserType resultType = new UserType(BinaryNotation.NAME, DataInteger.DATATYPE); @ParameterizedTest @CsvSource({"1, 2, 3", "100,100,200", "100, -1, 99", "99, -100, -1"}) @@ -104,7 +104,7 @@ def myAddFunc(one, two): * Test that Null Key/Values are accepted as parameters */ void testNullKeyValue() { - final var stringResultType = new UserType(BinaryNotation.NOTATION_NAME, DataString.DATATYPE); + final var stringResultType = new UserType(BinaryNotation.NAME, DataString.DATATYPE); final var concatDef = FunctionDefinition.as("concat", params, null, null, "str(one is None) + ' ' + str(two is None)", stringResultType, null); final var concat = PythonFunction.forFunction(context, "test", "adder", concatDef); diff --git a/ksml/src/test/java/io/axual/ksml/testutil/KSMLTestExtension.java b/ksml/src/test/java/io/axual/ksml/testutil/KSMLTestExtension.java index ea4f304e..33ff64f2 100644 --- a/ksml/src/test/java/io/axual/ksml/testutil/KSMLTestExtension.java +++ b/ksml/src/test/java/io/axual/ksml/testutil/KSMLTestExtension.java @@ -92,8 +92,8 @@ public void beforeAll(ExtensionContext extensionContext) { log.debug("registering notations"); final var mapper = new NativeDataObjectMapper(); final var jsonNotation = new JsonNotation(mapper, null); - NotationLibrary.register(BinaryNotation.NOTATION_NAME, new BinaryNotation(mapper, jsonNotation::serde)); - NotationLibrary.register("json", jsonNotation); + NotationLibrary.register(BinaryNotation.NAME, new BinaryNotation(mapper, jsonNotation::serde)); + NotationLibrary.register(JsonNotation.NAME, jsonNotation); } @Override @@ -124,7 +124,7 @@ public void beforeEach(ExtensionContext extensionContext) throws Exception { avroLoader = new AvroSchemaLoader(schemaPath); log.debug("registered schema path: {}", schemaPath); } - NotationLibrary.register(MockAvroNotation.NOTATION_NAME, new MockAvroNotation(new HashMap<>(), avroLoader)); + NotationLibrary.register(MockAvroNotation.NAME, new MockAvroNotation(new HashMap<>(), avroLoader)); // get the KSML definition classpath relative path and load the topology into the test driver String topologyName = ksmlTest.topology(); @@ -208,7 +208,7 @@ private Serializer getValueSerializer(KSMLTopic ksmlTopic) { private Serializer getSerializer(KSMLTopic ksmlTopic, boolean isKey) { return switch (isKey ? ksmlTopic.keySerde() : ksmlTopic.valueSerde()) { case AVRO -> { - final var avroNotation = (MockAvroNotation) NotationLibrary.notation(MockAvroNotation.NOTATION_NAME); + final var avroNotation = (MockAvroNotation) NotationLibrary.notation(MockAvroNotation.NAME); var result = new KafkaAvroSerializer(avroNotation.mockSchemaRegistryClient()); result.configure(avroNotation.getSchemaRegistryConfigs(), isKey); yield result; @@ -228,7 +228,7 @@ private Deserializer getValueDeserializer(KSMLTopic kamlTopic) { private Deserializer getDeserializer(KSMLTopic kamlTopic, boolean isKey) { return switch (isKey ? kamlTopic.keySerde() : kamlTopic.valueSerde()) { case AVRO -> { - final var avroNotation = (MockAvroNotation) NotationLibrary.notation(MockAvroNotation.NOTATION_NAME); + final var avroNotation = (MockAvroNotation) NotationLibrary.notation(MockAvroNotation.NAME); final var result = new KafkaAvroDeserializer(avroNotation.mockSchemaRegistryClient()); result.configure(avroNotation.getSchemaRegistryConfigs(), isKey); yield result; From d425d70b5debd9eff1e36ff5608e0c4617fbe2fb Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Fri, 11 Oct 2024 11:23:46 +0200 Subject: [PATCH 05/55] Update to latest AVRO dependency, removing security alert --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3581e077..06dcdb6a 100644 --- a/pom.xml +++ b/pom.xml @@ -61,7 +61,7 @@ - 1.11.3 + 1.12.0 1.12.0 1.26.2 3.8.0 From 7e6863c46c5aeefd93b917f1902b351d7fac169f Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Tue, 15 Oct 2024 17:21:10 +0200 Subject: [PATCH 06/55] Improved configuration syntax + introduced named notation factories --- ksml-data-avro/NOTICE.txt | 2 +- .../java/io/axual/ksml/runner/KSMLRunner.java | 33 +++++++++-------- ...ngConfig.java => ErrorHandlingConfig.java} | 37 +++++++++---------- .../axual/ksml/runner/config/KSMLConfig.java | 6 +-- .../ksml/runner/config/KSMLRunnerConfig.java | 4 +- .../ksml/runner/config/NotationConfig.java | 3 -- .../runner/notation/NotationFactories.java | 4 +- .../io/axual/ksml/parser/UserTypeParser.java | 2 +- 8 files changed, 45 insertions(+), 46 deletions(-) rename ksml-runner/src/main/java/io/axual/ksml/runner/config/{KSMLErrorHandlingConfig.java => ErrorHandlingConfig.java} (73%) diff --git a/ksml-data-avro/NOTICE.txt b/ksml-data-avro/NOTICE.txt index 5c8b2038..7202a245 100644 --- a/ksml-data-avro/NOTICE.txt +++ b/ksml-data-avro/NOTICE.txt @@ -60,7 +60,7 @@ Lists of 72 third-party dependencies. (EPL 2.0) (GPL2 w/ CPE) Jakarta Annotations API (jakarta.annotation:jakarta.annotation-api:3.0.0 - https://projects.eclipse.org/projects/ee4j.ca) (Apache License 2.0) Jakarta Bean Validation API (jakarta.validation:jakarta.validation-api:3.0.2 - https://beanvalidation.org) (EPL-2.0) (GPL-2.0-with-classpath-exception) Jakarta RESTful WS API (jakarta.ws.rs:jakarta.ws.rs-api:3.1.0 - https://github.com/eclipse-ee4j/jaxrs-api) - (Apache-2.0) Apache Avro (org.apache.avro:avro:1.11.3 - https://avro.apache.org) + (Apache-2.0) Apache Avro (org.apache.avro:avro:1.12.0 - https://avro.apache.org) (Apache-2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.26.2 - https://commons.apache.org/proper/commons-compress/) (Apache-2.0) Apache Commons Lang (org.apache.commons:commons-lang3:3.14.0 - https://commons.apache.org/proper/commons-lang/) (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-clients:3.8.0 - https://kafka.apache.org) diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java index 2e836572..d7dead95 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java @@ -39,7 +39,7 @@ import io.axual.ksml.runner.backend.KafkaProducerRunner; import io.axual.ksml.runner.backend.KafkaStreamsRunner; import io.axual.ksml.runner.backend.Runner; -import io.axual.ksml.runner.config.KSMLErrorHandlingConfig; +import io.axual.ksml.runner.config.ErrorHandlingConfig; import io.axual.ksml.runner.config.KSMLRunnerConfig; import io.axual.ksml.runner.exception.ConfigException; import io.axual.ksml.runner.notation.NotationFactories; @@ -112,31 +112,34 @@ public static void main(String[] args) { // Set up all notation overrides from the KSML config for (final var notationEntry : ksmlConfig.notations().entrySet()) { - final var notation = notationEntry.getKey(); + final var notationStr = notationEntry.getKey() != null ? notationEntry.getKey() : "undefined"; final var notationConfig = notationEntry.getValue(); - if (notation != null && notationConfig != null) { - final var n = notationFactories.notations().get(notationConfig.type()); - if (n == null) { - throw FatalError.reportAndExit(new ConfigException("Notation type '" + notationConfig.type() + "' not found")); + final var factoryName = notationConfig != null ? notationConfig.type() : "unknown"; + if (notationConfig != null && factoryName != null) { + final var factory = notationFactories.notations().get(factoryName); + if (factory == null) { + throw FatalError.reportAndExit(new ConfigException("Unknown notation type: " + factoryName)); } - NotationLibrary.register(notation, n.create(notationConfig.config())); + NotationLibrary.register(notationStr, factory.create(notationConfig.config())); + } else { + log.warn("Notation configuration incomplete: notation=" + notationStr + ", type=" + factoryName); } } - // Ensure typical defaults are used - // WARNING: Defaults for notations will be deprecated in the future. Make sure you explicitly define - // notations that have multiple implementations in your ksml-runner.yaml. + // Ensure typical defaults are used for AVRO + // WARNING: Defaults for notations will be deprecated in the future. Make sure you explicitly configure + // notations with multiple implementations (like AVRO) in your ksml-runner.yaml. if (!NotationLibrary.exists(NotationFactories.AVRO)) { final var defaultAvro = notationFactories.confluentAvro(); NotationLibrary.register(NotationFactories.AVRO, defaultAvro.create(null)); - log.warn("No implementation specified for AVRO notation. If you plan to use AVRO, add the required implementation to the ksml-runner.yaml"); + log.warn("No implementation specified for AVRO notation. If you use AVRO in your KSML definition, add the required configuration to the ksml-runner.yaml"); } final var errorHandling = ksmlConfig.getErrorHandlingConfig(); if (errorHandling != null) { - ExecutionContext.INSTANCE.setConsumeHandler(getErrorHandler(errorHandling.getConsumerErrorHandlingConfig())); - ExecutionContext.INSTANCE.setProduceHandler(getErrorHandler(errorHandling.getProducerErrorHandlingConfig())); - ExecutionContext.INSTANCE.setProcessHandler(getErrorHandler(errorHandling.getProcessErrorHandlingConfig())); + ExecutionContext.INSTANCE.setConsumeHandler(getErrorHandler(errorHandling.consumerErrorHandlingConfig())); + ExecutionContext.INSTANCE.setProduceHandler(getErrorHandler(errorHandling.producerErrorHandlingConfig())); + ExecutionContext.INSTANCE.setProcessHandler(getErrorHandler(errorHandling.processErrorHandlingConfig())); } ExecutionContext.INSTANCE.serdeWrapper( serde -> new Serdes.WrapperSerde<>( @@ -371,7 +374,7 @@ private static KSMLRunnerConfig readConfiguration(File configFile) { } - private static ErrorHandler getErrorHandler(KSMLErrorHandlingConfig.ErrorHandlingConfig config) { + private static ErrorHandler getErrorHandler(ErrorHandlingConfig.ErrorTypeHandlingConfig config) { final var handlerType = switch (config.handler()) { case CONTINUE -> ErrorHandler.HandlerType.CONTINUE_ON_FAIL; case STOP -> ErrorHandler.HandlerType.STOP_ON_FAIL; diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/ErrorHandlingConfig.java similarity index 73% rename from ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java rename to ksml-runner/src/main/java/io/axual/ksml/runner/config/ErrorHandlingConfig.java index 79ab9f01..ea4815e2 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/ErrorHandlingConfig.java @@ -23,22 +23,19 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Builder; -import lombok.Getter; -import lombok.Setter; +import lombok.Data; import lombok.extern.jackson.Jacksonized; @JsonIgnoreProperties(ignoreUnknown = true) -@Builder -@Jacksonized -public class KSMLErrorHandlingConfig { - private ErrorHandlingConfig consume; - private ErrorHandlingConfig produce; - private ErrorHandlingConfig process; +@Data +public class ErrorHandlingConfig { + private ErrorTypeHandlingConfig consume; + private ErrorTypeHandlingConfig produce; + private ErrorTypeHandlingConfig process; - - public ErrorHandlingConfig getConsumerErrorHandlingConfig() { + public ErrorTypeHandlingConfig consumerErrorHandlingConfig() { if (consume == null) { - return getDefaultErrorHandlingConfig("ConsumeError"); + return defaultErrorTypeHandlingConfig("ConsumeError"); } if (consume.loggerName() == null) { consume.loggerName("ConsumeError"); @@ -46,9 +43,9 @@ public ErrorHandlingConfig getConsumerErrorHandlingConfig() { return consume; } - public ErrorHandlingConfig getProducerErrorHandlingConfig() { + public ErrorTypeHandlingConfig producerErrorHandlingConfig() { if (produce == null) { - return getDefaultErrorHandlingConfig("ProduceError"); + return defaultErrorTypeHandlingConfig("ProduceError"); } if (produce.loggerName() == null) { produce.loggerName("ProduceError"); @@ -56,9 +53,9 @@ public ErrorHandlingConfig getProducerErrorHandlingConfig() { return produce; } - public ErrorHandlingConfig getProcessErrorHandlingConfig() { + public ErrorTypeHandlingConfig processErrorHandlingConfig() { if (process == null) { - return getDefaultErrorHandlingConfig("ProcessError"); + return defaultErrorTypeHandlingConfig("ProcessError"); } if (process.loggerName() == null) { process.loggerName("ProcessError"); @@ -66,15 +63,15 @@ public ErrorHandlingConfig getProcessErrorHandlingConfig() { return process; } - private ErrorHandlingConfig getDefaultErrorHandlingConfig(String logger) { - var errorHandlingConfig = new ErrorHandlingConfig(); + private ErrorTypeHandlingConfig defaultErrorTypeHandlingConfig(String logger) { + var errorHandlingConfig = new ErrorTypeHandlingConfig(); errorHandlingConfig.loggerName(logger); return errorHandlingConfig; } - @Setter - @Getter - public static class ErrorHandlingConfig { + @JsonIgnoreProperties(ignoreUnknown = true) + @Data + public static class ErrorTypeHandlingConfig { private boolean log = true; private boolean logPayload = false; private String loggerName; diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java index cb29ea08..cdafbaac 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java @@ -69,7 +69,7 @@ public class KSMLConfig { private boolean enablePipelines = true; @JsonProperty("errorHandling") - private KSMLErrorHandlingConfig errorHandling; + private ErrorHandlingConfig errorHandling; @JsonProperty("notations") private Map notations; @JsonProperty("definitions") @@ -101,8 +101,8 @@ public ApplicationServerConfig getApplicationServerConfig() { return applicationServer; } - public KSMLErrorHandlingConfig getErrorHandlingConfig() { - if (errorHandling == null) return KSMLErrorHandlingConfig.builder().build(); + public ErrorHandlingConfig getErrorHandlingConfig() { + if (errorHandling == null) return new ErrorHandlingConfig(); return errorHandling; } diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java index f5bac5e5..78619580 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java @@ -68,7 +68,9 @@ public static class KafkaConfig { // Capture all other fields that Jackson do not match other members @JsonAnyGetter public Map kafkaConfig() { - return kafkaConfig; + final var result = new HashMap<>(kafkaConfig); + result.put("application.id", applicationId); + return result; } @JsonAnySetter diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/NotationConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/NotationConfig.java index 10ac3516..448f65be 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/NotationConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/NotationConfig.java @@ -20,12 +20,9 @@ * =========================LICENSE_END================================== */ -import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Builder; import lombok.Data; -import lombok.Getter; -import lombok.Setter; import lombok.extern.jackson.Jacksonized; import java.util.Map; diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/notation/NotationFactories.java b/ksml-runner/src/main/java/io/axual/ksml/runner/notation/NotationFactories.java index 50bab6f5..114b2f64 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/notation/NotationFactories.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/notation/NotationFactories.java @@ -46,8 +46,8 @@ public interface NotationFactory { // Notation constants public static final String AVRO = "avro"; - public static final String APICURIO_AVRO = "apicurioAvro"; - public static final String CONFLUENT_AVRO = "confluentAvro"; + public static final String APICURIO_AVRO = "apicurio_avro"; + public static final String CONFLUENT_AVRO = "confluent_avro"; public static final String BINARY = "binary"; public static final String CSV = "csv"; public static final String JSON = "json"; diff --git a/ksml/src/main/java/io/axual/ksml/parser/UserTypeParser.java b/ksml/src/main/java/io/axual/ksml/parser/UserTypeParser.java index 2d82ad69..45008b23 100644 --- a/ksml/src/main/java/io/axual/ksml/parser/UserTypeParser.java +++ b/ksml/src/main/java/io/axual/ksml/parser/UserTypeParser.java @@ -160,7 +160,7 @@ private static UserType parseTypeAndNotation(String composedType, String default // Notation without specific schema if (NotationLibrary.exists(datatype)) { - return new UserType(notation, NotationLibrary.notation(datatype).defaultType()); + return new UserType(datatype, NotationLibrary.notation(datatype).defaultType()); } throw new TopologyException("Unknown data type: " + datatype); From 7897833b45f21d411050a52a59a382f24f3d2088 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Tue, 15 Oct 2024 19:58:38 +0200 Subject: [PATCH 07/55] Small refinements for the DataObjectConverter --- .../io/axual/ksml/data/mapper/DataObjectConverter.java | 8 ++++++-- .../io/axual/ksml/generator/TopologyBuildContext.java | 7 +++---- .../java/io/axual/ksml/operation/ConvertKeyOperation.java | 8 +------- .../io/axual/ksml/operation/ConvertKeyValueOperation.java | 2 +- .../io/axual/ksml/operation/ConvertValueOperation.java | 3 +-- 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/ksml/src/main/java/io/axual/ksml/data/mapper/DataObjectConverter.java b/ksml/src/main/java/io/axual/ksml/data/mapper/DataObjectConverter.java index 9b60e792..69c0d391 100644 --- a/ksml/src/main/java/io/axual/ksml/data/mapper/DataObjectConverter.java +++ b/ksml/src/main/java/io/axual/ksml/data/mapper/DataObjectConverter.java @@ -31,9 +31,11 @@ import static io.axual.ksml.data.notation.UserType.DEFAULT_NOTATION; // This DataObjectConverter makes expected data types compatible with the actual data that was -// created. It does so by converting numbers to strings, and vice versa. It can convert complex +// created. It does so by converting numbers to strings, and vice versa. It converts complex // data objects like Enums, Lists and Structs too, recursively going through sub-elements if -// necessary. +// necessary. When no standard conversion is possible, the mechanism reverts to using the +// source or target notation's converter to respectively export or import the data type to +// the desired type. public class DataObjectConverter { private static final DataTypeSchemaMapper SCHEMA_MAPPER = new DataTypeFlattener(); @@ -77,6 +79,8 @@ private DataObject convert(String sourceNotation, DataObject value, UserType tar return convertTuple(targetTupleType, valueTuple); } + // When we reach this point, real data conversion needs to happen + // First step is to use the converters from the notations to convert the type into the desired target type var convertedValue = applyNotationConverters(sourceNotation, value, targetType); diff --git a/ksml/src/main/java/io/axual/ksml/generator/TopologyBuildContext.java b/ksml/src/main/java/io/axual/ksml/generator/TopologyBuildContext.java index a4fb0d51..c22de2ee 100644 --- a/ksml/src/main/java/io/axual/ksml/generator/TopologyBuildContext.java +++ b/ksml/src/main/java/io/axual/ksml/generator/TopologyBuildContext.java @@ -30,6 +30,7 @@ import io.axual.ksml.stream.*; import io.axual.ksml.user.UserFunction; import io.axual.ksml.user.UserTimestampExtractor; +import lombok.Getter; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.StreamsBuilder; @@ -52,6 +53,8 @@ public class TopologyBuildContext { private final StreamsBuilder builder; private final TopologyResources resources; private final PythonContext pythonContext; + @Getter + private final DataObjectConverter converter = new DataObjectConverter(); // All wrapped KStreams, KTables and KGlobalTables private final Map streamWrappersByName = new HashMap<>(); @@ -67,10 +70,6 @@ public String namespace() { return resources.namespace(); } - public DataObjectConverter getDataObjectConverter() { - return new DataObjectConverter(); - } - public Materialized> materialize(KeyValueStateStoreDefinition store) { resources.register(store.name(), store); return StoreUtil.materialize(store, getTopicName(store)).materialized(); diff --git a/ksml/src/main/java/io/axual/ksml/operation/ConvertKeyOperation.java b/ksml/src/main/java/io/axual/ksml/operation/ConvertKeyOperation.java index fb08d383..ff23899f 100644 --- a/ksml/src/main/java/io/axual/ksml/operation/ConvertKeyOperation.java +++ b/ksml/src/main/java/io/axual/ksml/operation/ConvertKeyOperation.java @@ -20,18 +20,12 @@ * =========================LICENSE_END================================== */ -import io.axual.ksml.data.mapper.NativeDataObjectMapper; import io.axual.ksml.data.notation.UserType; import io.axual.ksml.generator.TopologyBuildContext; import io.axual.ksml.stream.KStreamWrapper; import io.axual.ksml.stream.StreamWrapper; -import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.kstream.KeyValueMapper; import org.apache.kafka.streams.kstream.Named; -import org.apache.kafka.streams.kstream.Suppressed; -import org.apache.kafka.streams.kstream.TimeWindows; - -import java.time.Duration; public class ConvertKeyOperation extends BaseOperation { private final UserType targetKeyType; @@ -46,7 +40,7 @@ public StreamWrapper apply(KStreamWrapper input, TopologyBuildContext context) { final var k = input.keyType().flatten(); final var v = input.valueType(); final var kr = streamDataTypeOf(targetKeyType, true).flatten(); - final var mapper = context.getDataObjectConverter(); + final var mapper = context.converter(); // Set up the mapping function to convert the value final KeyValueMapper converter = (key, value) -> { diff --git a/ksml/src/main/java/io/axual/ksml/operation/ConvertKeyValueOperation.java b/ksml/src/main/java/io/axual/ksml/operation/ConvertKeyValueOperation.java index f1731084..cf205bc6 100644 --- a/ksml/src/main/java/io/axual/ksml/operation/ConvertKeyValueOperation.java +++ b/ksml/src/main/java/io/axual/ksml/operation/ConvertKeyValueOperation.java @@ -44,7 +44,7 @@ public StreamWrapper apply(KStreamWrapper input, TopologyBuildContext context) { final var v = input.valueType().flatten(); final var kr = streamDataTypeOf(targetKeyType, true).flatten(); final var vr = streamDataTypeOf(targetValueType, false).flatten(); - final var mapper = context.getDataObjectConverter(); + final var mapper = context.converter(); // Set up the mapping function to convert the key and value KeyValueMapper> converter = (key, value) -> { diff --git a/ksml/src/main/java/io/axual/ksml/operation/ConvertValueOperation.java b/ksml/src/main/java/io/axual/ksml/operation/ConvertValueOperation.java index c4d3d218..3e19be54 100644 --- a/ksml/src/main/java/io/axual/ksml/operation/ConvertValueOperation.java +++ b/ksml/src/main/java/io/axual/ksml/operation/ConvertValueOperation.java @@ -20,7 +20,6 @@ * =========================LICENSE_END================================== */ -import io.axual.ksml.data.mapper.NativeDataObjectMapper; import io.axual.ksml.data.notation.UserType; import io.axual.ksml.generator.TopologyBuildContext; import io.axual.ksml.stream.KStreamWrapper; @@ -41,7 +40,7 @@ public StreamWrapper apply(KStreamWrapper input, TopologyBuildContext context) { final var k = input.keyType(); final var v = input.valueType().flatten(); final var vr = streamDataTypeOf(targetValueType, false).flatten(); - final var mapper = context.getDataObjectConverter(); + final var mapper = context.converter(); // Set up the mapping function to convert the value ValueMapper converter = value -> { From c4637c07642daa09488634004fac1707af8427d9 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Fri, 18 Oct 2024 17:00:41 +0200 Subject: [PATCH 08/55] Update documentation and example runner definitions --- docs/types.md | 77 ++++++++++++++++--- examples/ksml-data-generator.yaml | 71 +++++++++++++++++- examples/ksml-runner.yaml | 119 ++++++++++++++++++++++-------- 3 files changed, 224 insertions(+), 43 deletions(-) diff --git a/docs/types.md b/docs/types.md index edc8a04a..d32a3e3f 100644 --- a/docs/types.md +++ b/docs/types.md @@ -3,19 +3,71 @@ ### Table of Contents 1. [Introduction](#introduction) -2. [Primitives](#primitives) -3. [Any](#any) -4. [Duration](#duration) -5. [Enum](#enum) -6. [List](#list) -7. [Struct](#struct) -8. [Tuple](#tuple) -9. [Windowed](#windowed) +2. [Notations](#notations) +3. [Primitives](#primitives) +4. [Any](#any) +5. [Duration](#duration) +6. [Enum](#enum) +7. [List](#list) +8. [Struct](#struct) +9. [Tuple](#tuple) +10. [Windowed](#windowed) ## Introduction -KSML supports a wide range of simple and complex types. -Types are very useful. +KSML supports a wide range of simple and complex types. In addition, each data type +can be read from Kafka, and written to Kafka, using _Notations_. More about Notations +in the following paragraph. The paragraphs after will dive more into the data +types that KSML supports. + +## Notations + +KSML has a number of simple and complex data types. Simple data types contain a +single value. For example, a `long` or a `string` type can hold only one value. +Complex data types contain multiple values. For example, a `list` or a `struct` +can contain zero or more other simple or complex values. For example, a `struct` +is comparable to a key/value map, where all keys are of type `string`. Its contents +could be represented as: + +```json +{ + "name": "Albert", + "lastName": "Einstein", + "dateOfBirth": "14-08-1879", + "profession": "Patent clerk", + "children": 3, + "isAlive": false +} +``` + +There are several ways in which structured objects are read from or written to +Kafka topics. For instance, using AVRO, JSON or XML. Each use the same `struct` +but translate it differently to a binary or string representation that can be +stored in Kafka. + +To facilitate these different ways of writing, KSML supports _Notations_. Each +notation can be seen as 'the format that data is written in' to Kafka. Below is +a list of all supported notations: + +| Notation | Link | Implementations | +|----------|--------------------------------------------------------------------------------------------------------------|-------------------------------| +| AVRO | [https://avro.apache.org](https://avro.apache.org) | apicurio_avro, confluent_avro | +| CSV | [https://en.wikipedia.org/wiki/Comma-separated_values](https://en.wikipedia.org/wiki/Comma-separated_values) | | +| JSON | [https://json.org](https://json.org) | | +| SOAP | [https://en.wikipedia.org/wiki/SOAP](https://en.wikipedia.org/wiki/SOAP) | | +| XML | [https://en.wikipedia.org/wiki/XML](https://en.wikipedia.org/wiki/XML) | | + +Notations are a natural extension to data types. They can be used as: +``` +notation:datatype +``` + +For example: +``` +avro:SensorData # AVRO with schema SensorData (loaded from SensorData.avsc) +json # Schemaless JSON +xml:PersonSchema # XML with schema PersonSchema (loaded from PersonSchema.xsd) +``` ## Primitives @@ -83,7 +135,10 @@ Examples: Structs are key-value maps, as usually found in AVRO or JSON messages. They are defined using: -```struct``` + +``` +struct +``` ## Tuple diff --git a/examples/ksml-data-generator.yaml b/examples/ksml-data-generator.yaml index 12f88908..a707f3f8 100644 --- a/examples/ksml-data-generator.yaml +++ b/examples/ksml-data-generator.yaml @@ -1,9 +1,74 @@ ksml: # The examples directory is mounted to /ksml in the docker compose definition - configDirectory: /ksml # When not set defaults to the working directory - schemaDirectory: /ksml # When not set defaults to the config directory - enableProducers: true + configDirectory: /ksml # When not set defaults to the working directory + schemaDirectory: /ksml # When not set defaults to the config directory + storageDirectory: /tmp # When not set defaults to the default JVM temp directory + + # This section defines if a REST endpoint is opened on the KSML runner, through which + # state stores and/or readiness / liveness probes can be accessed. + applicationServer: + enabled: false # Set to true to enable, or false to disable + host: 0.0.0.0 # IP address to bind the REST server to + port: 8080 # Port number to listen on + + # This section defines whether a Prometheus endpoint is opened to allow metric scraping. + prometheus: + enabled: true # Set to true to enable, or false to disable + host: 0.0.0.0 # IP address to bind the Prometheus agent server to + port: 8080 # Port number to listen on + + # This section enables error handling or error ignoring for certain types of errors. + errorHandling: + consume: # Error handling definitions for consume errors + log: true # Log errors true/false + logPayload: true # Upon error, should the payload of the message be dumped to the log file. + loggerName: ConsumeError # Definition of the error logger name. + handler: stopOnFail # How to proceed after encountering the error. Either continueOnFail or stopOnFail. + process: + log: true # Log errors true/false + logPayload: true # Upon error, should the payload of the message be dumped to the log file. + loggerName: ProcessError # Definition of the error logger name. + handler: continueOnFail # How to proceed after encountering the error. Either continueOnFail or stopOnFail. + produce: + log: true # Log errors true/false + logPayload: true # Upon error, should the payload of the message be dumped to the log file. + loggerName: ProduceError # Definition of the error logger name. + handler: continueOnFail # How to proceed after encountering the error. Either continueOnFail or stopOnFail. + + # This section tells KSML with which serializers / deserializers to handle certain notation types. + notations: + avro: # Definition for "avro" notation + type: confluent_avro # For AVRO there are two implementations: apicurio_avro and confluent_avro + ## Below this line, specify properties to be passed into Confluent's KafkaAvroSerializer and KafkaAvroDeserializer + config: + schema.registry.url: https://url.to.schena.registry.org + normalize.schemas: true + schema.registry.ssl.protocol: TLSv1.3 + schema.registry.ssl.enabled.protocols: TLSv1.3,TLSv1.2 + schema.registry.ssl.endpoint.identification.algorithm: "" + schema.registry.ssl.keystore.type: JKS + schema.registry.ssl.truststore.type: JKS + schema.registry.ssl.key.password: password + schema.registry.ssl.keystore.password: password + schema.registry.ssl.keystore.location: /path/to/keystore.jks + schema.registry.ssl.truststore.password: password + schema.registry.ssl.truststore.location: /path/to/truststore.jks + auto.register.schemas: false + # Definition for "other_avro" notation. With this config, you can use a type like otherAvro:SchenaName in your + # KSML definition, which then uses the Apicurio implementation for AVRO. + otherAvro: + type: apicurio_avro # For AVRO there are two implementations: apicurio_avro and confluent_avro + ## Below this line, specify properties to be passed into Apicurio's AvroKafkaSerializer and AvroKafkaDeserializer + config: + apicurio.registry.url: https://sr2 + apicurio.registry.schema-resolver: CustomResolver + + enableProducers: true # Set to true to allow producer definitions to be parsed in the KSML definitions and be executed. + enablePipelines: false # Set to true to allow pipeline definitions to be parsed in the KSML definitions and be executed. + + # Section where you specify which KSML definitions to load, parse and execute. definitions: + # Format is : generate_alert_setting: 00-example-generate-alertsettings.yaml generate_sensor_data: 00-example-generate-sensordata.yaml diff --git a/examples/ksml-runner.yaml b/examples/ksml-runner.yaml index 4db219f5..a88579a6 100644 --- a/examples/ksml-runner.yaml +++ b/examples/ksml-runner.yaml @@ -1,10 +1,72 @@ ksml: - # The examples directory is mounted to /ksml in the run script -# configDirectory: /ksml # When not set defaults to the working directory -# schemaDirectory: /ksml # When not set defaults to the config directory -# storageDirectory: /tmp # When not set defaults to the default JVM temp directory + # The examples directory is mounted to /ksml in the Docker container + configDirectory: . # When not set defaults to the working directory + schemaDirectory: . # When not set defaults to the config directory + storageDirectory: /tmp # When not set defaults to the default JVM temp directory + + # This section defines if a REST endpoint is opened on the KSML runner, through which + # state stores and/or readiness / liveness probes can be accessed. + applicationServer: + enabled: false # Set to true to enable, or false to disable + host: 0.0.0.0 # IP address to bind the REST server to + port: 8080 # Port number to listen on + + # This section defines whether a Prometheus endpoint is opened to allow metric scraping. prometheus: - enablePipelines: true + enabled: true # Set to true to enable, or false to disable + host: 0.0.0.0 # IP address to bind the Prometheus agent server to + port: 8080 # Port number to listen on + + # This section enables error handling or error ignoring for certain types of errors. + errorHandling: + consume: # Error handling definitions for consume errors + log: true # Log errors true/false + logPayload: true # Upon error, should the payload of the message be dumped to the log file. + loggerName: ConsumeError # Definition of the error logger name. + handler: stopOnFail # How to proceed after encountering the error. Either continueOnFail or stopOnFail. + process: + log: true # Log errors true/false + logPayload: true # Upon error, should the payload of the message be dumped to the log file. + loggerName: ProcessError # Definition of the error logger name. + handler: continueOnFail # How to proceed after encountering the error. Either continueOnFail or stopOnFail. + produce: + log: true # Log errors true/false + logPayload: true # Upon error, should the payload of the message be dumped to the log file. + loggerName: ProduceError # Definition of the error logger name. + handler: continueOnFail # How to proceed after encountering the error. Either continueOnFail or stopOnFail. + + # This section tells KSML with which serializers / deserializers to handle certain notation types. + notations: + avro: # Definition for "avro" notation + type: confluent_avro # For AVRO there are two implementations: apicurio_avro and confluent_avro + ## Below this line, specify properties to be passed into Confluent's KafkaAvroSerializer and KafkaAvroDeserializer + config: + schema.registry.url: https://url.to.schena.registry.org + normalize.schemas: true + schema.registry.ssl.protocol: TLSv1.3 + schema.registry.ssl.enabled.protocols: TLSv1.3,TLSv1.2 + schema.registry.ssl.endpoint.identification.algorithm: "" + schema.registry.ssl.keystore.type: JKS + schema.registry.ssl.truststore.type: JKS + schema.registry.ssl.key.password: password + schema.registry.ssl.keystore.password: password + schema.registry.ssl.keystore.location: /path/to/keystore.jks + schema.registry.ssl.truststore.password: password + schema.registry.ssl.truststore.location: /path/to/truststore.jks + auto.register.schemas: false + # Definition for "other_avro" notation. With this config, you can use a type like otherAvro:SchenaName in your + # KSML definition, which then uses the Apicurio implementation for AVRO. + otherAvro: + type: apicurio_avro # For AVRO there are two implementations: apicurio_avro and confluent_avro + ## Below this line, specify properties to be passed into Apicurio's AvroKafkaSerializer and AvroKafkaDeserializer + config: + apicurio.registry.url: https://sr2 + apicurio.registry.schema-resolver: CustomResolver + + enableProducers: true # Set to true to allow producer definitions to be parsed in the KSML definitions and be executed. + enablePipelines: true # Set to true to allow pipeline definitions to be parsed in the KSML definitions and be executed. + + # Section where you specify which KSML definitions to load, parse and execute. definitions: # Format is : inspect: 01-example-inspect.yaml @@ -30,38 +92,37 @@ ksml: # These examples are intended to run from a inside a container on the same network kafka: application.id: io.ksml.example.streaming -# bootstrap.servers: localhost:9092 -# schema.registry.url: http://localhost:8081 bootstrap.servers: broker:9093 - schema.registry.url: http://schema_registry:8081 + schema.registry.url: http://localhost:8081 security.protocol: PLAINTEXT auto.offset.reset: earliest acks: all -# These are Kafka SSL configuration properties. Check the documentation at1 -# Check the documentation at https://kafka.apache.org/documentation/#producerconfigs for more properties - -# security.protocol: SSL -# ssl.protocol: TLSv1.3 -# ssl.enabled.protocols: TLSv1.3,TLSv1.2 -# ssl.endpoint.identification.algorithm: "" -# ssl.keystore.type: JKS -# ssl.truststore.type: JKS -# ssl.key.password: xxx -# ssl.keystore.password: xxx -# ssl.keystore.location: /path/to/ksml.keystore.jks -# ssl.truststore.password: xxx -# ssl.truststore.location: /path/to/ksml.truststore.jks + # These are Kafka SSL configuration properties. Check the documentation at1 + # Check the documentation at https://kafka.apache.org/documentation/#producerconfigs for more properties -# Use these configuration properties when connecting to a cluster using the Axual naming patterns. -# These patterns are resolved into the actual name used on Kafka using the values in this configuration map -# and the topic names specified in the definition YAML files + # security.protocol: SSL + # ssl.protocol: TLSv1.3 + # ssl.enabled.protocols: TLSv1.3,TLSv1.2 + # ssl.endpoint.identification.algorithm: "" + # ssl.keystore.type: JKS + # ssl.truststore.type: JKS + # ssl.key.password: xxx + # ssl.keystore.password: xxx + # ssl.keystore.location: /path/to/ksml.keystore.jks + # ssl.truststore.password: xxx + # ssl.truststore.location: /path/to/ksml.truststore.jks + # Use these configuration properties when connecting to a cluster using the Axual naming patterns. + # These patterns are resolved into the actual name used on Kafka using the values in this configuration map + # and the topic names specified in the definition YAML files + # tenant: "ksmldemo" + # instance: "dta" + # environment: "dev" -# tenant: "ksmldemo" -# instance: "dta" -# environment: "dev" + # Results in Kafka topic ksmldemo-dta-dev- # topic.pattern: "{tenant}-{instance}-{environment}-{topic}" -# # Results in Kafka topic ksmldemo-dta-dev- + # Results in Kafka topic ksmldemo-dta-dev- # group.id.pattern: "{tenant}-{instance}-{environment}-{group.id}" + # Results in Kafka topic ksmldemo-dta-dev- # transactional.id.pattern: "{tenant}-{instance}-{environment}-{transactional.id}" From 6a3f069cec8a1c856da03965b3a257f96287fede Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Fri, 18 Oct 2024 17:22:11 +0200 Subject: [PATCH 09/55] Update apicurio serde version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2bf34f8b..cd807def 100644 --- a/pom.xml +++ b/pom.xml @@ -65,7 +65,7 @@ 1.12.0 1.26.2 3.8.0 - 3.0.0-SNAPSHOT + 3.0.0.M4 7.6.1 23.1.2 23.1.2 From 1e3d9f3677f7e8c5a2cfb0f8c19a8a59377156eb Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Tue, 22 Oct 2024 07:32:57 +0200 Subject: [PATCH 10/55] Updated library for Apicurio serdes --- ksml-data-avro/NOTICE.txt | 32 +++++++++++++++++--------------- ksml-data-avro/pom.xml | 2 +- pom.xml | 4 ++-- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/ksml-data-avro/NOTICE.txt b/ksml-data-avro/NOTICE.txt index 98e62ec7..dbbe7319 100644 --- a/ksml-data-avro/NOTICE.txt +++ b/ksml-data-avro/NOTICE.txt @@ -1,5 +1,5 @@ -Lists of 72 third-party dependencies. +Lists of 74 third-party dependencies. (MIT License) minimal-json (com.eclipsesource.minimal-json:minimal-json:0.9.5 - https://github.com/ralfstx/minimal-json) (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.17.1 - https://github.com/FasterXML/jackson) (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.17.1 - https://github.com/FasterXML/jackson-core) @@ -14,17 +14,19 @@ Lists of 72 third-party dependencies. (The Apache Software License, Version 2.0) Guava ListenableFuture only (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava - https://github.com/google/guava/listenablefuture) (Apache License, Version 2.0) J2ObjC Annotations (com.google.j2objc:j2objc-annotations:3.0.0 - https://github.com/google/j2objc/) (Go License) RE2/J (com.google.re2j:re2j:1.6 - http://github.com/google/re2j) - (MIT License) Microsoft Kiota-Java Abstractions (com.microsoft.kiota:microsoft-kiota-abstractions:1.1.14 - https://github.com/microsoft/kiota-java) - (MIT License) Microsoft Kiota-Java Serialization-Form (com.microsoft.kiota:microsoft-kiota-serialization-form:1.1.14 - https://github.com/microsoft/kiota-java) - (MIT License) Microsoft Kiota-Java Serialization-Multipart (com.microsoft.kiota:microsoft-kiota-serialization-multipart:1.1.14 - https://github.com/microsoft/kiota-java) - (MIT License) Microsoft Kiota-Java Serialization-Text (com.microsoft.kiota:microsoft-kiota-serialization-text:1.1.14 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Abstractions (com.microsoft.kiota:microsoft-kiota-abstractions:1.7.0 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Form (com.microsoft.kiota:microsoft-kiota-serialization-form:1.7.0 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Multipart (com.microsoft.kiota:microsoft-kiota-serialization-multipart:1.7.0 - https://github.com/microsoft/kiota-java) + (MIT License) Microsoft Kiota-Java Serialization-Text (com.microsoft.kiota:microsoft-kiota-serialization-text:1.7.0 - https://github.com/microsoft/kiota-java) (Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.17.0 - https://commons.apache.org/proper/commons-codec/) (Apache-2.0) Apache Commons IO (commons-io:commons-io:2.16.1 - https://commons.apache.org/proper/commons-io/) - (Apache License Version 2.0) apicurio-registry-common (io.apicurio:apicurio-registry-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-common/) - (Apache License Version 2.0) apicurio-registry-java-sdk (io.apicurio:apicurio-registry-java-sdk:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-java-sdk/) - (Apache License Version 2.0) apicurio-registry-schema-resolver (io.apicurio:apicurio-registry-schema-resolver:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-schema-resolver/) - (Apache License Version 2.0) apicurio-registry-serde-common (io.apicurio:apicurio-registry-serde-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serde-common/) - (Apache License Version 2.0) apicurio-registry-serdes-avro-serde (io.apicurio:apicurio-registry-serdes-avro-serde:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serdes-avro-serde/) + (Apache License Version 2.0) apicurio-registry-avro-serde-kafka (io.apicurio:apicurio-registry-avro-serde-kafka:3.0.2 - https://www.apicur.io/apicurio-registry-avro-serde-kafka/) + (Apache License Version 2.0) apicurio-registry-common (io.apicurio:apicurio-registry-common:3.0.2 - https://www.apicur.io/apicurio-registry-common/) + (Apache License Version 2.0) apicurio-registry-java-sdk (io.apicurio:apicurio-registry-java-sdk:3.0.2 - https://www.apicur.io/apicurio-registry-java-sdk/) + (Apache License Version 2.0) apicurio-registry-schema-resolver (io.apicurio:apicurio-registry-schema-resolver:3.0.2 - https://www.apicur.io/apicurio-registry-schema-resolver/) + (Apache License Version 2.0) apicurio-registry-serde-common (io.apicurio:apicurio-registry-serde-common:3.0.2 - https://www.apicur.io/apicurio-registry-serde-common/) + (Apache License Version 2.0) apicurio-registry-serde-avro-common (io.apicurio:apicurio-registry-serde-common-avro:3.0.2 - https://www.apicur.io/apicurio-registry-serde-common-avro/) + (Apache License Version 2.0) apicurio-registry-serde-kafka-common (io.apicurio:apicurio-registry-serde-kafka-common:3.0.2 - https://www.apicur.io/apicurio-registry-serde-kafka-common/) (Apache 2.0) KSML Data Library (io.axual.ksml:ksml-data:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-data) (Apache License 2.0) utils (io.confluent:common-utils:7.6.1 - https://confluent.io/common-utils) (Apache License 2.0) kafka-avro-serializer (io.confluent:kafka-avro-serializer:7.6.1 - http://confluent.io/kafka-avro-serializer) @@ -32,9 +34,9 @@ Lists of 72 third-party dependencies. (Apache License 2.0) kafka-schema-serializer (io.confluent:kafka-schema-serializer:7.6.1 - http://confluent.io/kafka-schema-serializer) (The Apache Software License, Version 2.0) Log Redactor (io.confluent:logredactor:1.0.12 - https://github.com/confluentinc/logredactor) (The Apache Software License, Version 2.0) Log Redactor Metrics (io.confluent:logredactor-metrics:1.0.12 - https://github.com/confluentinc/logredactor) - (Apache License, Version 2.0) Standard Uri Template (io.github.std-uritemplate:std-uritemplate:0.0.59 - https://std-uritemplate.github.io/) - (Apache-2.0) Kiota - Http - Vert.X (io.kiota:kiota-http-vertx:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-http-vertx) - (Apache-2.0) Kiota - Serialization - Jackson (io.kiota:kiota-serialization-jackson:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-serialization-jackson-parent/kiota-serialization-jackson) + (Apache License, Version 2.0) Standard Uri Template (io.github.std-uritemplate:std-uritemplate:2.0.0 - https://std-uritemplate.github.io/) + (Apache-2.0) Kiota - Http - Vert.X (io.kiota:kiota-http-vertx:0.0.18 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-http-vertx) + (Apache-2.0) Kiota - Serialization - Jackson (io.kiota:kiota-serialization-jackson:0.0.18 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-serialization-jackson-parent/kiota-serialization-jackson) (Apache License, Version 2.0) Netty/Buffer (io.netty:netty-buffer:4.1.108.Final - https://netty.io/netty-buffer/) (Apache License, Version 2.0) Netty/Codec (io.netty:netty-codec:4.1.108.Final - https://netty.io/netty-codec/) (Apache License, Version 2.0) Netty/Codec/DNS (io.netty:netty-codec-dns:4.1.108.Final - https://netty.io/netty-codec-dns/) @@ -48,8 +50,8 @@ Lists of 72 third-party dependencies. (Apache License, Version 2.0) Netty/Resolver/DNS (io.netty:netty-resolver-dns:4.1.108.Final - https://netty.io/netty-resolver-dns/) (Apache License, Version 2.0) Netty/Transport (io.netty:netty-transport:4.1.108.Final - https://netty.io/netty-transport/) (Apache License, Version 2.0) Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.108.Final - https://netty.io/netty-transport-native-unix-common/) - (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-api:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) - (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-context:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) + (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-api:1.42.1 - https://github.com/open-telemetry/opentelemetry-java) + (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-context:1.42.1 - https://github.com/open-telemetry/opentelemetry-java) (Apache License 2.0) swagger-annotations (io.swagger.core.v3:swagger-annotations:2.1.10 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations) (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-common (io.vertx:vertx-auth-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-common) (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-oauth2 (io.vertx:vertx-auth-oauth2:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-oauth2) diff --git a/ksml-data-avro/pom.xml b/ksml-data-avro/pom.xml index 1b74d8b0..4cb81910 100644 --- a/ksml-data-avro/pom.xml +++ b/ksml-data-avro/pom.xml @@ -45,7 +45,7 @@ io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-avro-serde-kafka io.confluent diff --git a/pom.xml b/pom.xml index cd807def..78a6b0a0 100644 --- a/pom.xml +++ b/pom.xml @@ -65,7 +65,7 @@ 1.12.0 1.26.2 3.8.0 - 3.0.0.M4 + 3.0.2 7.6.1 23.1.2 23.1.2 @@ -231,7 +231,7 @@ io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-avro-serde-kafka ${apicurio.avro.version} From f54045bdb9c03fe0c5de357a47654590fba7342e Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 30 Oct 2024 18:29:50 +0100 Subject: [PATCH 11/55] Processed Richard's review comments for MR --- examples/ksml-data-generator.yaml | 2 +- examples/ksml-runner.yaml | 4 ++-- .../java/io/axual/ksml/runner/config/KSMLRunnerConfig.java | 6 ------ .../java/io/axual/ksml/runner/config/NotationConfig.java | 5 +---- 4 files changed, 4 insertions(+), 13 deletions(-) diff --git a/examples/ksml-data-generator.yaml b/examples/ksml-data-generator.yaml index a707f3f8..76270c86 100644 --- a/examples/ksml-data-generator.yaml +++ b/examples/ksml-data-generator.yaml @@ -15,7 +15,7 @@ ksml: prometheus: enabled: true # Set to true to enable, or false to disable host: 0.0.0.0 # IP address to bind the Prometheus agent server to - port: 8080 # Port number to listen on + port: 9999 # Port number to listen on # This section enables error handling or error ignoring for certain types of errors. errorHandling: diff --git a/examples/ksml-runner.yaml b/examples/ksml-runner.yaml index a88579a6..dc23ae90 100644 --- a/examples/ksml-runner.yaml +++ b/examples/ksml-runner.yaml @@ -15,7 +15,7 @@ ksml: prometheus: enabled: true # Set to true to enable, or false to disable host: 0.0.0.0 # IP address to bind the Prometheus agent server to - port: 8080 # Port number to listen on + port: 9999 # Port number to listen on # This section enables error handling or error ignoring for certain types of errors. errorHandling: @@ -93,7 +93,7 @@ ksml: kafka: application.id: io.ksml.example.streaming bootstrap.servers: broker:9093 - schema.registry.url: http://localhost:8081 + schema.registry.url: http://schema_registry:8081 security.protocol: PLAINTEXT auto.offset.reset: earliest acks: all diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java index 78619580..df8c4292 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java @@ -46,12 +46,6 @@ public class KSMLRunnerConfig { @JsonProperty("kafka") private KafkaConfig kafka; -// public Map kafkaConfig() { -// var newConfig = new HashMap<>(kafka.kafkaConfig()); -// newConfig.put("application.id", kafka.applicationId); -// return newConfig; -// } - public String applicationId() { return kafka.applicationId; } diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/NotationConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/NotationConfig.java index 448f65be..ef2d040e 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/NotationConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/NotationConfig.java @@ -30,8 +30,5 @@ @JsonIgnoreProperties(ignoreUnknown = true) @Builder @Jacksonized -@Data -public class NotationConfig { - private String type; - private Map config; +public record NotationConfig(String type, Map config) { } From 63b3bd1a6fdd6cc9af91f7a551040f351371ce2d Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 30 Oct 2024 18:35:53 +0100 Subject: [PATCH 12/55] Move lombok config to parent project level, remove submodule definitions --- ksml-data-csv/src/main/java/io/axual/ksml/data/lombok.config | 3 --- ksml-data-soap/src/main/java/io/axual/ksml/data/lombok.config | 3 --- ksml-data-xml/src/main/java/io/axual/ksml/data/lombok.config | 3 --- ksml-data/src/main/java/io/axual/ksml/data/lombok.config | 3 --- .../src/main/java/io/axual/ksml/client/lombok.config | 3 --- ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config | 3 --- ksml/src/main/java/lombok.config | 3 --- .../java/io/axual/ksml/data/lombok.config => lombok.config | 0 8 files changed, 21 deletions(-) delete mode 100644 ksml-data-csv/src/main/java/io/axual/ksml/data/lombok.config delete mode 100644 ksml-data-soap/src/main/java/io/axual/ksml/data/lombok.config delete mode 100644 ksml-data-xml/src/main/java/io/axual/ksml/data/lombok.config delete mode 100644 ksml-data/src/main/java/io/axual/ksml/data/lombok.config delete mode 100644 ksml-kafka-clients/src/main/java/io/axual/ksml/client/lombok.config delete mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config delete mode 100644 ksml/src/main/java/lombok.config rename ksml-data-avro/src/main/java/io/axual/ksml/data/lombok.config => lombok.config (100%) diff --git a/ksml-data-csv/src/main/java/io/axual/ksml/data/lombok.config b/ksml-data-csv/src/main/java/io/axual/ksml/data/lombok.config deleted file mode 100644 index 881a8aa5..00000000 --- a/ksml-data-csv/src/main/java/io/axual/ksml/data/lombok.config +++ /dev/null @@ -1,3 +0,0 @@ -lombok.accessors.fluent=true -lombok.accessors.chain=false -lombok.equalsAndHashCode.callSuper=call \ No newline at end of file diff --git a/ksml-data-soap/src/main/java/io/axual/ksml/data/lombok.config b/ksml-data-soap/src/main/java/io/axual/ksml/data/lombok.config deleted file mode 100644 index 881a8aa5..00000000 --- a/ksml-data-soap/src/main/java/io/axual/ksml/data/lombok.config +++ /dev/null @@ -1,3 +0,0 @@ -lombok.accessors.fluent=true -lombok.accessors.chain=false -lombok.equalsAndHashCode.callSuper=call \ No newline at end of file diff --git a/ksml-data-xml/src/main/java/io/axual/ksml/data/lombok.config b/ksml-data-xml/src/main/java/io/axual/ksml/data/lombok.config deleted file mode 100644 index 881a8aa5..00000000 --- a/ksml-data-xml/src/main/java/io/axual/ksml/data/lombok.config +++ /dev/null @@ -1,3 +0,0 @@ -lombok.accessors.fluent=true -lombok.accessors.chain=false -lombok.equalsAndHashCode.callSuper=call \ No newline at end of file diff --git a/ksml-data/src/main/java/io/axual/ksml/data/lombok.config b/ksml-data/src/main/java/io/axual/ksml/data/lombok.config deleted file mode 100644 index 881a8aa5..00000000 --- a/ksml-data/src/main/java/io/axual/ksml/data/lombok.config +++ /dev/null @@ -1,3 +0,0 @@ -lombok.accessors.fluent=true -lombok.accessors.chain=false -lombok.equalsAndHashCode.callSuper=call \ No newline at end of file diff --git a/ksml-kafka-clients/src/main/java/io/axual/ksml/client/lombok.config b/ksml-kafka-clients/src/main/java/io/axual/ksml/client/lombok.config deleted file mode 100644 index 881a8aa5..00000000 --- a/ksml-kafka-clients/src/main/java/io/axual/ksml/client/lombok.config +++ /dev/null @@ -1,3 +0,0 @@ -lombok.accessors.fluent=true -lombok.accessors.chain=false -lombok.equalsAndHashCode.callSuper=call \ No newline at end of file diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config b/ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config deleted file mode 100644 index 881a8aa5..00000000 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config +++ /dev/null @@ -1,3 +0,0 @@ -lombok.accessors.fluent=true -lombok.accessors.chain=false -lombok.equalsAndHashCode.callSuper=call \ No newline at end of file diff --git a/ksml/src/main/java/lombok.config b/ksml/src/main/java/lombok.config deleted file mode 100644 index 881a8aa5..00000000 --- a/ksml/src/main/java/lombok.config +++ /dev/null @@ -1,3 +0,0 @@ -lombok.accessors.fluent=true -lombok.accessors.chain=false -lombok.equalsAndHashCode.callSuper=call \ No newline at end of file diff --git a/ksml-data-avro/src/main/java/io/axual/ksml/data/lombok.config b/lombok.config similarity index 100% rename from ksml-data-avro/src/main/java/io/axual/ksml/data/lombok.config rename to lombok.config From a098f8ee07f59d0056b8e584a9f00ec4c6adf422 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 30 Oct 2024 23:00:54 +0100 Subject: [PATCH 13/55] Improve toString() function for DataObjects * DataObject interface extended with Printer enum, used to consistently print data object schema and/or value * Updated all DataObject implementing classes to implement the new toString() method * Make calls to toString() explicitly use a Printer type --- .../data/notation/csv/CsvSchemaMapper.java | 10 ++++- .../notation/xml/XmlDataObjectMapper.java | 2 +- .../data/notation/xml/XmlSchemaMapper.java | 11 ++++- .../io/axual/ksml/data/object/DataBytes.java | 16 +++---- .../io/axual/ksml/data/object/DataEnum.java | 5 --- .../io/axual/ksml/data/object/DataList.java | 11 ++++- .../io/axual/ksml/data/object/DataObject.java | 23 ++++++++++ .../axual/ksml/data/object/DataPrimitive.java | 10 ++++- .../io/axual/ksml/data/object/DataStruct.java | 42 ++++++++++--------- .../io/axual/ksml/data/object/DataTuple.java | 8 +++- .../io/axual/ksml/data/object/DataUnion.java | 4 +- .../io/axual/ksml/data/util/ValuePrinter.java | 30 +++++++++++++ .../java/io/axual/ksml/runner/KSMLRunner.java | 26 ++++++------ .../runner/backend/KafkaStreamsRunner.java | 4 +- .../config/KSMLErrorHandlingConfig.java | 15 ++++--- .../ksml/runner/config/KSMLRunnerConfig.java | 4 +- .../runner/prometheus/PrometheusExport.java | 2 +- .../runner/config/KSMLRunnerConfigTest.java | 2 +- .../java/io/axual/ksml/user/UserFunction.java | 5 +-- 19 files changed, 158 insertions(+), 72 deletions(-) create mode 100644 ksml-data/src/main/java/io/axual/ksml/data/util/ValuePrinter.java diff --git a/ksml-data-csv/src/main/java/io/axual/ksml/data/notation/csv/CsvSchemaMapper.java b/ksml-data-csv/src/main/java/io/axual/ksml/data/notation/csv/CsvSchemaMapper.java index 6368072b..5b1fd669 100644 --- a/ksml-data-csv/src/main/java/io/axual/ksml/data/notation/csv/CsvSchemaMapper.java +++ b/ksml-data-csv/src/main/java/io/axual/ksml/data/notation/csv/CsvSchemaMapper.java @@ -22,6 +22,7 @@ import io.axual.ksml.data.mapper.DataSchemaMapper; import io.axual.ksml.data.object.DataList; +import io.axual.ksml.data.object.DataObject; import io.axual.ksml.data.schema.DataField; import io.axual.ksml.data.schema.DataSchema; import io.axual.ksml.data.schema.DataValue; @@ -48,7 +49,14 @@ public DataSchema toDataSchema(String namespace, String name, String value) { private DataSchema toDataSchema(String namespace, String name, DataList fieldNames) { List fields = new ArrayList<>(); for (var fieldName : fieldNames) { - fields.add(new DataField(fieldName.toString(), DataSchema.create(DataSchema.Type.STRING), fieldName.toString(), NO_INDEX, true, false, new DataValue(""))); + fields.add(new DataField( + fieldName.toString(DataObject.Printer.INTERNAL), + DataSchema.create(DataSchema.Type.STRING), + fieldName.toString(DataObject.Printer.INTERNAL), + NO_INDEX, + true, + false, + new DataValue(""))); } return new StructSchema(namespace, name, "CSV schema", fields); } diff --git a/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlDataObjectMapper.java b/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlDataObjectMapper.java index c31add53..c7f5bcf7 100644 --- a/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlDataObjectMapper.java +++ b/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlDataObjectMapper.java @@ -230,7 +230,7 @@ private void elementFromDataObject(ElementCreator elementCreator, Element elemen return; } if (value != null) { - element.setTextContent(value.toString()); + element.setTextContent(value.toString(DataObject.Printer.INTERNAL)); } } } diff --git a/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlSchemaMapper.java b/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlSchemaMapper.java index 684744bb..2191cf12 100644 --- a/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlSchemaMapper.java +++ b/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlSchemaMapper.java @@ -170,7 +170,7 @@ private DataField parseField(DataStruct fieldStruct) { var fieldType = fieldStruct.get(ATTRIBUTES_ELEMENT_NAME) instanceof DataStruct attributeStruct ? attributeStruct.get(TYPE_NAME) : null; if (fieldType instanceof DataString fieldTypeString) { var type = fieldTypeString.value().contains(":") ? fieldTypeString.value().substring(fieldTypeString.value().indexOf(":") + 1) : fieldTypeString.value(); - return simpleField(fieldName.toString(), type); + return simpleField(fieldName.toString(DataObject.Printer.INTERNAL), type); } else { // Field type is not specified, so dig down into the elements below to find out the type var complexTypeElement = fieldStruct.get(COMPLEX_TYPE_NAME); @@ -178,7 +178,14 @@ private DataField parseField(DataStruct fieldStruct) { var sequenceElement = complexTypeStruct.get(SEQUENCE_NAME); if (sequenceElement instanceof DataStruct sequenceStruct) { var fields = parseFields(sequenceStruct); - return new DataField(fieldName.toString(), new StructSchema(null, fieldName.toString(), "Converted from XSD", fields), null); + return new DataField( + fieldName.toString(DataObject.Printer.INTERNAL), + new StructSchema( + null, + fieldName.toString(DataObject.Printer.INTERNAL), + "Converted from XSD", + fields), + null); } } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/object/DataBytes.java b/ksml-data/src/main/java/io/axual/ksml/data/object/DataBytes.java index 0cba562a..af78b06c 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/object/DataBytes.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/object/DataBytes.java @@ -37,15 +37,15 @@ public DataBytes(byte[] value) { } @Override - public String toString() { - StringBuilder builder = new StringBuilder(type().toString()).append(": "); - if (value() == null) return builder.append("null").toString(); - builder.append("["); + public String toString(Printer printer) { + final var sb = new StringBuilder(printer.schemaString(this)); + if (value() == null) return sb.append("null").toString(); + sb.append("["); for (int index = 0; index < value().length; index++) { - if (index > 0) builder.append(", "); - builder.append(String.format("%02X ", value()[index])); + if (index > 0) sb.append(", "); + sb.append(String.format("%02X ", value()[index])); } - builder.append("]"); - return builder.toString(); + sb.append("]"); + return sb.toString(); } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/object/DataEnum.java b/ksml-data/src/main/java/io/axual/ksml/data/object/DataEnum.java index 1753949b..34a6ebf0 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/object/DataEnum.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/object/DataEnum.java @@ -38,9 +38,4 @@ private boolean validateValue(String value) { } return false; } - - @Override - public String toString() { - return type().toString() + ": " + super.toString(); - } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/object/DataList.java b/ksml-data/src/main/java/io/axual/ksml/data/object/DataList.java index feae82e6..28b678bf 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/object/DataList.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/object/DataList.java @@ -20,6 +20,7 @@ * =========================LICENSE_END================================== */ +import io.axual.ksml.data.exception.DataException; import io.axual.ksml.data.exception.ExecutionException; import io.axual.ksml.data.type.DataType; import io.axual.ksml.data.type.ListType; @@ -114,10 +115,16 @@ public int hashCode() { @Override public String toString() { - StringBuilder sb = new StringBuilder(type.toString()).append(": ["); + return toString(Printer.INTERNAL); + } + + @Override + public String toString(Printer printer) { + final var sb = new StringBuilder(printer.schemaString(this)); + sb.append("["); for (int index = 0; index < size(); index++) { if (index > 0) sb.append(", "); - sb.append(get(index).toString()); + sb.append(get(index).toString(printer.childObjectPrinter())); } sb.append("]"); return sb.toString(); diff --git a/ksml-data/src/main/java/io/axual/ksml/data/object/DataObject.java b/ksml-data/src/main/java/io/axual/ksml/data/object/DataObject.java index 5aed19dc..6fa8dbb8 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/object/DataObject.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/object/DataObject.java @@ -24,4 +24,27 @@ public interface DataObject { DataType type(); + + enum Printer { + INTERNAL, + EXTERNAL_NO_SCHEMA, + EXTERNAL_TOP_SCHEMA, + EXTERNAL_ALL_SCHEMA; + + public Printer childObjectPrinter() { + if (this == INTERNAL) return INTERNAL; + return this == EXTERNAL_ALL_SCHEMA ? EXTERNAL_ALL_SCHEMA : EXTERNAL_NO_SCHEMA; + } + + public String forceSchemaString(DataObject value) { + return value.type().schemaName() + ": "; + } + + public String schemaString(DataObject value) { + if (this == INTERNAL || this == EXTERNAL_NO_SCHEMA) return ""; + return forceSchemaString(value); + } + } + + String toString(Printer printer); } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/object/DataPrimitive.java b/ksml-data/src/main/java/io/axual/ksml/data/object/DataPrimitive.java index 59c561bf..4c2faa5c 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/object/DataPrimitive.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/object/DataPrimitive.java @@ -22,6 +22,7 @@ import io.axual.ksml.data.exception.ExecutionException; import io.axual.ksml.data.type.DataType; +import io.axual.ksml.data.util.ValuePrinter; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -46,6 +47,13 @@ private void checkValue() { @Override public String toString() { - return value != null ? value.toString() : type + ": null"; + return toString(Printer.INTERNAL); + } + + @Override + public String toString(Printer printer) { + return value != null + ? ValuePrinter.print(value, printer != Printer.INTERNAL) + : printer.forceSchemaString(this) + ValuePrinter.print(value, printer != Printer.INTERNAL); } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/object/DataStruct.java b/ksml-data/src/main/java/io/axual/ksml/data/object/DataStruct.java index 1d742fb5..936ff008 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/object/DataStruct.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/object/DataStruct.java @@ -23,6 +23,7 @@ import io.axual.ksml.data.exception.ExecutionException; import io.axual.ksml.data.schema.StructSchema; import io.axual.ksml.data.type.StructType; +import io.axual.ksml.data.util.ValuePrinter; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -50,7 +51,6 @@ public class DataStruct implements DataObject { // If both (do not) start with the meta char, then sort as normal return o1.compareTo(o2); }; - private static final String QUOTE = "\""; public interface DataStructApplier { void apply(T value) throws Exception; @@ -93,7 +93,7 @@ public DataObject get(String key) { } public void getIfPresent(String key, Class clazz, DataStructApplier applier) { - var value = get(key); + final var value = get(key); if (value != null && clazz.isAssignableFrom(value.getClass())) { try { applier.apply((T) value); @@ -108,7 +108,7 @@ public T getAs(String key, Class clazz) { } public T getAs(String key, Class clazz, T defaultValue) { - var value = get(key); + final var value = get(key); if (value != null && clazz.isAssignableFrom(value.getClass())) { return (T) value; } @@ -116,7 +116,7 @@ public T getAs(String key, Class clazz, T defaultValue) { } public DataString getAsString(String key) { - var result = get(key); + final var result = get(key); return result instanceof DataString str ? str : result != null ? new DataString(result.toString()) : null; } @@ -136,29 +136,33 @@ public int size() { @Override public String toString() { - var schemaName = type.schemaName() + ": "; + return toString(Printer.INTERNAL); + } + + @Override + public String toString(Printer printer) { + final var schemaName = printer.schemaString(this); // Return NULL as value if (isNull()) return schemaName + "null"; - Iterator> i = entrySet().iterator(); + final var iterator = entrySet().iterator(); // Return empty struct as value - if (!i.hasNext()) return schemaName + "{}"; + if (!iterator.hasNext()) return schemaName + "{}"; - StringBuilder sb = new StringBuilder(); - sb.append(schemaName).append('{'); + // Iterate through all entries + final var sb = new StringBuilder(schemaName).append("{"); while (true) { - Map.Entry e = i.next(); - String key = e.getKey(); - DataObject value = e.getValue(); - var quote = (value instanceof DataString ? QUOTE : ""); - sb.append(QUOTE).append(key).append(QUOTE); - sb.append(':'); - sb.append(quote).append(value == this ? "(this Map)" : value).append(quote); - if (!i.hasNext()) - return sb.append('}').toString(); - sb.append(',').append(' '); + final var entry = iterator.next(); + final var key = entry.getKey(); + final var value = entry.getValue(); + sb.append(ValuePrinter.print(key, true)); + sb.append(": "); + sb.append(value.toString(printer.childObjectPrinter())); + if (!iterator.hasNext()) + return sb.append("}").toString(); + sb.append(", "); } } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/object/DataTuple.java b/ksml-data/src/main/java/io/axual/ksml/data/object/DataTuple.java index 972b649c..068fd90e 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/object/DataTuple.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/object/DataTuple.java @@ -20,6 +20,7 @@ * =========================LICENSE_END================================== */ +import io.axual.ksml.data.exception.DataException; import io.axual.ksml.data.type.DataType; import io.axual.ksml.data.type.TupleType; import io.axual.ksml.data.value.Tuple; @@ -46,6 +47,11 @@ private static DataType[] convertElements(DataObject... elements) { @Override public String toString() { - return type.toString() + ": " + super.toString(); + return toString(Printer.INTERNAL); + } + + @Override + public String toString(Printer printer) { + return printer.schemaString(this) + super.toString(); } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/object/DataUnion.java b/ksml-data/src/main/java/io/axual/ksml/data/object/DataUnion.java index adb58c48..75cc196a 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/object/DataUnion.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/object/DataUnion.java @@ -28,7 +28,7 @@ public DataUnion(UnionType type, DataObject value) { } @Override - public String toString() { - return type().toString() + ": " + super.toString(); + public String toString(Printer printer) { + return printer.schemaString(this) + (value() != null ? value().toString(printer.childObjectPrinter()) : "null"); } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/util/ValuePrinter.java b/ksml-data/src/main/java/io/axual/ksml/data/util/ValuePrinter.java new file mode 100644 index 00000000..3f9629c6 --- /dev/null +++ b/ksml-data/src/main/java/io/axual/ksml/data/util/ValuePrinter.java @@ -0,0 +1,30 @@ +package io.axual.ksml.data.util; + +/*- + * ========================LICENSE_START================================= + * KSML Data Library + * %% + * Copyright (C) 2021 - 2024 Axual B.V. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * =========================LICENSE_END================================== + */ + +public class ValuePrinter { + private static final String QUOTE = "\""; + + public static String print(Object value, boolean quoted) { + var quote = quoted && value instanceof String ? QUOTE : ""; + return value != null ? quote + value + quote : "null"; + } +} diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java index fdff5b20..77c550ec 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java @@ -106,7 +106,7 @@ public static void main(String[] args) { } final var config = readConfiguration(configFile); - final var ksmlConfig = config.getKsmlConfig(); + final var ksmlConfig = config.ksmlConfig(); log.info("Using directories: config: {}, schema: {}, storage: {}", ksmlConfig.getConfigDirectory(), ksmlConfig.getSchemaDirectory(), ksmlConfig.getStorageDirectory()); final var definitions = ksmlConfig.getDefinitions(); if (definitions == null || definitions.isEmpty()) { @@ -119,7 +119,7 @@ public static void main(String[] args) { final var appServer = ksmlConfig.getApplicationServerConfig(); RestServer restServer = null; // Start rest server to provide service endpoints - if (appServer.isEnabled()) { + if (appServer.enabled()) { HostInfo hostInfo = new HostInfo(appServer.getHost(), appServer.getPort()); restServer = new RestServer(hostInfo); restServer.start(); @@ -163,11 +163,11 @@ public static void main(String[] args) { }); - if (!ksmlConfig.isEnableProducers() && !producerSpecs.isEmpty()) { + if (!ksmlConfig.enableProducers() && !producerSpecs.isEmpty()) { log.warn("Producers are disabled for this runner. The supplied producer specifications will be ignored."); producerSpecs.clear(); } - if (!ksmlConfig.isEnablePipelines() && !pipelineSpecs.isEmpty()) { + if (!ksmlConfig.enablePipelines() && !pipelineSpecs.isEmpty()) { log.warn("Pipelines are disabled for this runner. The supplied pipeline specifications will be ignored."); pipelineSpecs.clear(); } @@ -196,7 +196,7 @@ public static void main(String[] args) { Runtime.getRuntime().addShutdownHook(shutdownHook); - try (var prometheusExport = new PrometheusExport(config.getKsmlConfig().getPrometheusConfig())) { + try (var prometheusExport = new PrometheusExport(config.ksmlConfig().prometheusConfig())) { prometheusExport.start(); final var executorService = Executors.newFixedThreadPool(2); @@ -270,7 +270,7 @@ public Collection allMetadataForStore(String storeName) { if (streamsRunner == null) { return List.of(); } - return streamsRunner.getKafkaStreams().streamsMetadataForStore(storeName); + return streamsRunner.kafkaStreams().streamsMetadataForStore(storeName); } @Override @@ -278,7 +278,7 @@ public KeyQueryMetadata queryMetadataForKey(String storeName, K key, Seriali if (streamsRunner == null) { return null; } - return streamsRunner.getKafkaStreams().queryMetadataForKey(storeName, key, keySerializer); + return streamsRunner.kafkaStreams().queryMetadataForKey(storeName, key, keySerializer); } @Override @@ -286,7 +286,7 @@ public T store(StoreQueryParameters storeQueryParameters) { if (streamsRunner == null) { return null; } - return streamsRunner.getKafkaStreams().store(storeQueryParameters); + return streamsRunner.kafkaStreams().store(storeQueryParameters); } @Override @@ -369,7 +369,7 @@ private static KSMLRunnerConfig readConfiguration(File configFile) { try { final var config = mapper.readValue(configFile, KSMLRunnerConfig.class); if (config != null) { - if (config.getKsmlConfig() == null) { + if (config.ksmlConfig() == null) { throw new ConfigException("Section \"ksml\" is missing in configuration"); } if (config.getKafkaConfig() == null) { @@ -385,14 +385,14 @@ private static KSMLRunnerConfig readConfiguration(File configFile) { private static ErrorHandler getErrorHandler(KSMLErrorHandlingConfig.ErrorHandlingConfig config) { - final var handlerType = switch (config.getHandler()) { + final var handlerType = switch (config.handler()) { case CONTINUE -> ErrorHandler.HandlerType.CONTINUE_ON_FAIL; case STOP -> ErrorHandler.HandlerType.STOP_ON_FAIL; }; return new ErrorHandler( - config.isLog(), - config.isLogPayload(), - config.getLoggerName(), + config.log(), + config.logPayload(), + config.loggerName(), handlerType); } } diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaStreamsRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaStreamsRunner.java index 8e75a0d9..fbd58535 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaStreamsRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaStreamsRunner.java @@ -21,6 +21,7 @@ */ +import io.axual.ksml.runner.streams.KSMLClientSupplier; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.StreamsBuilder; @@ -40,7 +41,6 @@ import io.axual.ksml.generator.TopologyDefinition; import io.axual.ksml.runner.config.ApplicationServerConfig; import io.axual.ksml.runner.exception.RunnerException; -import io.axual.ksml.runner.streams.KSMLClientSupplier; import lombok.Builder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -99,7 +99,7 @@ private Map getStreamsConfig(Map initialConfigs, } result.put(StreamsConfig.STATE_DIR_CONFIG, storageDirectory); - if (appServer != null && appServer.isEnabled()) { + if (appServer != null && appServer.enabled()) { result.put(StreamsConfig.APPLICATION_SERVER_CONFIG, appServer.getApplicationServer()); } diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java index cacdad28..0dba68e8 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java @@ -35,13 +35,12 @@ public class KSMLErrorHandlingConfig { private ErrorHandlingConfig produce; private ErrorHandlingConfig process; - public ErrorHandlingConfig getConsumerErrorHandlingConfig() { if (consume == null) { return getDefaultErrorHandlingConfig("ConsumeError"); } - if (consume.getLoggerName() == null) { - consume.setLoggerName("ConsumeError"); + if (consume.loggerName() == null) { + consume.loggerName("ConsumeError"); } return consume; } @@ -50,8 +49,8 @@ public ErrorHandlingConfig getProducerErrorHandlingConfig() { if (produce == null) { return getDefaultErrorHandlingConfig("ProduceError"); } - if (produce.getLoggerName() == null) { - produce.setLoggerName("ProduceError"); + if (produce.loggerName() == null) { + produce.loggerName("ProduceError"); } return produce; } @@ -60,15 +59,15 @@ public ErrorHandlingConfig getProcessErrorHandlingConfig() { if (process == null) { return getDefaultErrorHandlingConfig("ProcessError"); } - if (process.getLoggerName() == null) { - process.setLoggerName("ProcessError"); + if (process.loggerName() == null) { + process.loggerName("ProcessError"); } return process; } private ErrorHandlingConfig getDefaultErrorHandlingConfig(String logger) { var errorHandlingConfig = new ErrorHandlingConfig(); - errorHandlingConfig.setLoggerName(logger); + errorHandlingConfig.loggerName(logger); return errorHandlingConfig; } diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java index 0a8a5cee..f11135d4 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java @@ -48,12 +48,12 @@ public class KSMLRunnerConfig { public Map getKafkaConfig(){ var newConfig = new HashMap<>(kafka.kafkaConfig()); - newConfig.put("application.id", kafka.getApplicationId()); + newConfig.put("application.id", kafka.applicationId()); return newConfig; } public String getApplicationId(){ - return kafka.getApplicationId(); + return kafka.applicationId(); } @Data diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/prometheus/PrometheusExport.java b/ksml-runner/src/main/java/io/axual/ksml/runner/prometheus/PrometheusExport.java index 8963d569..ff09c629 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/prometheus/PrometheusExport.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/prometheus/PrometheusExport.java @@ -54,7 +54,7 @@ public PrometheusExport(PrometheusConfig config) { @Synchronized public void start() throws Exception { Metrics.init(); - if (!config.isEnabled()) { + if (!config.enabled()) { log.info("Prometheus export is disabled"); return; } diff --git a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java index 1130d858..4f829794 100644 --- a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java +++ b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java @@ -45,7 +45,7 @@ void shouldLoadWithoutExceptions() throws IOException { final var yaml = getClass().getClassLoader().getResourceAsStream("ksml-runner-config.yaml"); final var ksmlRunnerConfig = objectMapper.readValue(yaml, KSMLRunnerConfig.class); - assertNotNull(ksmlRunnerConfig.getKsmlConfig()); + assertNotNull(ksmlRunnerConfig.ksmlConfig()); assertNotNull(ksmlRunnerConfig.getKafkaConfig()); } } diff --git a/ksml/src/main/java/io/axual/ksml/user/UserFunction.java b/ksml/src/main/java/io/axual/ksml/user/UserFunction.java index cf232cb9..b321800b 100644 --- a/ksml/src/main/java/io/axual/ksml/user/UserFunction.java +++ b/ksml/src/main/java/io/axual/ksml/user/UserFunction.java @@ -107,15 +107,14 @@ protected void checkType(ParameterDefinition definition, DataObject value) { checkType(definition.type(), value); } - protected void logCall(Object[] parameters, Object result) { + protected void logCall(DataObject[] parameters, Object result) { // Log the called function in debug if (LOG.isDebugEnabled()) { // Check all parameters and copy them into the interpreter as prefixed globals StringBuilder paramsAndValues = new StringBuilder(); for (int index = 0; index < parameters.length; index++) { paramsAndValues.append(!paramsAndValues.isEmpty() ? ", " : ""); - String valueQuote = parameters[index] instanceof String ? "'" : ""; - paramsAndValues.append(this.parameters[index].name()).append("=").append(valueQuote).append(parameters[index] != null ? parameters[index] : "null").append(valueQuote); + paramsAndValues.append(this.parameters[index].name()).append("=").append(parameters[index] != null ? parameters[index].toString(DataObject.Printer.EXTERNAL_ALL_SCHEMA) : "null"); } if (result != null) { LOG.debug("User function {}.{}({}) returned {}", namespace, name, paramsAndValues, result); From 1a60ef2febbb66e20f2aed0ea5fa3112f989fb7b Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 30 Oct 2024 23:16:19 +0100 Subject: [PATCH 14/55] Unify lombok configurations --- .../java/io/axual/ksml/data/lombok.config | 3 --- .../java/io/axual/ksml/data/lombok.config | 3 --- .../java/io/axual/ksml/data/lombok.config | 3 --- .../java/io/axual/ksml/data/lombok.config | 3 --- .../java/io/axual/ksml/client/lombok.config | 3 --- .../java/io/axual/ksml/runner/KSMLRunner.java | 20 +++++++++---------- .../axual/ksml/runner/config/KSMLConfig.java | 16 +++++++-------- .../config/KSMLErrorHandlingConfig.java | 14 ++++++------- .../ksml/runner/config/KSMLRunnerConfig.java | 8 ++++---- .../config/KSMLRunnerKSMLConfigTest.java | 6 +++--- ksml/src/main/java/lombok.config | 3 --- .../ksml/data/lombok.config => lombok.config | 0 12 files changed, 32 insertions(+), 50 deletions(-) delete mode 100644 ksml-data-csv/src/main/java/io/axual/ksml/data/lombok.config delete mode 100644 ksml-data-soap/src/main/java/io/axual/ksml/data/lombok.config delete mode 100644 ksml-data-xml/src/main/java/io/axual/ksml/data/lombok.config delete mode 100644 ksml-data/src/main/java/io/axual/ksml/data/lombok.config delete mode 100644 ksml-kafka-clients/src/main/java/io/axual/ksml/client/lombok.config delete mode 100644 ksml/src/main/java/lombok.config rename ksml-data-avro/src/main/java/io/axual/ksml/data/lombok.config => lombok.config (100%) diff --git a/ksml-data-csv/src/main/java/io/axual/ksml/data/lombok.config b/ksml-data-csv/src/main/java/io/axual/ksml/data/lombok.config deleted file mode 100644 index 881a8aa5..00000000 --- a/ksml-data-csv/src/main/java/io/axual/ksml/data/lombok.config +++ /dev/null @@ -1,3 +0,0 @@ -lombok.accessors.fluent=true -lombok.accessors.chain=false -lombok.equalsAndHashCode.callSuper=call \ No newline at end of file diff --git a/ksml-data-soap/src/main/java/io/axual/ksml/data/lombok.config b/ksml-data-soap/src/main/java/io/axual/ksml/data/lombok.config deleted file mode 100644 index 881a8aa5..00000000 --- a/ksml-data-soap/src/main/java/io/axual/ksml/data/lombok.config +++ /dev/null @@ -1,3 +0,0 @@ -lombok.accessors.fluent=true -lombok.accessors.chain=false -lombok.equalsAndHashCode.callSuper=call \ No newline at end of file diff --git a/ksml-data-xml/src/main/java/io/axual/ksml/data/lombok.config b/ksml-data-xml/src/main/java/io/axual/ksml/data/lombok.config deleted file mode 100644 index 881a8aa5..00000000 --- a/ksml-data-xml/src/main/java/io/axual/ksml/data/lombok.config +++ /dev/null @@ -1,3 +0,0 @@ -lombok.accessors.fluent=true -lombok.accessors.chain=false -lombok.equalsAndHashCode.callSuper=call \ No newline at end of file diff --git a/ksml-data/src/main/java/io/axual/ksml/data/lombok.config b/ksml-data/src/main/java/io/axual/ksml/data/lombok.config deleted file mode 100644 index 881a8aa5..00000000 --- a/ksml-data/src/main/java/io/axual/ksml/data/lombok.config +++ /dev/null @@ -1,3 +0,0 @@ -lombok.accessors.fluent=true -lombok.accessors.chain=false -lombok.equalsAndHashCode.callSuper=call \ No newline at end of file diff --git a/ksml-kafka-clients/src/main/java/io/axual/ksml/client/lombok.config b/ksml-kafka-clients/src/main/java/io/axual/ksml/client/lombok.config deleted file mode 100644 index 881a8aa5..00000000 --- a/ksml-kafka-clients/src/main/java/io/axual/ksml/client/lombok.config +++ /dev/null @@ -1,3 +0,0 @@ -lombok.accessors.fluent=true -lombok.accessors.chain=false -lombok.equalsAndHashCode.callSuper=call \ No newline at end of file diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java index 77c550ec..6dbd3c32 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java @@ -107,8 +107,8 @@ public static void main(String[] args) { final var config = readConfiguration(configFile); final var ksmlConfig = config.ksmlConfig(); - log.info("Using directories: config: {}, schema: {}, storage: {}", ksmlConfig.getConfigDirectory(), ksmlConfig.getSchemaDirectory(), ksmlConfig.getStorageDirectory()); - final var definitions = ksmlConfig.getDefinitions(); + log.info("Using directories: config: {}, schema: {}, storage: {}", ksmlConfig.configDirectory(), ksmlConfig.schemaDirectory(), ksmlConfig.storageDirectory()); + final var definitions = ksmlConfig.definitions(); if (definitions == null || definitions.isEmpty()) { throw new ConfigException("definitions", definitions, "No KSML definitions found in configuration"); } @@ -116,7 +116,7 @@ public static void main(String[] args) { KsmlInfo.registerKsmlAppInfo(config.getApplicationId()); // Start the appserver if needed - final var appServer = ksmlConfig.getApplicationServerConfig(); + final var appServer = ksmlConfig.applicationServerConfig(); RestServer restServer = null; // Start rest server to provide service endpoints if (appServer.enabled()) { @@ -136,17 +136,17 @@ public static void main(String[] args) { NotationLibrary.register(XmlNotation.NOTATION_NAME, new XmlNotation(nativeMapper), new XmlDataObjectConverter()); // Register schema loaders - final var schemaDirectory = ksmlConfig.getSchemaDirectory(); + final var schemaDirectory = ksmlConfig.schemaDirectory(); SchemaLibrary.registerLoader(AvroNotation.NOTATION_NAME, new AvroSchemaLoader(schemaDirectory)); SchemaLibrary.registerLoader(CsvNotation.NOTATION_NAME, new CsvSchemaLoader(schemaDirectory)); SchemaLibrary.registerLoader(JsonNotation.NOTATION_NAME, new JsonSchemaLoader(schemaDirectory)); SchemaLibrary.registerLoader(XmlNotation.NOTATION_NAME, new XmlSchemaLoader(schemaDirectory)); - final var errorHandling = ksmlConfig.getErrorHandlingConfig(); + final var errorHandling = ksmlConfig.errorHandlingConfig(); if (errorHandling != null) { - ExecutionContext.INSTANCE.setConsumeHandler(getErrorHandler(errorHandling.getConsumerErrorHandlingConfig())); - ExecutionContext.INSTANCE.setProduceHandler(getErrorHandler(errorHandling.getProducerErrorHandlingConfig())); - ExecutionContext.INSTANCE.setProcessHandler(getErrorHandler(errorHandling.getProcessErrorHandlingConfig())); + ExecutionContext.INSTANCE.setConsumeHandler(getErrorHandler(errorHandling.consumerErrorHandlingConfig())); + ExecutionContext.INSTANCE.setProduceHandler(getErrorHandler(errorHandling.producerErrorHandlingConfig())); + ExecutionContext.INSTANCE.setProcessHandler(getErrorHandler(errorHandling.processErrorHandlingConfig())); } ExecutionContext.INSTANCE.serdeWrapper( serde -> new Serdes.WrapperSerde<>( @@ -177,8 +177,8 @@ public static void main(String[] args) { .kafkaConfig(config.getKafkaConfig()) .build()); final var streams = pipelineSpecs.isEmpty() ? null : new KafkaStreamsRunner(KafkaStreamsRunner.Config.builder() - .storageDirectory(ksmlConfig.getStorageDirectory()) - .appServer(ksmlConfig.getApplicationServerConfig()) + .storageDirectory(ksmlConfig.storageDirectory()) + .appServer(ksmlConfig.applicationServerConfig()) .definitions(pipelineSpecs) .kafkaConfig(config.getKafkaConfig()) .build()); diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java index 736dae2a..e72f4ab2 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java @@ -82,33 +82,33 @@ private String getDirectory(String configVariable, String directory) { return configPath.toAbsolutePath().normalize().toString(); } - public String getConfigDirectory() { + public String configDirectory() { return getDirectory("configDirectory", configDirectory != null ? configDirectory : System.getProperty("user.dir")); } - public String getSchemaDirectory() { - return getDirectory("schemaDirectory", schemaDirectory != null ? schemaDirectory : getConfigDirectory()); + public String schemaDirectory() { + return getDirectory("schemaDirectory", schemaDirectory != null ? schemaDirectory : configDirectory()); } - public String getStorageDirectory() { + public String storageDirectory() { return getDirectory("storageDirectory", storageDirectory != null ? storageDirectory : System.getProperty("java.io.tmpdir")); } - public ApplicationServerConfig getApplicationServerConfig() { + public ApplicationServerConfig applicationServerConfig() { return applicationServer; } - public KSMLErrorHandlingConfig getErrorHandlingConfig() { + public KSMLErrorHandlingConfig errorHandlingConfig() { if (errorHandling == null) return KSMLErrorHandlingConfig.builder().build(); return errorHandling; } - public Map getDefinitions() { + public Map definitions() { final var result = new HashMap(); if (definitions != null) { for (Map.Entry definition : definitions.entrySet()) { if (definition.getValue() instanceof String definitionFile) { - final var definitionFilePath = Paths.get(getConfigDirectory(), definitionFile); + final var definitionFilePath = Paths.get(configDirectory(), definitionFile); if (Files.notExists(definitionFilePath) || !Files.isRegularFile(definitionFilePath)) { throw new ConfigException("definitionFile", definitionFilePath, "The provided KSML definition file does not exists or is not a regular file"); } diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java index 0dba68e8..6c01c5f3 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java @@ -35,9 +35,9 @@ public class KSMLErrorHandlingConfig { private ErrorHandlingConfig produce; private ErrorHandlingConfig process; - public ErrorHandlingConfig getConsumerErrorHandlingConfig() { + public ErrorHandlingConfig consumerErrorHandlingConfig() { if (consume == null) { - return getDefaultErrorHandlingConfig("ConsumeError"); + return defaultErrorHandlingConfig("ConsumeError"); } if (consume.loggerName() == null) { consume.loggerName("ConsumeError"); @@ -45,9 +45,9 @@ public ErrorHandlingConfig getConsumerErrorHandlingConfig() { return consume; } - public ErrorHandlingConfig getProducerErrorHandlingConfig() { + public ErrorHandlingConfig producerErrorHandlingConfig() { if (produce == null) { - return getDefaultErrorHandlingConfig("ProduceError"); + return defaultErrorHandlingConfig("ProduceError"); } if (produce.loggerName() == null) { produce.loggerName("ProduceError"); @@ -55,9 +55,9 @@ public ErrorHandlingConfig getProducerErrorHandlingConfig() { return produce; } - public ErrorHandlingConfig getProcessErrorHandlingConfig() { + public ErrorHandlingConfig processErrorHandlingConfig() { if (process == null) { - return getDefaultErrorHandlingConfig("ProcessError"); + return defaultErrorHandlingConfig("ProcessError"); } if (process.loggerName() == null) { process.loggerName("ProcessError"); @@ -65,7 +65,7 @@ public ErrorHandlingConfig getProcessErrorHandlingConfig() { return process; } - private ErrorHandlingConfig getDefaultErrorHandlingConfig(String logger) { + private ErrorHandlingConfig defaultErrorHandlingConfig(String logger) { var errorHandlingConfig = new ErrorHandlingConfig(); errorHandlingConfig.loggerName(logger); return errorHandlingConfig; diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java index f11135d4..46050918 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java @@ -44,16 +44,16 @@ public class KSMLRunnerConfig { private KSMLConfig ksmlConfig; @JsonProperty("kafka") - private KafkaConfig kafka; + private KafkaConfig kafkaConfig; public Map getKafkaConfig(){ - var newConfig = new HashMap<>(kafka.kafkaConfig()); - newConfig.put("application.id", kafka.applicationId()); + var newConfig = new HashMap<>(kafkaConfig.kafkaConfig()); + newConfig.put("application.id", kafkaConfig.applicationId()); return newConfig; } public String getApplicationId(){ - return kafka.applicationId(); + return kafkaConfig.applicationId(); } @Data diff --git a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerKSMLConfigTest.java b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerKSMLConfigTest.java index 94ece259..267c979e 100644 --- a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerKSMLConfigTest.java +++ b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerKSMLConfigTest.java @@ -44,7 +44,7 @@ void setup() { void shouldValidateConfig() throws Exception { final var yaml = getClass().getClassLoader().getResourceAsStream("ksml-config.yaml"); final var ksmlConfig = objectMapper.readValue(yaml, KSMLConfig.class); - ksmlConfig.getConfigDirectory(); + ksmlConfig.configDirectory(); } @Test @@ -52,7 +52,7 @@ void shouldValidateConfig() throws Exception { void shouldThrowOnWrongConfigdir() throws Exception { final var yaml = getClass().getClassLoader().getResourceAsStream("ksml-config-wrong-configdir.yaml"); final var ksmlConfig = objectMapper.readValue(yaml, KSMLConfig.class); - assertThrows(ConfigException.class, ksmlConfig::getConfigDirectory, "should throw exception for wrong configdir"); + assertThrows(ConfigException.class, ksmlConfig::configDirectory, "should throw exception for wrong configdir"); } @Test @@ -60,6 +60,6 @@ void shouldThrowOnWrongConfigdir() throws Exception { void shouldDefaultConfigToWorkdir() throws Exception { final var yaml = getClass().getClassLoader().getResourceAsStream("ksml-config-no-configdir.yaml"); final var ksmlConfig = objectMapper.readValue(yaml, KSMLConfig.class); - assertEquals(System.getProperty("user.dir"), ksmlConfig.getConfigDirectory(), "config dir should default to working dir"); + assertEquals(System.getProperty("user.dir"), ksmlConfig.configDirectory(), "config dir should default to working dir"); } } diff --git a/ksml/src/main/java/lombok.config b/ksml/src/main/java/lombok.config deleted file mode 100644 index 881a8aa5..00000000 --- a/ksml/src/main/java/lombok.config +++ /dev/null @@ -1,3 +0,0 @@ -lombok.accessors.fluent=true -lombok.accessors.chain=false -lombok.equalsAndHashCode.callSuper=call \ No newline at end of file diff --git a/ksml-data-avro/src/main/java/io/axual/ksml/data/lombok.config b/lombok.config similarity index 100% rename from ksml-data-avro/src/main/java/io/axual/ksml/data/lombok.config rename to lombok.config From 1f5aefc1de5086a3533e0473223387b62190f691 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Thu, 31 Oct 2024 00:11:32 +0100 Subject: [PATCH 15/55] Implement Batch Producer * Add batchSize option to KSML config * Rework producer logic to allow producing messages in batches per interval * Pretty print ksml-language-spec.json for easier future diffs --- docs/ksml-language-spec.json | 5121 ++++++++++++++++- .../json/JsonDataObjectConverter.java | 2 +- .../notation/json/JsonDataObjectMapper.java | 10 +- .../data/notation/json/JsonSchemaLoader.java | 2 +- .../data/notation/json/JsonSchemaMapper.java | 10 +- .../data/notation/json/JsonStringMapper.java | 12 +- .../io/axual/ksml/data/serde/JsonSerde.java | 2 +- .../java/io/axual/ksml/runner/KSMLRunner.java | 2 +- .../ksml/runner/backend/AlwaysReschedule.java | 56 - .../runner/backend/CountingReschedule.java | 41 - .../runner/backend/KafkaProducerRunner.java | 22 +- .../runner/backend/RescheduleStrategy.java | 63 - .../runner/backend/SingleShotReschedule.java | 34 - .../ksml/runner/backend/UntilReschedule.java | 43 - .../axual/ksml/runner/config/KSMLConfig.java | 10 +- .../config/KSMLErrorHandlingConfig.java | 17 +- .../ExecutableProducer.java | 186 +- .../IntervalSchedule.java | 2 +- .../runner/producer/ProducerStrategy.java | 126 + .../runner/backend/IntervalScheduleTest.java | 2 + .../backend/KafkaProducerRunnerTest.java | 2 +- .../ksml/definition/ProducerDefinition.java | 10 +- .../parser/ProducerDefinitionParser.java | 9 +- .../main/java/io/axual/ksml/dsl/KSMLDSL.java | 1 + .../axual/ksml/parser/DefinitionParser.java | 8 + 25 files changed, 5432 insertions(+), 361 deletions(-) delete mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/backend/AlwaysReschedule.java delete mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/backend/CountingReschedule.java delete mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/backend/RescheduleStrategy.java delete mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/backend/SingleShotReschedule.java delete mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/backend/UntilReschedule.java rename ksml-runner/src/main/java/io/axual/ksml/runner/{backend => producer}/ExecutableProducer.java (54%) rename ksml-runner/src/main/java/io/axual/ksml/runner/{backend => producer}/IntervalSchedule.java (98%) create mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/producer/ProducerStrategy.java diff --git a/docs/ksml-language-spec.json b/docs/ksml-language-spec.json index 4764401c..91193b76 100644 --- a/docs/ksml-language-spec.json +++ b/docs/ksml-language-spec.json @@ -1 +1,5120 @@ -{"additionalProperties":false,"definitions":{"AggregateOperation":{"additionalProperties":false,"description":"An aggregate operation","properties":{"adder":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/AggregatorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* (GroupedTable) A function that adds a record to the aggregation result"},"aggregator":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/AggregatorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* (GroupedStream, SessionWindowedStream, TimeWindowedStream) The aggregator function, which combines a value with the previous aggregation result and outputs a new aggregation result"},"initializer":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/InitializerDefinitionWithImplicitType","type":"object"}],"description":"The initializer function, which generates an initial value for every set of aggregated records"},"merger":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/MergerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* (SessionWindowedStream, SessionWindowedCogroupedStream) A function that combines two aggregation results"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/WindowStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the result aggregation"},"subtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/AggregatorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* (GroupedTable) A function that removes a record from the aggregation result"},"type":{"description":"The type of the operation","enum":["aggregate"]}},"required":["initializer","type"],"title":"AggregateOperation","type":"object"},"AggregatorDefinition":{"additionalProperties":false,"description":"Defines a aggregator function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the aggregator"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the aggregator. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the aggregator. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the aggregator. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the aggregator","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the aggregator. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["aggregator"]}},"required":["type"],"title":"AggregatorDefinition","type":"object"},"AggregatorDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a aggregator function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the aggregator"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the aggregator. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the aggregator. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the aggregator. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the aggregator","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the aggregator. Only required for function types, which are not pre-defined.","type":"string"}},"title":"AggregatorDefinitionWithImplicitType","type":"object"},"BranchDefinitionWithPipeline":{"additionalProperties":false,"description":"Defines a branch with sub-pipeline in a BranchOperation","properties":{"as":{"description":"*(optional)* The name to register the pipeline result under, which can be used as source by follow-up pipelines","type":"string"},"branch":{"description":"*(optional)* Defines a single branch, consisting of a condition and a pipeline to execute for messages that fulfil the predicate","items":{"$ref":"#/definitions/StringOrInlinePredicateDefinitionWithImplicitType","type":"object"},"type":"array"},"forEach":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ForEachActionDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that gets called for every message in the stream"},"if":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/PredicateDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Defines the condition under which messages get sent down this branch"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"print":{"$ref":"#/definitions/PrintOperation","description":"*(optional)* The specification of where to print messages to","type":"object"},"to":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ToTopicDefinition","type":"object"}],"description":"*(optional)* Ends the pipeline by sending all messages to a stream, table or globalTable, or to an inline defined output topic and optional partitioner"},"toTopicNameExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ToTopicNameExtractorDefinition","type":"object"}],"description":"*(optional)* Ends the pipeline by sending all messages to a topic provided by a pre-defined topic name extractor function, or to a topic provided by an inline defined topic name extractor and optional partitioner"},"via":{"description":"*(optional)* A series of operations performed on the input stream","items":{"anyOf":[{"$ref":"#/definitions/AggregateOperation","type":"object"},{"$ref":"#/definitions/CogroupOperation","type":"object"},{"$ref":"#/definitions/ConvertKeyOperation","type":"object"},{"$ref":"#/definitions/ConvertKeyValueOperation","type":"object"},{"$ref":"#/definitions/ConvertValueOperation","type":"object"},{"$ref":"#/definitions/CountOperation","type":"object"},{"$ref":"#/definitions/FilterNotOperation","type":"object"},{"$ref":"#/definitions/FilterOperation","type":"object"},{"$ref":"#/definitions/GroupByKeyOperation","type":"object"},{"$ref":"#/definitions/GroupByOperation","type":"object"},{"$ref":"#/definitions/JoinOperationWithGlobalTable","type":"object"},{"$ref":"#/definitions/JoinOperationWithStream","type":"object"},{"$ref":"#/definitions/JoinOperationWithTable","type":"object"},{"$ref":"#/definitions/LeftJoinOperationWithGlobalTable","type":"object"},{"$ref":"#/definitions/LeftJoinOperationWithStream","type":"object"},{"$ref":"#/definitions/LeftJoinOperationWithTable","type":"object"},{"$ref":"#/definitions/MergeOperation","type":"object"},{"$ref":"#/definitions/OuterJoinOperationWithStream","type":"object"},{"$ref":"#/definitions/OuterJoinOperationWithTable","type":"object"},{"$ref":"#/definitions/PeekOperation","type":"object"},{"$ref":"#/definitions/ReduceOperationWithAdderAndSubtractor","type":"object"},{"$ref":"#/definitions/ReduceOperationWithReducer","type":"object"},{"$ref":"#/definitions/RepartitionOperation","type":"object"},{"$ref":"#/definitions/SuppressOperationUntilTimeLimit","type":"object"},{"$ref":"#/definitions/SuppressOperationUntilWindowCloses","type":"object"},{"$ref":"#/definitions/ToStreamOperation","type":"object"},{"$ref":"#/definitions/ToTableOperation","type":"object"},{"$ref":"#/definitions/TransformKeyOperation","type":"object"},{"$ref":"#/definitions/TransformKeyValueOperation","type":"object"},{"$ref":"#/definitions/TransformKeyValueToKeyValueListOperation","type":"object"},{"$ref":"#/definitions/TransformKeyValueToValueListOperation","type":"object"},{"$ref":"#/definitions/TransformMetadataOperation","type":"object"},{"$ref":"#/definitions/TransformValueOperation","type":"object"},{"$ref":"#/definitions/WindowBySessionOperation","type":"object"},{"$ref":"#/definitions/WindowByTimeOperationWithHoppingWindow","type":"object"},{"$ref":"#/definitions/WindowByTimeOperationWithSlidingWindow","type":"object"},{"$ref":"#/definitions/WindowByTimeOperationWithTumblingWindow","type":"object"}]},"type":"array"}},"title":"BranchDefinitionWithPipeline","type":"object"},"CogroupOperation":{"additionalProperties":false,"description":"A cogroup operation","properties":{"aggregator":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/AggregatorDefinitionWithImplicitType","type":"object"}],"description":"(GroupedStream, SessionWindowedStream, TimeWindowedStream) The aggregator function, which combines a value with the previous aggregation result and outputs a new aggregation result"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/WindowStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the co-grouped stream"},"type":{"description":"The type of the operation","enum":["cogroup"]}},"required":["aggregator","type"],"title":"CogroupOperation","type":"object"},"ConvertKeyOperation":{"additionalProperties":false,"description":"An operation to convert the stream key type to another type. Conversion is only syntactic, eg. from Avro to XML.","properties":{"into":{"description":"The type to convert the stream key into","type":"string"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["convertKey"]}},"required":["into","type"],"title":"ConvertKeyOperation","type":"object"},"ConvertKeyValueOperation":{"additionalProperties":false,"description":"An operation to convert the stream key and value types to other types. Conversion is only syntactic, eg. from Avro to XML.","properties":{"into":{"description":"The tuple type to convert the stream key/value into","type":"string"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["convertKeyValue"]}},"required":["into","type"],"title":"ConvertKeyValueOperation","type":"object"},"ConvertValueOperation":{"additionalProperties":false,"description":"An operation to convert the stream value type to another type. Conversion is only syntactic, eg. from Avro to XML.","properties":{"into":{"description":"The type to convert the stream value into","type":"string"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["convertValue"]}},"required":["into","type"],"title":"ConvertValueOperation","type":"object"},"CountOperation":{"additionalProperties":false,"description":"Count the number of times a key is seen in a given window","properties":{"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinition","type":"object"},{"$ref":"#/definitions/SessionStateStoreDefinition","type":"object"},{"$ref":"#/definitions/WindowStateStoreDefinition","type":"object"}],"description":"*(optional)* Materialized view of the count operation's result"},"type":{"description":"The type of the operation","enum":["count"]}},"required":["type"],"title":"CountOperation","type":"object"},"FilterNotOperation":{"additionalProperties":false,"description":"Filter records based on the inverse result of a predicate function","properties":{"if":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/PredicateDefinitionWithImplicitType","type":"object"}],"description":"A function that returns \"false\" when records are accepted, \"true\" otherwise"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the filtered table (only applies to tables, ignored for streams)"},"type":{"description":"The type of the operation","enum":["filterNot"]}},"required":["if","type"],"title":"FilterNotOperation","type":"object"},"FilterOperation":{"additionalProperties":false,"description":"Filter records based on a predicate function","properties":{"if":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/PredicateDefinitionWithImplicitType","type":"object"}],"description":"A function that returns \"true\" when records are accepted, \"false\" otherwise"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the filtered table (only applies to tables, ignored for streams)"},"type":{"description":"The type of the operation","enum":["filter"]}},"required":["if","type"],"title":"FilterOperation","type":"object"},"ForEachActionDefinition":{"additionalProperties":false,"description":"Defines a foreach action function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the foreach action"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the foreach action. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the foreach action. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the foreach action. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the foreach action","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the foreach action. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the foreach action uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"},"type":{"description":"The type of the function","enum":["forEach"]}},"required":["type"],"title":"ForEachActionDefinition","type":"object"},"ForEachActionDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a foreach action function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the foreach action"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the foreach action. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the foreach action. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the foreach action. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the foreach action","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the foreach action. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the foreach action uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"}},"title":"ForEachActionDefinitionWithImplicitType","type":"object"},"ForeignKeyExtractorDefinition":{"additionalProperties":false,"description":"Defines a foreign key extractor function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the foreign key extractor"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the foreign key extractor. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the foreign key extractor. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the foreign key extractor. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the foreign key extractor","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the foreign key extractor. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["foreignKeyExtractor"]}},"required":["type"],"title":"ForeignKeyExtractorDefinition","type":"object"},"ForeignKeyExtractorDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a foreign key extractor function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the foreign key extractor"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the foreign key extractor. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the foreign key extractor. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the foreign key extractor. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the foreign key extractor","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the foreign key extractor. Only required for function types, which are not pre-defined.","type":"string"}},"title":"ForeignKeyExtractorDefinitionWithImplicitType","type":"object"},"GeneratorDefinition":{"additionalProperties":false,"description":"Defines a message generator function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the message generator"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the message generator. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the message generator. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the message generator. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the message generator","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the message generator. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["generator"]}},"required":["type"],"title":"GeneratorDefinition","type":"object"},"GeneratorDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a message generator function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the message generator"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the message generator. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the message generator. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the message generator. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the message generator","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the message generator. Only required for function types, which are not pre-defined.","type":"string"}},"title":"GeneratorDefinitionWithImplicitType","type":"object"},"GenericFunctionDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a generic function function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the generic function"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the generic function. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the generic function. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the generic function. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the generic function","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the generic function. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["generic"]}},"title":"GenericFunctionDefinitionWithImplicitType","type":"object"},"GlobalTableDefinition":{"additionalProperties":false,"description":"Contains a definition of a GlobalTable, which can be referenced by producers and pipelines","properties":{"keyType":{"description":"*(optional)* The key type of the global table","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* KeyValue state store definition"},"topic":{"description":"The name of the Kafka topic for this global table","type":"string"},"valueType":{"description":"*(optional)* The value type of the global table","type":"string"}},"required":["topic"],"title":"GlobalTableDefinition","type":"object"},"GlobalTableDefinitionSource":{"additionalProperties":false,"description":"Contains a definition of a GlobalTable, which can be referenced by producers and pipelines","properties":{"keyType":{"description":"The key type of the global table","type":"string"},"offsetResetPolicy":{"description":"*(optional)* Policy that determines what to do when there is no initial offset in Kafka, or if the current offset does not exist any more on the server (e.g. because that data has been deleted)","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* KeyValue state store definition"},"timestampExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TimestampExtractorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function extracts the event time from a consumed record"},"topic":{"description":"The name of the Kafka topic for this global table","type":"string"},"valueType":{"description":"The value type of the global table","type":"string"}},"required":["topic","keyType","valueType"],"title":"GlobalTableDefinitionSource","type":"object"},"GroupByKeyOperation":{"additionalProperties":false,"description":"Operation to group all messages with the same key together","properties":{"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the grouped stream"},"type":{"description":"The type of the operation","enum":["groupByKey"]}},"required":["type"],"title":"GroupByKeyOperation","type":"object"},"GroupByOperation":{"additionalProperties":false,"description":"Operation to group all messages with together based on a keying function","properties":{"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueMapperDefinitionWithImplicitType","type":"object"}],"description":"Function to map records to a key they can be grouped on"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the grouped stream or table"},"type":{"description":"The type of the operation","enum":["groupBy"]}},"required":["mapper","type"],"title":"GroupByOperation","type":"object"},"InitializerDefinition":{"additionalProperties":false,"description":"Defines a initializer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the initializer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the initializer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the initializer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the initializer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the initializer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the initializer. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["initializer"]}},"required":["type"],"title":"InitializerDefinition","type":"object"},"InitializerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a initializer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the initializer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the initializer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the initializer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the initializer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the initializer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the initializer. Only required for function types, which are not pre-defined.","type":"string"}},"title":"InitializerDefinitionWithImplicitType","type":"object"},"JoinOperationWithGlobalTable":{"additionalProperties":false,"description":"Operation to join with a table","properties":{"globalTable":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/GlobalTableDefinition","type":"object"}],"description":"A reference to the globalTable, or an inline definition of the globalTable to join with"},"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueMapperDefinitionWithImplicitType","type":"object"}],"description":"A function that maps the key value from the stream to the primary key type of the globalTable"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["join"]},"valueJoiner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueJoinerDefinitionWithImplicitType","type":"object"}],"description":"A function that joins two values"}},"required":["globalTable","mapper","valueJoiner","type"],"title":"JoinOperationWithGlobalTable","type":"object"},"JoinOperationWithStream":{"additionalProperties":false,"description":"Operation to join with a stream","properties":{"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinition","type":"object"},{"$ref":"#/definitions/SessionStateStoreDefinition","type":"object"},{"$ref":"#/definitions/WindowStateStoreDefinition","type":"object"}],"description":"*(optional)* Materialized view of the joined streams"},"stream":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamDefinition","type":"object"}],"description":"A reference to the Stream, or an inline definition of the stream to join with"},"timeDifference":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The maximum time difference for a join over two streams on the same key"},"type":{"description":"The type of the operation","enum":["join"]},"valueJoiner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueJoinerDefinitionWithImplicitType","type":"object"}],"description":"A function that joins two values"}},"required":["stream","valueJoiner","timeDifference","type"],"title":"JoinOperationWithStream","type":"object"},"JoinOperationWithTable":{"additionalProperties":false,"description":"Operation to join with a table","properties":{"foreignKeyExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ForeignKeyExtractorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that can translate the join table value to a primary key"},"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"otherPartitioner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamPartitionerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that partitions the records on the join table"},"partitioner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamPartitionerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that partitions the records on the primary table"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinition","type":"object"},{"$ref":"#/definitions/SessionStateStoreDefinition","type":"object"},{"$ref":"#/definitions/WindowStateStoreDefinition","type":"object"}],"description":"*(optional)* Materialized view of the joined streams"},"table":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TableDefinition","type":"object"}],"description":"A reference to the table, or an inline definition of the table to join with"},"type":{"description":"The type of the operation","enum":["join"]},"valueJoiner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueJoinerDefinitionWithImplicitType","type":"object"}],"description":"A function that joins two values"}},"required":["table","valueJoiner","type"],"title":"JoinOperationWithTable","type":"object"},"KeyTransformerDefinition":{"additionalProperties":false,"description":"Defines a key transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the key transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the key transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the key transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the key transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the key transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the key transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the key transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"},"type":{"description":"The type of the function","enum":["keyTransformer"]}},"required":["type"],"title":"KeyTransformerDefinition","type":"object"},"KeyTransformerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a key transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the key transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the key transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the key transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the key transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the key transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the key transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the key transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"}},"title":"KeyTransformerDefinitionWithImplicitType","type":"object"},"KeyValueMapperDefinition":{"additionalProperties":false,"description":"Defines a keyvalue mapper function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue mapper"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue mapper. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue mapper. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue mapper. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue mapper","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue mapper. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["keyValueMapper"]}},"required":["type"],"title":"KeyValueMapperDefinition","type":"object"},"KeyValueMapperDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a keyvalue mapper function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue mapper"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue mapper. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue mapper. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue mapper. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue mapper","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue mapper. Only required for function types, which are not pre-defined.","type":"string"}},"title":"KeyValueMapperDefinitionWithImplicitType","type":"object"},"KeyValuePrinterDefinition":{"additionalProperties":false,"description":"Defines a keyvalue printer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue printer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue printer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue printer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue printer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue printer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue printer. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["keyValuePrinter"]}},"required":["type"],"title":"KeyValuePrinterDefinition","type":"object"},"KeyValuePrinterDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a keyvalue printer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue printer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue printer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue printer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue printer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue printer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue printer. Only required for function types, which are not pre-defined.","type":"string"}},"title":"KeyValuePrinterDefinitionWithImplicitType","type":"object"},"KeyValueStateStoreDefinition":{"additionalProperties":false,"description":"Definition of a keyValue state store","properties":{"caching":{"description":"*(optional)* \"true\" if changed to the keyValue store need to be buffered and periodically released, \"false\" to emit all changes directly","type":"boolean"},"historyRetention":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* (Versioned only) The duration for which old record versions are available for query (cannot be negative)"},"keyType":{"description":"*(optional)* The key type of the keyValue store","type":"string"},"logging":{"description":"*(optional)* \"true\" if a changelog topic should be set up on Kafka for this keyValue store, \"false\" otherwise","type":"boolean"},"name":{"description":"*(optional)* The name of the keyValue store. If this field is not defined, then the name is derived from the context.","type":"string"},"persistent":{"description":"*(optional)* \"true\" if this keyValue store needs to be stored on disk, \"false\" otherwise","type":"boolean"},"segmentInterval":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* Size of segments for storing old record versions (must be positive). Old record versions for the same key in a single segment are stored (updated and accessed) together. The only impact of this parameter is performance. If segments are large and a workload results in many record versions for the same key being collected in a single segment, performance may degrade as a result. On the other hand, historical reads (which access older segments) and out-of-order writes may slow down if there are too many segments."},"timestamped":{"description":"*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise","type":"boolean"},"type":{"description":"The type of the state store","enum":["keyValue"]},"valueType":{"description":"*(optional)* The value type of the keyValue store","type":"string"},"versioned":{"description":"*(optional)* \"true\" if elements in the store are versioned, \"false\" otherwise","type":"boolean"}},"required":["type"],"title":"KeyValueStateStoreDefinition","type":"object"},"KeyValueStateStoreDefinitionWithImplicitType":{"additionalProperties":false,"description":"Definition of a keyValue state store","properties":{"caching":{"description":"*(optional)* \"true\" if changed to the keyValue store need to be buffered and periodically released, \"false\" to emit all changes directly","type":"boolean"},"historyRetention":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* (Versioned only) The duration for which old record versions are available for query (cannot be negative)"},"keyType":{"description":"*(optional)* The key type of the keyValue store","type":"string"},"logging":{"description":"*(optional)* \"true\" if a changelog topic should be set up on Kafka for this keyValue store, \"false\" otherwise","type":"boolean"},"name":{"description":"*(optional)* The name of the keyValue store. If this field is not defined, then the name is derived from the context.","type":"string"},"persistent":{"description":"*(optional)* \"true\" if this keyValue store needs to be stored on disk, \"false\" otherwise","type":"boolean"},"segmentInterval":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* Size of segments for storing old record versions (must be positive). Old record versions for the same key in a single segment are stored (updated and accessed) together. The only impact of this parameter is performance. If segments are large and a workload results in many record versions for the same key being collected in a single segment, performance may degrade as a result. On the other hand, historical reads (which access older segments) and out-of-order writes may slow down if there are too many segments."},"timestamped":{"description":"*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise","type":"boolean"},"type":{"description":"The type of the state store","enum":["keyValue"]},"valueType":{"description":"*(optional)* The value type of the keyValue store","type":"string"},"versioned":{"description":"*(optional)* \"true\" if elements in the store are versioned, \"false\" otherwise","type":"boolean"}},"title":"KeyValueStateStoreDefinitionWithImplicitType","type":"object"},"KeyValueToKeyValueListTransformerDefinition":{"additionalProperties":false,"description":"Defines a keyvalue-to-keyvaluelist transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue-to-keyvaluelist transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue-to-keyvaluelist transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue-to-keyvaluelist transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue-to-keyvaluelist transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue-to-keyvaluelist transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue-to-keyvaluelist transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the keyvalue-to-keyvaluelist transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"},"type":{"description":"The type of the function","enum":["keyValueToKeyValueListTransformer"]}},"required":["type"],"title":"KeyValueToKeyValueListTransformerDefinition","type":"object"},"KeyValueToKeyValueListTransformerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a keyvalue-to-keyvaluelist transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue-to-keyvaluelist transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue-to-keyvaluelist transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue-to-keyvaluelist transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue-to-keyvaluelist transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue-to-keyvaluelist transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue-to-keyvaluelist transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the keyvalue-to-keyvaluelist transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"}},"title":"KeyValueToKeyValueListTransformerDefinitionWithImplicitType","type":"object"},"KeyValueToValueListTransformerDefinition":{"additionalProperties":false,"description":"Defines a keyvalue-to-valuelist transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue-to-valuelist transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue-to-valuelist transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue-to-valuelist transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue-to-valuelist transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue-to-valuelist transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue-to-valuelist transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the keyvalue-to-valuelist transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"},"type":{"description":"The type of the function","enum":["keyValueToValueListTransformer"]}},"required":["type"],"title":"KeyValueToValueListTransformerDefinition","type":"object"},"KeyValueToValueListTransformerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a keyvalue-to-valuelist transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue-to-valuelist transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue-to-valuelist transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue-to-valuelist transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue-to-valuelist transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue-to-valuelist transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue-to-valuelist transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the keyvalue-to-valuelist transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"}},"title":"KeyValueToValueListTransformerDefinitionWithImplicitType","type":"object"},"KeyValueTransformerDefinition":{"additionalProperties":false,"description":"Defines a keyvalue transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the keyvalue transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"},"type":{"description":"The type of the function","enum":["keyValueTransformer"]}},"required":["type"],"title":"KeyValueTransformerDefinition","type":"object"},"KeyValueTransformerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a keyvalue transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the keyvalue transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"}},"title":"KeyValueTransformerDefinitionWithImplicitType","type":"object"},"LeftJoinOperationWithGlobalTable":{"additionalProperties":false,"description":"Operation to leftJoin with a globalTable","properties":{"globalTable":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/GlobalTableDefinition","type":"object"}],"description":"A reference to the globalTable, or an inline definition of the globalTable to join with"},"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueMapperDefinitionWithImplicitType","type":"object"}],"description":"A function that maps the key value from the stream with the primary key of the globalTable"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["leftJoin"]},"valueJoiner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueJoinerDefinitionWithImplicitType","type":"object"}],"description":"A function that joins two values"}},"required":["globalTable","mapper","valueJoiner","type"],"title":"LeftJoinOperationWithGlobalTable","type":"object"},"LeftJoinOperationWithStream":{"additionalProperties":false,"description":"Operation to leftJoin with a stream","properties":{"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinition","type":"object"},{"$ref":"#/definitions/SessionStateStoreDefinition","type":"object"},{"$ref":"#/definitions/WindowStateStoreDefinition","type":"object"}],"description":"*(optional)* Materialized view of the joined streams"},"stream":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamDefinition","type":"object"}],"description":"A reference to the stream, or an inline definition of the stream to leftJoin with"},"timeDifference":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The maximum time difference for a leftJoin over two streams on the same key"},"type":{"description":"The type of the operation","enum":["leftJoin"]},"valueJoiner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueJoinerDefinitionWithImplicitType","type":"object"}],"description":"A function that joins two values"}},"required":["stream","valueJoiner","timeDifference","type"],"title":"LeftJoinOperationWithStream","type":"object"},"LeftJoinOperationWithTable":{"additionalProperties":false,"description":"Operation to leftJoin with a table","properties":{"foreignKeyExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ForeignKeyExtractorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that can translate the join table value to a primary key"},"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"otherPartitioner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamPartitionerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that partitions the records on the join table"},"partitioner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamPartitionerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that partitions the records on the primary table"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinition","type":"object"},{"$ref":"#/definitions/SessionStateStoreDefinition","type":"object"},{"$ref":"#/definitions/WindowStateStoreDefinition","type":"object"}],"description":"*(optional)* Materialized view of the joined streams"},"table":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TableDefinition","type":"object"}],"description":"A reference to the Table, or an inline definition of the Table to join with"},"type":{"description":"The type of the operation","enum":["leftJoin"]},"valueJoiner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueJoinerDefinitionWithImplicitType","type":"object"}],"description":"A function that joins two values"}},"required":["table","valueJoiner","type"],"title":"LeftJoinOperationWithTable","type":"object"},"MergeOperation":{"additionalProperties":false,"description":"A merge operation to join two Streams","properties":{"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"stream":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamDefinitionSource","type":"object"}],"description":"The stream to merge with"},"type":{"description":"The type of the operation","enum":["merge"]}},"required":["stream","type"],"title":"MergeOperation","type":"object"},"MergerDefinition":{"additionalProperties":false,"description":"Defines a merger function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the merger"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the merger. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the merger. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the merger. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the merger","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the merger. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["merger"]}},"required":["type"],"title":"MergerDefinition","type":"object"},"MergerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a merger function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the merger"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the merger. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the merger. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the merger. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the merger","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the merger. Only required for function types, which are not pre-defined.","type":"string"}},"title":"MergerDefinitionWithImplicitType","type":"object"},"MetadataTransformerDefinition":{"additionalProperties":false,"description":"Defines a metadata transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the metadata transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the metadata transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the metadata transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the metadata transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the metadata transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the metadata transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the metadata transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"},"type":{"description":"The type of the function","enum":["metadataTransformer"]}},"required":["type"],"title":"MetadataTransformerDefinition","type":"object"},"MetadataTransformerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a metadata transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the metadata transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the metadata transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the metadata transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the metadata transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the metadata transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the metadata transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the metadata transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"}},"title":"MetadataTransformerDefinitionWithImplicitType","type":"object"},"OuterJoinOperationWithStream":{"additionalProperties":false,"description":"Operation to outerJoin with a stream","properties":{"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinition","type":"object"},{"$ref":"#/definitions/SessionStateStoreDefinition","type":"object"},{"$ref":"#/definitions/WindowStateStoreDefinition","type":"object"}],"description":"*(optional)* Materialized view of the outerJoined streams"},"stream":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamDefinition","type":"object"}],"description":"A reference to the stream, or an inline definition of the stream to outerJoin with"},"timeDifference":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The maximum time difference for an outerJoin over two streams on the same key"},"type":{"description":"The type of the operation","enum":["outerJoin"]},"valueJoiner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueJoinerDefinitionWithImplicitType","type":"object"}],"description":"A function that joins two values"}},"required":["stream","valueJoiner","timeDifference","type"],"title":"OuterJoinOperationWithStream","type":"object"},"OuterJoinOperationWithTable":{"additionalProperties":false,"description":"Operation to outerJoin with a table","properties":{"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinition","type":"object"},{"$ref":"#/definitions/SessionStateStoreDefinition","type":"object"},{"$ref":"#/definitions/WindowStateStoreDefinition","type":"object"}],"description":"*(optional)* Materialized view of the outerJoined streams"},"table":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TableDefinition","type":"object"}],"description":"A reference to the table, or an inline definition of the table to outerJoin with"},"type":{"description":"The type of the operation","enum":["outerJoin"]},"valueJoiner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueJoinerDefinitionWithImplicitType","type":"object"}],"description":"A function that joins two values"}},"required":["table","valueJoiner","type"],"title":"OuterJoinOperationWithTable","type":"object"},"ParameterDefinition":{"additionalProperties":false,"description":"Defines a parameter for a user function","properties":{"defaultValue":{"description":"*(optional)* The default value for the parameter","type":"string"},"name":{"description":"The name of the parameter","type":"string"},"type":{"description":"The type of the parameter","type":"string"}},"required":["name","type"],"title":"ParameterDefinition","type":"object"},"PeekOperation":{"additionalProperties":false,"description":"Operation to peek into a stream, without modifying the stream contents","properties":{"forEach":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ForEachActionDefinitionWithImplicitType","type":"object"}],"description":"A function that gets called for every message in the stream"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["peek"]}},"required":["forEach","type"],"title":"PeekOperation","type":"object"},"PipelineDefinition":{"additionalProperties":false,"description":"Defines a pipeline through a source, a series of operations to perform on it and a sink operation to close the stream with","properties":{"as":{"description":"*(optional)* The name to register the pipeline result under, which can be used as source by follow-up pipelines","type":"string"},"branch":{"description":"*(optional)* Defines a single branch, consisting of a condition and a pipeline to execute for messages that fulfil the predicate","items":{"$ref":"#/definitions/BranchDefinitionWithPipeline","type":"object"},"type":"array"},"forEach":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ForEachActionDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that gets called for every message in the stream"},"from":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TopicDefinitionSource","type":"object"}],"description":"Pipeline source"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"print":{"$ref":"#/definitions/PrintOperation","description":"*(optional)* The specification of where to print messages to","type":"object"},"to":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ToTopicDefinition","type":"object"}],"description":"*(optional)* Ends the pipeline by sending all messages to a stream, table or globalTable, or to an inline defined output topic and optional partitioner"},"toTopicNameExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ToTopicNameExtractorDefinition","type":"object"}],"description":"*(optional)* Ends the pipeline by sending all messages to a topic provided by a pre-defined topic name extractor function, or to a topic provided by an inline defined topic name extractor and optional partitioner"},"via":{"description":"*(optional)* A series of operations performed on the input stream","items":{"anyOf":[{"$ref":"#/definitions/AggregateOperation","type":"object"},{"$ref":"#/definitions/CogroupOperation","type":"object"},{"$ref":"#/definitions/ConvertKeyOperation","type":"object"},{"$ref":"#/definitions/ConvertKeyValueOperation","type":"object"},{"$ref":"#/definitions/ConvertValueOperation","type":"object"},{"$ref":"#/definitions/CountOperation","type":"object"},{"$ref":"#/definitions/FilterNotOperation","type":"object"},{"$ref":"#/definitions/FilterOperation","type":"object"},{"$ref":"#/definitions/GroupByKeyOperation","type":"object"},{"$ref":"#/definitions/GroupByOperation","type":"object"},{"$ref":"#/definitions/JoinOperationWithGlobalTable","type":"object"},{"$ref":"#/definitions/JoinOperationWithStream","type":"object"},{"$ref":"#/definitions/JoinOperationWithTable","type":"object"},{"$ref":"#/definitions/LeftJoinOperationWithGlobalTable","type":"object"},{"$ref":"#/definitions/LeftJoinOperationWithStream","type":"object"},{"$ref":"#/definitions/LeftJoinOperationWithTable","type":"object"},{"$ref":"#/definitions/MergeOperation","type":"object"},{"$ref":"#/definitions/OuterJoinOperationWithStream","type":"object"},{"$ref":"#/definitions/OuterJoinOperationWithTable","type":"object"},{"$ref":"#/definitions/PeekOperation","type":"object"},{"$ref":"#/definitions/ReduceOperationWithAdderAndSubtractor","type":"object"},{"$ref":"#/definitions/ReduceOperationWithReducer","type":"object"},{"$ref":"#/definitions/RepartitionOperation","type":"object"},{"$ref":"#/definitions/SuppressOperationUntilTimeLimit","type":"object"},{"$ref":"#/definitions/SuppressOperationUntilWindowCloses","type":"object"},{"$ref":"#/definitions/ToStreamOperation","type":"object"},{"$ref":"#/definitions/ToTableOperation","type":"object"},{"$ref":"#/definitions/TransformKeyOperation","type":"object"},{"$ref":"#/definitions/TransformKeyValueOperation","type":"object"},{"$ref":"#/definitions/TransformKeyValueToKeyValueListOperation","type":"object"},{"$ref":"#/definitions/TransformKeyValueToValueListOperation","type":"object"},{"$ref":"#/definitions/TransformMetadataOperation","type":"object"},{"$ref":"#/definitions/TransformValueOperation","type":"object"},{"$ref":"#/definitions/WindowBySessionOperation","type":"object"},{"$ref":"#/definitions/WindowByTimeOperationWithHoppingWindow","type":"object"},{"$ref":"#/definitions/WindowByTimeOperationWithSlidingWindow","type":"object"},{"$ref":"#/definitions/WindowByTimeOperationWithTumblingWindow","type":"object"}]},"type":"array"}},"required":["from"],"title":"PipelineDefinition","type":"object"},"PredicateDefinition":{"additionalProperties":false,"description":"Defines a predicate function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the predicate"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the predicate. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the predicate. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the predicate. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the predicate","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the predicate. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the predicate uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"},"type":{"description":"The type of the function","enum":["predicate"]}},"required":["type"],"title":"PredicateDefinition","type":"object"},"PredicateDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a predicate function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the predicate"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the predicate. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the predicate. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the predicate. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the predicate","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the predicate. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the predicate uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"}},"title":"PredicateDefinitionWithImplicitType","type":"object"},"PrintOperation":{"additionalProperties":false,"description":"Operation to print the contents of a pipeline on the screen or to write them to a file","properties":{"filename":{"description":"*(optional)* The filename to output records to. If nothing is specified, then messages will be printed on stdout.","type":"string"},"label":{"description":"*(optional)* A label to attach to the output records","type":"string"},"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValuePrinterDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function to convert record into a string for output"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"}},"title":"PrintOperation","type":"object"},"ProducerDefinition":{"additionalProperties":false,"description":"Definition of a Producer that regularly generates messages for a topic","properties":{"condition":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/PredicateDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that validates the generator's result message. Returns \"true\" when the message may be produced on the topic, \"false\" otherwise."},"count":{"description":"*(optional)* The number of messages to produce."},"generator":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/GeneratorDefinitionWithImplicitType","type":"object"}],"description":"The function that generates records"},"interval":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The interval with which the generator is called"},"to":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TopicDefinition","type":"object"}],"description":"The topic to produce to"},"until":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/PredicateDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A predicate that returns true to indicate producing should stop."}},"required":["generator","interval","to"],"title":"ProducerDefinition","type":"object"},"ReduceOperationWithAdderAndSubtractor":{"additionalProperties":false,"description":"Operation to reduce a series of records into a single aggregate result","properties":{"adder":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ReducerDefinitionWithImplicitType","type":"object"}],"description":"A function that adds a record to the aggregate result"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/WindowStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the aggregation"},"subtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ReducerDefinitionWithImplicitType","type":"object"}],"description":"A function that removes a record from the aggregate result"},"type":{"description":"The type of the operation","enum":["reduce"]}},"required":["adder","subtractor","type"],"title":"ReduceOperationWithAdderAndSubtractor","type":"object"},"ReduceOperationWithReducer":{"additionalProperties":false,"description":"Operation to reduce a series of records into a single aggregate result","properties":{"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"reducer":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ReducerDefinitionWithImplicitType","type":"object"}],"description":"A function that computes a new aggregate result"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/WindowStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the aggregation"},"type":{"description":"The type of the operation","enum":["reduce"]}},"required":["reducer","type"],"title":"ReduceOperationWithReducer","type":"object"},"ReducerDefinition":{"additionalProperties":false,"description":"Defines a reducer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the reducer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the reducer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the reducer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the reducer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the reducer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the reducer. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["reducer"]}},"required":["type"],"title":"ReducerDefinition","type":"object"},"ReducerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a reducer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the reducer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the reducer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the reducer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the reducer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the reducer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the reducer. Only required for function types, which are not pre-defined.","type":"string"}},"title":"ReducerDefinitionWithImplicitType","type":"object"},"RepartitionOperation":{"additionalProperties":false,"description":"Operation to (re)partition a stream","properties":{"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"numberOfPartitions":{"description":"*(optional)* The target number of partitions"},"partitioner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamPartitionerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that partitions stream records"},"type":{"description":"The type of the operation","enum":["repartition"]}},"required":["type"],"title":"RepartitionOperation","type":"object"},"SessionStateStoreDefinition":{"additionalProperties":false,"description":"Definition of a session state store","properties":{"caching":{"description":"*(optional)* \"true\" if changed to the session store need to be buffered and periodically released, \"false\" to emit all changes directly","type":"boolean"},"keyType":{"description":"*(optional)* The key type of the session store","type":"string"},"logging":{"description":"*(optional)* \"true\" if a changelog topic should be set up on Kafka for this session store, \"false\" otherwise","type":"boolean"},"name":{"description":"*(optional)* The name of the session store. If this field is not defined, then the name is derived from the context.","type":"string"},"persistent":{"description":"*(optional)* \"true\" if this session store needs to be stored on disk, \"false\" otherwise","type":"boolean"},"retention":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The duration for which elements in the session store are retained"},"timestamped":{"description":"*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise","type":"boolean"},"type":{"description":"The type of the state store","enum":["session"]},"valueType":{"description":"*(optional)* The value type of the session store","type":"string"}},"required":["type"],"title":"SessionStateStoreDefinition","type":"object"},"StreamDefinition":{"additionalProperties":false,"description":"Contains a definition of a Stream, which can be referenced by producers and pipelines","properties":{"keyType":{"description":"*(optional)* The key type of the stream","type":"string"},"topic":{"description":"The name of the Kafka topic for this stream","type":"string"},"valueType":{"description":"*(optional)* The value type of the stream","type":"string"}},"required":["topic"],"title":"StreamDefinition","type":"object"},"StreamDefinitionSource":{"additionalProperties":false,"description":"Contains a definition of a Stream, which can be referenced by producers and pipelines","properties":{"keyType":{"description":"The key type of the stream","type":"string"},"offsetResetPolicy":{"description":"*(optional)* Policy that determines what to do when there is no initial offset in Kafka, or if the current offset does not exist any more on the server (e.g. because that data has been deleted)","type":"string"},"timestampExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TimestampExtractorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function extracts the event time from a consumed record"},"topic":{"description":"The name of the Kafka topic for this stream","type":"string"},"valueType":{"description":"The value type of the stream","type":"string"}},"required":["topic","keyType","valueType"],"title":"StreamDefinitionSource","type":"object"},"StreamPartitionerDefinition":{"additionalProperties":false,"description":"Defines a stream partitioner function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the stream partitioner"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the stream partitioner. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the stream partitioner. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the stream partitioner. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the stream partitioner","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the stream partitioner. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["streamPartitioner"]}},"required":["type"],"title":"StreamPartitionerDefinition","type":"object"},"StreamPartitionerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a stream partitioner function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the stream partitioner"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the stream partitioner. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the stream partitioner. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the stream partitioner. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the stream partitioner","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the stream partitioner. Only required for function types, which are not pre-defined.","type":"string"}},"title":"StreamPartitionerDefinitionWithImplicitType","type":"object"},"StringOrInlinePredicateDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines the condition under which messages get sent down this branch","properties":{"if":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/PredicateDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Defines the condition under which messages get sent down this branch"}},"title":"StringOrInlinePredicateDefinitionWithImplicitType","type":"object"},"SuppressOperationUntilTimeLimit":{"additionalProperties":false,"description":"Operation to suppress messages in the source stream until a time limit is reached","properties":{"bufferFullStrategy":{"description":"*(optional)* What to do when the buffer is full","enum":["emitEarlyWhenFull","shutdownWhenFull"]},"duration":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The duration for which messages are suppressed"},"maxBytes":{"description":"*(optional)* The maximum number of bytes in the buffer","type":"string"},"maxRecords":{"description":"*(optional)* The maximum number of records in the buffer","type":"string"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["suppress"]},"until":{"description":"The until of the Operation to suppress messages in the source stream until a certain limit is reached","enum":["timeLimit"]}},"required":["duration","until","type"],"title":"SuppressOperationUntilTimeLimit","type":"object"},"SuppressOperationUntilWindowCloses":{"additionalProperties":false,"description":"Operation to suppress messages in the source stream until a window limit is reached","properties":{"bufferFullStrategy":{"description":"*(optional)* What to do when the buffer is full","enum":["emitEarlyWhenFull","shutdownWhenFull"]},"maxBytes":{"description":"*(optional)* The maximum number of bytes in the buffer","type":"string"},"maxRecords":{"description":"*(optional)* The maximum number of records in the buffer","type":"string"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["suppress"]},"until":{"description":"The until of the Operation to suppress messages in the source stream until a certain limit is reached","enum":["windowCloses"]}},"required":["until","type"],"title":"SuppressOperationUntilWindowCloses","type":"object"},"TableDefinition":{"additionalProperties":false,"description":"Contains a definition of a Table, which can be referenced by producers and pipelines","properties":{"keyType":{"description":"*(optional)* The key type of the table","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* KeyValue state store definition"},"topic":{"description":"The name of the Kafka topic for this table","type":"string"},"valueType":{"description":"*(optional)* The value type of the table","type":"string"}},"required":["topic"],"title":"TableDefinition","type":"object"},"TableDefinitionSource":{"additionalProperties":false,"description":"Contains a definition of a Table, which can be referenced by producers and pipelines","properties":{"keyType":{"description":"The key type of the table","type":"string"},"offsetResetPolicy":{"description":"*(optional)* Policy that determines what to do when there is no initial offset in Kafka, or if the current offset does not exist any more on the server (e.g. because that data has been deleted)","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* KeyValue state store definition"},"timestampExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TimestampExtractorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function extracts the event time from a consumed record"},"topic":{"description":"The name of the Kafka topic for this table","type":"string"},"valueType":{"description":"The value type of the table","type":"string"}},"required":["topic","keyType","valueType"],"title":"TableDefinitionSource","type":"object"},"TimestampExtractorDefinition":{"additionalProperties":false,"description":"Defines a timestamp extractor function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the timestamp extractor"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the timestamp extractor. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the timestamp extractor. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the timestamp extractor. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the timestamp extractor","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the timestamp extractor. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["timestampExtractor"]}},"required":["type"],"title":"TimestampExtractorDefinition","type":"object"},"TimestampExtractorDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a timestamp extractor function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the timestamp extractor"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the timestamp extractor. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the timestamp extractor. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the timestamp extractor. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the timestamp extractor","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the timestamp extractor. Only required for function types, which are not pre-defined.","type":"string"}},"title":"TimestampExtractorDefinitionWithImplicitType","type":"object"},"ToStreamOperation":{"additionalProperties":false,"description":"Convert a Table into a Stream, optionally through a custom key transformer","properties":{"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyTransformerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that computes the output key for every record"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["toStream"]}},"required":["type"],"title":"ToStreamOperation","type":"object"},"ToTableOperation":{"additionalProperties":false,"description":"Convert a Stream into a Table","properties":{"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the result table"},"type":{"description":"The type of the operation","enum":["toTable"]}},"required":["type"],"title":"ToTableOperation","type":"object"},"ToTopicDefinition":{"additionalProperties":false,"description":"Writes out pipeline messages to a topic","properties":{"keyType":{"description":"*(optional)* The key type of the topic","type":"string"},"partitioner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamPartitionerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that partitions the records in the output topic"},"topic":{"description":"The name of the Kafka topic","type":"string"},"valueType":{"description":"*(optional)* The value type of the topic","type":"string"}},"required":["topic"],"title":"ToTopicDefinition","type":"object"},"ToTopicNameExtractorDefinition":{"additionalProperties":false,"description":"Writes out pipeline messages to a topic as given by a topic name extractor","properties":{"partitioner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamPartitionerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that partitions the records in the output topic"},"topicNameExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TopicNameExtractorDefinitionWithImplicitType","type":"object"}],"description":"Reference to a pre-defined topic name extractor, or an inline definition of a topic name extractor"}},"required":["topicNameExtractor"],"title":"ToTopicNameExtractorDefinition","type":"object"},"TopicDefinition":{"additionalProperties":false,"description":"Contains a definition of a Kafka topic, to be used by producers and pipelines","properties":{"keyType":{"description":"*(optional)* The key type of the topic","type":"string"},"topic":{"description":"The name of the Kafka topic","type":"string"},"valueType":{"description":"*(optional)* The value type of the topic","type":"string"}},"required":["topic"],"title":"TopicDefinition","type":"object"},"TopicDefinitionSource":{"additionalProperties":false,"description":"Contains a definition of a Kafka topic, to be used by producers and pipelines","properties":{"keyType":{"description":"The key type of the topic","type":"string"},"offsetResetPolicy":{"description":"*(optional)* Policy that determines what to do when there is no initial offset in Kafka, or if the current offset does not exist any more on the server (e.g. because that data has been deleted)","type":"string"},"timestampExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TimestampExtractorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function extracts the event time from a consumed record"},"topic":{"description":"The name of the Kafka topic","type":"string"},"valueType":{"description":"The value type of the topic","type":"string"}},"required":["topic","keyType","valueType"],"title":"TopicDefinitionSource","type":"object"},"TopicNameExtractorDefinition":{"additionalProperties":false,"description":"Defines a topic name extractor function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the topic name extractor"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the topic name extractor. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the topic name extractor. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the topic name extractor. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the topic name extractor","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the topic name extractor. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["topicNameExtractor"]}},"required":["type"],"title":"TopicNameExtractorDefinition","type":"object"},"TopicNameExtractorDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a topic name extractor function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the topic name extractor"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the topic name extractor. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the topic name extractor. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the topic name extractor. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the topic name extractor","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the topic name extractor. Only required for function types, which are not pre-defined.","type":"string"}},"title":"TopicNameExtractorDefinitionWithImplicitType","type":"object"},"TransformKeyOperation":{"additionalProperties":false,"description":"Convert the key of every record in the stream to another key","properties":{"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyTransformerDefinitionWithImplicitType","type":"object"}],"description":"A function that computes a new key for each record"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["transformKey","mapKey","selectKey"]}},"required":["mapper","type"],"title":"TransformKeyOperation","type":"object"},"TransformKeyValueOperation":{"additionalProperties":false,"description":"Convert the key/value of every record in the stream to another key/value","properties":{"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueTransformerDefinitionWithImplicitType","type":"object"}],"description":"A function that computes a new key/value for each record"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["mapKeyValue","map","transformKeyValue"]}},"required":["mapper","type"],"title":"TransformKeyValueOperation","type":"object"},"TransformKeyValueToKeyValueListOperation":{"additionalProperties":false,"description":"Convert a stream by transforming every record into a list of derived records","properties":{"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueToKeyValueListTransformerDefinitionWithImplicitType","type":"object"}],"description":"A function that converts every record of a stream to a list of output records."},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["transformKeyValueToKeyValueList","flatMap"]}},"required":["mapper","type"],"title":"TransformKeyValueToKeyValueListOperation","type":"object"},"TransformKeyValueToValueListOperation":{"additionalProperties":false,"description":"Convert every record in the stream to a list of output records with the same key","properties":{"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueToValueListTransformerDefinitionWithImplicitType","type":"object"}],"description":"A function that converts every key/value into a list of result values, each of which will be combined with the original key to form a new message in the output stream"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["transformKeyValueToValueList","flatMapValues"]}},"required":["mapper","type"],"title":"TransformKeyValueToValueListOperation","type":"object"},"TransformMetadataOperation":{"additionalProperties":false,"description":"Convert the metadata of every record in the stream","properties":{"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/MetadataTransformerDefinitionWithImplicitType","type":"object"}],"description":"A function that converts the metadata (Kafka headers, timestamp) of every record in the stream"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["transformMetadata"]}},"required":["mapper","type"],"title":"TransformMetadataOperation","type":"object"},"TransformValueOperation":{"additionalProperties":false,"description":"Convert the value of every record in the stream to another value","properties":{"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueTransformerDefinitionWithImplicitType","type":"object"}],"description":"A function that converts the value of every record into another value"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the transformed table (only applies to tables, ignored for streams)"},"type":{"description":"The type of the operation","enum":["mapValue","transformValue","mapValues"]}},"required":["mapper","type"],"title":"TransformValueOperation","type":"object"},"ValueJoinerDefinition":{"additionalProperties":false,"description":"Defines a value joiner function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the value joiner"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the value joiner. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the value joiner. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the value joiner. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the value joiner","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the value joiner. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["valueJoiner"]}},"required":["type"],"title":"ValueJoinerDefinition","type":"object"},"ValueJoinerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a value joiner function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the value joiner"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the value joiner. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the value joiner. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the value joiner. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the value joiner","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the value joiner. Only required for function types, which are not pre-defined.","type":"string"}},"title":"ValueJoinerDefinitionWithImplicitType","type":"object"},"ValueTransformerDefinition":{"additionalProperties":false,"description":"Defines a value transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the value transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the value transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the value transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the value transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the value transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the value transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the value transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"},"type":{"description":"The type of the function","enum":["valueTransformer"]}},"required":["type"],"title":"ValueTransformerDefinition","type":"object"},"ValueTransformerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a value transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the value transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the value transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the value transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the value transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the value transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the value transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the value transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"}},"title":"ValueTransformerDefinitionWithImplicitType","type":"object"},"WindowBySessionOperation":{"additionalProperties":false,"description":"Operation to window messages by session, configured by an inactivity gap","properties":{"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* (Tumbling, Hopping) The grace period, during which out-of-order records can still be processed"},"inactivityGap":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The inactivity gap, below which two messages are considered to be of the same session"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["windowBySession"]}},"required":["inactivityGap","type"],"title":"WindowBySessionOperation","type":"object"},"WindowByTimeOperationWithHoppingWindow":{"additionalProperties":false,"description":"Operation to window records based on time criteria","properties":{"advanceBy":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The amount of time to increase time windows by"},"duration":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The duration of time windows"},"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The grace period, during which out-of-order records can still be processed"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["windowByTime"]},"windowType":{"description":"The windowType of the time window","enum":["hopping"]}},"required":["duration","advanceBy","windowType","type"],"title":"WindowByTimeOperationWithHoppingWindow","type":"object"},"WindowByTimeOperationWithSlidingWindow":{"additionalProperties":false,"description":"Operation to window records based on time criteria","properties":{"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The grace period, during which out-of-order records can still be processed"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"timeDifference":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The maximum amount of time difference between two records"},"type":{"description":"The type of the operation","enum":["windowByTime"]},"windowType":{"description":"The windowType of the time window","enum":["sliding"]}},"required":["timeDifference","windowType","type"],"title":"WindowByTimeOperationWithSlidingWindow","type":"object"},"WindowByTimeOperationWithTumblingWindow":{"additionalProperties":false,"description":"Operation to window records based on time criteria","properties":{"duration":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The duration of time windows"},"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The grace period, during which out-of-order records can still be processed"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["windowByTime"]},"windowType":{"description":"The windowType of the time window","enum":["tumbling"]}},"required":["duration","windowType","type"],"title":"WindowByTimeOperationWithTumblingWindow","type":"object"},"WindowStateStoreDefinition":{"additionalProperties":false,"description":"Definition of a window state store","properties":{"caching":{"description":"*(optional)* \"true\" if changed to the window store need to be buffered and periodically released, \"false\" to emit all changes directly","type":"boolean"},"keyType":{"description":"*(optional)* The key type of the window store","type":"string"},"logging":{"description":"*(optional)* \"true\" if a changelog topic should be set up on Kafka for this window store, \"false\" otherwise","type":"boolean"},"name":{"description":"*(optional)* The name of the window store. If this field is not defined, then the name is derived from the context.","type":"string"},"persistent":{"description":"*(optional)* \"true\" if this window store needs to be stored on disk, \"false\" otherwise","type":"boolean"},"retainDuplicates":{"description":"*(optional)* Whether or not to retain duplicates","type":"boolean"},"retention":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The duration for which elements in the window store are retained"},"timestamped":{"description":"*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise","type":"boolean"},"type":{"description":"The type of the state store","enum":["window"]},"valueType":{"description":"*(optional)* The value type of the window store","type":"string"},"windowSize":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* Size of the windows (cannot be negative)"}},"required":["type"],"title":"WindowStateStoreDefinition","type":"object"},"WindowStateStoreDefinitionWithImplicitType":{"additionalProperties":false,"description":"Definition of a window state store","properties":{"caching":{"description":"*(optional)* \"true\" if changed to the window store need to be buffered and periodically released, \"false\" to emit all changes directly","type":"boolean"},"keyType":{"description":"*(optional)* The key type of the window store","type":"string"},"logging":{"description":"*(optional)* \"true\" if a changelog topic should be set up on Kafka for this window store, \"false\" otherwise","type":"boolean"},"name":{"description":"*(optional)* The name of the window store. If this field is not defined, then the name is derived from the context.","type":"string"},"persistent":{"description":"*(optional)* \"true\" if this window store needs to be stored on disk, \"false\" otherwise","type":"boolean"},"retainDuplicates":{"description":"*(optional)* Whether or not to retain duplicates","type":"boolean"},"retention":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The duration for which elements in the window store are retained"},"timestamped":{"description":"*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise","type":"boolean"},"type":{"description":"The type of the state store","enum":["window"]},"valueType":{"description":"*(optional)* The value type of the window store","type":"string"},"windowSize":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* Size of the windows (cannot be negative)"}},"title":"WindowStateStoreDefinitionWithImplicitType","type":"object"}},"description":"KSML definition","properties":{"functions":{"description":"*(optional)* Functions that can be referenced in producers and pipelines","patternProperties":{"^[a-zA-Z0-9_]+$":{"anyOf":[{"$ref":"#/definitions/AggregatorDefinition","type":"object"},{"$ref":"#/definitions/ForEachActionDefinition","type":"object"},{"$ref":"#/definitions/ForeignKeyExtractorDefinition","type":"object"},{"$ref":"#/definitions/GeneratorDefinition","type":"object"},{"$ref":"#/definitions/GenericFunctionDefinitionWithImplicitType","type":"object"},{"$ref":"#/definitions/InitializerDefinition","type":"object"},{"$ref":"#/definitions/KeyTransformerDefinition","type":"object"},{"$ref":"#/definitions/KeyValueMapperDefinition","type":"object"},{"$ref":"#/definitions/KeyValuePrinterDefinition","type":"object"},{"$ref":"#/definitions/KeyValueToKeyValueListTransformerDefinition","type":"object"},{"$ref":"#/definitions/KeyValueToValueListTransformerDefinition","type":"object"},{"$ref":"#/definitions/KeyValueTransformerDefinition","type":"object"},{"$ref":"#/definitions/MergerDefinition","type":"object"},{"$ref":"#/definitions/MetadataTransformerDefinition","type":"object"},{"$ref":"#/definitions/PredicateDefinition","type":"object"},{"$ref":"#/definitions/ReducerDefinition","type":"object"},{"$ref":"#/definitions/StreamPartitionerDefinition","type":"object"},{"$ref":"#/definitions/TimestampExtractorDefinition","type":"object"},{"$ref":"#/definitions/TopicNameExtractorDefinition","type":"object"},{"$ref":"#/definitions/ValueJoinerDefinition","type":"object"},{"$ref":"#/definitions/ValueTransformerDefinition","type":"object"}]}},"type":"object"},"globalTables":{"description":"*(optional)* GlobalTables that can be referenced in producers and pipelines","patternProperties":{"^[a-zA-Z0-9_]+$":{"$ref":"#/definitions/GlobalTableDefinitionSource","type":"object"}},"type":"object"},"pipelines":{"description":"*(optional)* Collection of named pipelines","patternProperties":{"^[a-zA-Z0-9_]+$":{"$ref":"#/definitions/PipelineDefinition","type":"object"}},"type":"object"},"producers":{"description":"*(optional)* Collection of named producers","patternProperties":{"^[a-zA-Z0-9_]+$":{"$ref":"#/definitions/ProducerDefinition","type":"object"}},"type":"object"},"stores":{"description":"*(optional)* State stores that can be referenced in producers and pipelines","patternProperties":{"^[a-zA-Z0-9_]+$":{"anyOf":[{"$ref":"#/definitions/KeyValueStateStoreDefinition","type":"object"},{"$ref":"#/definitions/SessionStateStoreDefinition","type":"object"},{"$ref":"#/definitions/WindowStateStoreDefinition","type":"object"}]}},"type":"object"},"streams":{"description":"*(optional)* Streams that can be referenced in producers and pipelines","patternProperties":{"^[a-zA-Z0-9_]+$":{"$ref":"#/definitions/StreamDefinitionSource","type":"object"}},"type":"object"},"tables":{"description":"*(optional)* Tables that can be referenced in producers and pipelines","patternProperties":{"^[a-zA-Z0-9_]+$":{"$ref":"#/definitions/TableDefinitionSource","type":"object"}},"type":"object"}},"title":"TopologyDefinition","type":"object"} +{ + "additionalProperties" : false, + "definitions" : { + "AggregateOperation" : { + "additionalProperties" : false, + "description" : "An aggregate operation", + "properties" : { + "adder" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/AggregatorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* (GroupedTable) A function that adds a record to the aggregation result" + }, + "aggregator" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/AggregatorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* (GroupedStream, SessionWindowedStream, TimeWindowedStream) The aggregator function, which combines a value with the previous aggregation result and outputs a new aggregation result" + }, + "initializer" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/InitializerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "The initializer function, which generates an initial value for every set of aggregated records" + }, + "merger" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/MergerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* (SessionWindowedStream, SessionWindowedCogroupedStream) A function that combines two aggregation results" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the result aggregation" + }, + "subtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/AggregatorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* (GroupedTable) A function that removes a record from the aggregation result" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "aggregate" ] + } + }, + "required" : [ "initializer", "type" ], + "title" : "AggregateOperation", + "type" : "object" + }, + "AggregatorDefinition" : { + "additionalProperties" : false, + "description" : "Defines a aggregator function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the aggregator" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the aggregator. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the aggregator. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the aggregator. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the aggregator", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the aggregator. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "aggregator" ] + } + }, + "required" : [ "type" ], + "title" : "AggregatorDefinition", + "type" : "object" + }, + "AggregatorDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a aggregator function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the aggregator" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the aggregator. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the aggregator. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the aggregator. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the aggregator", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the aggregator. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "AggregatorDefinitionWithImplicitType", + "type" : "object" + }, + "BranchDefinitionWithPipeline" : { + "additionalProperties" : false, + "description" : "Defines a branch with sub-pipeline in a BranchOperation", + "properties" : { + "as" : { + "description" : "*(optional)* The name to register the pipeline result under, which can be used as source by follow-up pipelines", + "type" : "string" + }, + "branch" : { + "description" : "*(optional)* Defines a single branch, consisting of a condition and a pipeline to execute for messages that fulfil the predicate", + "items" : { + "$ref" : "#/definitions/StringOrInlinePredicateDefinitionWithImplicitType", + "type" : "object" + }, + "type" : "array" + }, + "forEach" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ForEachActionDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that gets called for every message in the stream" + }, + "if" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/PredicateDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Defines the condition under which messages get sent down this branch" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "print" : { + "$ref" : "#/definitions/PrintOperation", + "description" : "*(optional)* The specification of where to print messages to", + "type" : "object" + }, + "to" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ToTopicDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Ends the pipeline by sending all messages to a stream, table or globalTable, or to an inline defined output topic and optional partitioner" + }, + "toTopicNameExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ToTopicNameExtractorDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Ends the pipeline by sending all messages to a topic provided by a pre-defined topic name extractor function, or to a topic provided by an inline defined topic name extractor and optional partitioner" + }, + "via" : { + "description" : "*(optional)* A series of operations performed on the input stream", + "items" : { + "anyOf" : [ { + "$ref" : "#/definitions/AggregateOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/CogroupOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ConvertKeyOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ConvertKeyValueOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ConvertValueOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/CountOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/FilterNotOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/FilterOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/GroupByKeyOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/GroupByOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/JoinOperationWithGlobalTable", + "type" : "object" + }, { + "$ref" : "#/definitions/JoinOperationWithStream", + "type" : "object" + }, { + "$ref" : "#/definitions/JoinOperationWithTable", + "type" : "object" + }, { + "$ref" : "#/definitions/LeftJoinOperationWithGlobalTable", + "type" : "object" + }, { + "$ref" : "#/definitions/LeftJoinOperationWithStream", + "type" : "object" + }, { + "$ref" : "#/definitions/LeftJoinOperationWithTable", + "type" : "object" + }, { + "$ref" : "#/definitions/MergeOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/OuterJoinOperationWithStream", + "type" : "object" + }, { + "$ref" : "#/definitions/OuterJoinOperationWithTable", + "type" : "object" + }, { + "$ref" : "#/definitions/PeekOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ReduceOperationWithAdderAndSubtractor", + "type" : "object" + }, { + "$ref" : "#/definitions/ReduceOperationWithReducer", + "type" : "object" + }, { + "$ref" : "#/definitions/RepartitionOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/SuppressOperationUntilTimeLimit", + "type" : "object" + }, { + "$ref" : "#/definitions/SuppressOperationUntilWindowCloses", + "type" : "object" + }, { + "$ref" : "#/definitions/ToStreamOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ToTableOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformKeyOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformKeyValueOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformKeyValueToKeyValueListOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformKeyValueToValueListOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformMetadataOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformValueOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowBySessionOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowByTimeOperationWithHoppingWindow", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowByTimeOperationWithSlidingWindow", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowByTimeOperationWithTumblingWindow", + "type" : "object" + } ] + }, + "type" : "array" + } + }, + "title" : "BranchDefinitionWithPipeline", + "type" : "object" + }, + "CogroupOperation" : { + "additionalProperties" : false, + "description" : "A cogroup operation", + "properties" : { + "aggregator" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/AggregatorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "(GroupedStream, SessionWindowedStream, TimeWindowedStream) The aggregator function, which combines a value with the previous aggregation result and outputs a new aggregation result" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the co-grouped stream" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "cogroup" ] + } + }, + "required" : [ "aggregator", "type" ], + "title" : "CogroupOperation", + "type" : "object" + }, + "ConvertKeyOperation" : { + "additionalProperties" : false, + "description" : "An operation to convert the stream key type to another type. Conversion is only syntactic, eg. from Avro to XML.", + "properties" : { + "into" : { + "description" : "The type to convert the stream key into", + "type" : "string" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "convertKey" ] + } + }, + "required" : [ "into", "type" ], + "title" : "ConvertKeyOperation", + "type" : "object" + }, + "ConvertKeyValueOperation" : { + "additionalProperties" : false, + "description" : "An operation to convert the stream key and value types to other types. Conversion is only syntactic, eg. from Avro to XML.", + "properties" : { + "into" : { + "description" : "The tuple type to convert the stream key/value into", + "type" : "string" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "convertKeyValue" ] + } + }, + "required" : [ "into", "type" ], + "title" : "ConvertKeyValueOperation", + "type" : "object" + }, + "ConvertValueOperation" : { + "additionalProperties" : false, + "description" : "An operation to convert the stream value type to another type. Conversion is only syntactic, eg. from Avro to XML.", + "properties" : { + "into" : { + "description" : "The type to convert the stream value into", + "type" : "string" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "convertValue" ] + } + }, + "required" : [ "into", "type" ], + "title" : "ConvertValueOperation", + "type" : "object" + }, + "CountOperation" : { + "additionalProperties" : false, + "description" : "Count the number of times a key is seen in a given window", + "properties" : { + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/SessionStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the count operation's result" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "count" ] + } + }, + "required" : [ "type" ], + "title" : "CountOperation", + "type" : "object" + }, + "FilterNotOperation" : { + "additionalProperties" : false, + "description" : "Filter records based on the inverse result of a predicate function", + "properties" : { + "if" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/PredicateDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that returns \"false\" when records are accepted, \"true\" otherwise" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the filtered table (only applies to tables, ignored for streams)" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "filterNot" ] + } + }, + "required" : [ "if", "type" ], + "title" : "FilterNotOperation", + "type" : "object" + }, + "FilterOperation" : { + "additionalProperties" : false, + "description" : "Filter records based on a predicate function", + "properties" : { + "if" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/PredicateDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that returns \"true\" when records are accepted, \"false\" otherwise" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the filtered table (only applies to tables, ignored for streams)" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "filter" ] + } + }, + "required" : [ "if", "type" ], + "title" : "FilterOperation", + "type" : "object" + }, + "ForEachActionDefinition" : { + "additionalProperties" : false, + "description" : "Defines a foreach action function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the foreach action" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the foreach action. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the foreach action. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the foreach action. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the foreach action", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the foreach action. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the foreach action uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "forEach" ] + } + }, + "required" : [ "type" ], + "title" : "ForEachActionDefinition", + "type" : "object" + }, + "ForEachActionDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a foreach action function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the foreach action" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the foreach action. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the foreach action. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the foreach action. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the foreach action", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the foreach action. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the foreach action uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + } + }, + "title" : "ForEachActionDefinitionWithImplicitType", + "type" : "object" + }, + "ForeignKeyExtractorDefinition" : { + "additionalProperties" : false, + "description" : "Defines a foreign key extractor function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the foreign key extractor" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the foreign key extractor. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the foreign key extractor. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the foreign key extractor. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the foreign key extractor", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the foreign key extractor. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "foreignKeyExtractor" ] + } + }, + "required" : [ "type" ], + "title" : "ForeignKeyExtractorDefinition", + "type" : "object" + }, + "ForeignKeyExtractorDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a foreign key extractor function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the foreign key extractor" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the foreign key extractor. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the foreign key extractor. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the foreign key extractor. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the foreign key extractor", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the foreign key extractor. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "ForeignKeyExtractorDefinitionWithImplicitType", + "type" : "object" + }, + "GeneratorDefinition" : { + "additionalProperties" : false, + "description" : "Defines a message generator function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the message generator" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the message generator. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the message generator. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the message generator. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the message generator", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the message generator. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "generator" ] + } + }, + "required" : [ "type" ], + "title" : "GeneratorDefinition", + "type" : "object" + }, + "GeneratorDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a message generator function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the message generator" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the message generator. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the message generator. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the message generator. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the message generator", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the message generator. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "GeneratorDefinitionWithImplicitType", + "type" : "object" + }, + "GenericFunctionDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a generic function function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the generic function" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the generic function. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the generic function. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the generic function. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the generic function", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the generic function. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "generic" ] + } + }, + "title" : "GenericFunctionDefinitionWithImplicitType", + "type" : "object" + }, + "GlobalTableDefinition" : { + "additionalProperties" : false, + "description" : "Contains a definition of a GlobalTable, which can be referenced by producers and pipelines", + "properties" : { + "keyType" : { + "description" : "*(optional)* The key type of the global table", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* KeyValue state store definition" + }, + "topic" : { + "description" : "The name of the Kafka topic for this global table", + "type" : "string" + }, + "valueType" : { + "description" : "*(optional)* The value type of the global table", + "type" : "string" + } + }, + "required" : [ "topic" ], + "title" : "GlobalTableDefinition", + "type" : "object" + }, + "GlobalTableDefinitionSource" : { + "additionalProperties" : false, + "description" : "Contains a definition of a GlobalTable, which can be referenced by producers and pipelines", + "properties" : { + "keyType" : { + "description" : "The key type of the global table", + "type" : "string" + }, + "offsetResetPolicy" : { + "description" : "*(optional)* Policy that determines what to do when there is no initial offset in Kafka, or if the current offset does not exist any more on the server (e.g. because that data has been deleted)", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* KeyValue state store definition" + }, + "timestampExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TimestampExtractorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function extracts the event time from a consumed record" + }, + "topic" : { + "description" : "The name of the Kafka topic for this global table", + "type" : "string" + }, + "valueType" : { + "description" : "The value type of the global table", + "type" : "string" + } + }, + "required" : [ "topic", "keyType", "valueType" ], + "title" : "GlobalTableDefinitionSource", + "type" : "object" + }, + "GroupByKeyOperation" : { + "additionalProperties" : false, + "description" : "Operation to group all messages with the same key together", + "properties" : { + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the grouped stream" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "groupByKey" ] + } + }, + "required" : [ "type" ], + "title" : "GroupByKeyOperation", + "type" : "object" + }, + "GroupByOperation" : { + "additionalProperties" : false, + "description" : "Operation to group all messages with together based on a keying function", + "properties" : { + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueMapperDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "Function to map records to a key they can be grouped on" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the grouped stream or table" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "groupBy" ] + } + }, + "required" : [ "mapper", "type" ], + "title" : "GroupByOperation", + "type" : "object" + }, + "InitializerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a initializer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the initializer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the initializer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the initializer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the initializer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the initializer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the initializer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "initializer" ] + } + }, + "required" : [ "type" ], + "title" : "InitializerDefinition", + "type" : "object" + }, + "InitializerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a initializer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the initializer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the initializer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the initializer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the initializer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the initializer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the initializer. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "InitializerDefinitionWithImplicitType", + "type" : "object" + }, + "JoinOperationWithGlobalTable" : { + "additionalProperties" : false, + "description" : "Operation to join with a table", + "properties" : { + "globalTable" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/GlobalTableDefinition", + "type" : "object" + } ], + "description" : "A reference to the globalTable, or an inline definition of the globalTable to join with" + }, + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueMapperDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that maps the key value from the stream to the primary key type of the globalTable" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "join" ] + }, + "valueJoiner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueJoinerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that joins two values" + } + }, + "required" : [ "globalTable", "mapper", "valueJoiner", "type" ], + "title" : "JoinOperationWithGlobalTable", + "type" : "object" + }, + "JoinOperationWithStream" : { + "additionalProperties" : false, + "description" : "Operation to join with a stream", + "properties" : { + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/SessionStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the joined streams" + }, + "stream" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamDefinition", + "type" : "object" + } ], + "description" : "A reference to the Stream, or an inline definition of the stream to join with" + }, + "timeDifference" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The maximum time difference for a join over two streams on the same key" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "join" ] + }, + "valueJoiner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueJoinerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that joins two values" + } + }, + "required" : [ "stream", "valueJoiner", "timeDifference", "type" ], + "title" : "JoinOperationWithStream", + "type" : "object" + }, + "JoinOperationWithTable" : { + "additionalProperties" : false, + "description" : "Operation to join with a table", + "properties" : { + "foreignKeyExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ForeignKeyExtractorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that can translate the join table value to a primary key" + }, + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "otherPartitioner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamPartitionerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that partitions the records on the join table" + }, + "partitioner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamPartitionerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that partitions the records on the primary table" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/SessionStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the joined streams" + }, + "table" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TableDefinition", + "type" : "object" + } ], + "description" : "A reference to the table, or an inline definition of the table to join with" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "join" ] + }, + "valueJoiner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueJoinerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that joins two values" + } + }, + "required" : [ "table", "valueJoiner", "type" ], + "title" : "JoinOperationWithTable", + "type" : "object" + }, + "KeyTransformerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a key transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the key transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the key transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the key transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the key transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the key transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the key transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the key transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "keyTransformer" ] + } + }, + "required" : [ "type" ], + "title" : "KeyTransformerDefinition", + "type" : "object" + }, + "KeyTransformerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a key transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the key transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the key transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the key transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the key transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the key transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the key transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the key transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + } + }, + "title" : "KeyTransformerDefinitionWithImplicitType", + "type" : "object" + }, + "KeyValueMapperDefinition" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue mapper function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue mapper" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue mapper. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue mapper. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue mapper. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue mapper", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue mapper. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "keyValueMapper" ] + } + }, + "required" : [ "type" ], + "title" : "KeyValueMapperDefinition", + "type" : "object" + }, + "KeyValueMapperDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue mapper function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue mapper" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue mapper. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue mapper. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue mapper. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue mapper", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue mapper. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "KeyValueMapperDefinitionWithImplicitType", + "type" : "object" + }, + "KeyValuePrinterDefinition" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue printer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue printer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue printer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue printer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue printer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue printer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue printer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "keyValuePrinter" ] + } + }, + "required" : [ "type" ], + "title" : "KeyValuePrinterDefinition", + "type" : "object" + }, + "KeyValuePrinterDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue printer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue printer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue printer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue printer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue printer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue printer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue printer. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "KeyValuePrinterDefinitionWithImplicitType", + "type" : "object" + }, + "KeyValueStateStoreDefinition" : { + "additionalProperties" : false, + "description" : "Definition of a keyValue state store", + "properties" : { + "caching" : { + "description" : "*(optional)* \"true\" if changed to the keyValue store need to be buffered and periodically released, \"false\" to emit all changes directly", + "type" : "boolean" + }, + "historyRetention" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* (Versioned only) The duration for which old record versions are available for query (cannot be negative)" + }, + "keyType" : { + "description" : "*(optional)* The key type of the keyValue store", + "type" : "string" + }, + "logging" : { + "description" : "*(optional)* \"true\" if a changelog topic should be set up on Kafka for this keyValue store, \"false\" otherwise", + "type" : "boolean" + }, + "name" : { + "description" : "*(optional)* The name of the keyValue store. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "persistent" : { + "description" : "*(optional)* \"true\" if this keyValue store needs to be stored on disk, \"false\" otherwise", + "type" : "boolean" + }, + "segmentInterval" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Size of segments for storing old record versions (must be positive). Old record versions for the same key in a single segment are stored (updated and accessed) together. The only impact of this parameter is performance. If segments are large and a workload results in many record versions for the same key being collected in a single segment, performance may degrade as a result. On the other hand, historical reads (which access older segments) and out-of-order writes may slow down if there are too many segments." + }, + "timestamped" : { + "description" : "*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise", + "type" : "boolean" + }, + "type" : { + "description" : "The type of the state store", + "enum" : [ "keyValue" ] + }, + "valueType" : { + "description" : "*(optional)* The value type of the keyValue store", + "type" : "string" + }, + "versioned" : { + "description" : "*(optional)* \"true\" if elements in the store are versioned, \"false\" otherwise", + "type" : "boolean" + } + }, + "required" : [ "type" ], + "title" : "KeyValueStateStoreDefinition", + "type" : "object" + }, + "KeyValueStateStoreDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Definition of a keyValue state store", + "properties" : { + "caching" : { + "description" : "*(optional)* \"true\" if changed to the keyValue store need to be buffered and periodically released, \"false\" to emit all changes directly", + "type" : "boolean" + }, + "historyRetention" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* (Versioned only) The duration for which old record versions are available for query (cannot be negative)" + }, + "keyType" : { + "description" : "*(optional)* The key type of the keyValue store", + "type" : "string" + }, + "logging" : { + "description" : "*(optional)* \"true\" if a changelog topic should be set up on Kafka for this keyValue store, \"false\" otherwise", + "type" : "boolean" + }, + "name" : { + "description" : "*(optional)* The name of the keyValue store. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "persistent" : { + "description" : "*(optional)* \"true\" if this keyValue store needs to be stored on disk, \"false\" otherwise", + "type" : "boolean" + }, + "segmentInterval" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Size of segments for storing old record versions (must be positive). Old record versions for the same key in a single segment are stored (updated and accessed) together. The only impact of this parameter is performance. If segments are large and a workload results in many record versions for the same key being collected in a single segment, performance may degrade as a result. On the other hand, historical reads (which access older segments) and out-of-order writes may slow down if there are too many segments." + }, + "timestamped" : { + "description" : "*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise", + "type" : "boolean" + }, + "type" : { + "description" : "The type of the state store", + "enum" : [ "keyValue" ] + }, + "valueType" : { + "description" : "*(optional)* The value type of the keyValue store", + "type" : "string" + }, + "versioned" : { + "description" : "*(optional)* \"true\" if elements in the store are versioned, \"false\" otherwise", + "type" : "boolean" + } + }, + "title" : "KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + }, + "KeyValueToKeyValueListTransformerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue-to-keyvaluelist transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue-to-keyvaluelist transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue-to-keyvaluelist transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue-to-keyvaluelist transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue-to-keyvaluelist transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue-to-keyvaluelist transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue-to-keyvaluelist transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the keyvalue-to-keyvaluelist transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "keyValueToKeyValueListTransformer" ] + } + }, + "required" : [ "type" ], + "title" : "KeyValueToKeyValueListTransformerDefinition", + "type" : "object" + }, + "KeyValueToKeyValueListTransformerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue-to-keyvaluelist transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue-to-keyvaluelist transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue-to-keyvaluelist transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue-to-keyvaluelist transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue-to-keyvaluelist transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue-to-keyvaluelist transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue-to-keyvaluelist transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the keyvalue-to-keyvaluelist transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + } + }, + "title" : "KeyValueToKeyValueListTransformerDefinitionWithImplicitType", + "type" : "object" + }, + "KeyValueToValueListTransformerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue-to-valuelist transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue-to-valuelist transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue-to-valuelist transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue-to-valuelist transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue-to-valuelist transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue-to-valuelist transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue-to-valuelist transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the keyvalue-to-valuelist transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "keyValueToValueListTransformer" ] + } + }, + "required" : [ "type" ], + "title" : "KeyValueToValueListTransformerDefinition", + "type" : "object" + }, + "KeyValueToValueListTransformerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue-to-valuelist transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue-to-valuelist transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue-to-valuelist transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue-to-valuelist transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue-to-valuelist transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue-to-valuelist transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue-to-valuelist transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the keyvalue-to-valuelist transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + } + }, + "title" : "KeyValueToValueListTransformerDefinitionWithImplicitType", + "type" : "object" + }, + "KeyValueTransformerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the keyvalue transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "keyValueTransformer" ] + } + }, + "required" : [ "type" ], + "title" : "KeyValueTransformerDefinition", + "type" : "object" + }, + "KeyValueTransformerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the keyvalue transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + } + }, + "title" : "KeyValueTransformerDefinitionWithImplicitType", + "type" : "object" + }, + "LeftJoinOperationWithGlobalTable" : { + "additionalProperties" : false, + "description" : "Operation to leftJoin with a globalTable", + "properties" : { + "globalTable" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/GlobalTableDefinition", + "type" : "object" + } ], + "description" : "A reference to the globalTable, or an inline definition of the globalTable to join with" + }, + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueMapperDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that maps the key value from the stream with the primary key of the globalTable" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "leftJoin" ] + }, + "valueJoiner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueJoinerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that joins two values" + } + }, + "required" : [ "globalTable", "mapper", "valueJoiner", "type" ], + "title" : "LeftJoinOperationWithGlobalTable", + "type" : "object" + }, + "LeftJoinOperationWithStream" : { + "additionalProperties" : false, + "description" : "Operation to leftJoin with a stream", + "properties" : { + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/SessionStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the joined streams" + }, + "stream" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamDefinition", + "type" : "object" + } ], + "description" : "A reference to the stream, or an inline definition of the stream to leftJoin with" + }, + "timeDifference" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The maximum time difference for a leftJoin over two streams on the same key" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "leftJoin" ] + }, + "valueJoiner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueJoinerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that joins two values" + } + }, + "required" : [ "stream", "valueJoiner", "timeDifference", "type" ], + "title" : "LeftJoinOperationWithStream", + "type" : "object" + }, + "LeftJoinOperationWithTable" : { + "additionalProperties" : false, + "description" : "Operation to leftJoin with a table", + "properties" : { + "foreignKeyExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ForeignKeyExtractorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that can translate the join table value to a primary key" + }, + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "otherPartitioner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamPartitionerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that partitions the records on the join table" + }, + "partitioner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamPartitionerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that partitions the records on the primary table" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/SessionStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the joined streams" + }, + "table" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TableDefinition", + "type" : "object" + } ], + "description" : "A reference to the Table, or an inline definition of the Table to join with" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "leftJoin" ] + }, + "valueJoiner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueJoinerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that joins two values" + } + }, + "required" : [ "table", "valueJoiner", "type" ], + "title" : "LeftJoinOperationWithTable", + "type" : "object" + }, + "MergeOperation" : { + "additionalProperties" : false, + "description" : "A merge operation to join two Streams", + "properties" : { + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "stream" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamDefinitionSource", + "type" : "object" + } ], + "description" : "The stream to merge with" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "merge" ] + } + }, + "required" : [ "stream", "type" ], + "title" : "MergeOperation", + "type" : "object" + }, + "MergerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a merger function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the merger" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the merger. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the merger. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the merger. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the merger", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the merger. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "merger" ] + } + }, + "required" : [ "type" ], + "title" : "MergerDefinition", + "type" : "object" + }, + "MergerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a merger function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the merger" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the merger. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the merger. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the merger. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the merger", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the merger. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "MergerDefinitionWithImplicitType", + "type" : "object" + }, + "MetadataTransformerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a metadata transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the metadata transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the metadata transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the metadata transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the metadata transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the metadata transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the metadata transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the metadata transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "metadataTransformer" ] + } + }, + "required" : [ "type" ], + "title" : "MetadataTransformerDefinition", + "type" : "object" + }, + "MetadataTransformerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a metadata transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the metadata transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the metadata transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the metadata transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the metadata transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the metadata transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the metadata transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the metadata transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + } + }, + "title" : "MetadataTransformerDefinitionWithImplicitType", + "type" : "object" + }, + "OuterJoinOperationWithStream" : { + "additionalProperties" : false, + "description" : "Operation to outerJoin with a stream", + "properties" : { + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/SessionStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the outerJoined streams" + }, + "stream" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamDefinition", + "type" : "object" + } ], + "description" : "A reference to the stream, or an inline definition of the stream to outerJoin with" + }, + "timeDifference" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The maximum time difference for an outerJoin over two streams on the same key" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "outerJoin" ] + }, + "valueJoiner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueJoinerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that joins two values" + } + }, + "required" : [ "stream", "valueJoiner", "timeDifference", "type" ], + "title" : "OuterJoinOperationWithStream", + "type" : "object" + }, + "OuterJoinOperationWithTable" : { + "additionalProperties" : false, + "description" : "Operation to outerJoin with a table", + "properties" : { + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/SessionStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the outerJoined streams" + }, + "table" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TableDefinition", + "type" : "object" + } ], + "description" : "A reference to the table, or an inline definition of the table to outerJoin with" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "outerJoin" ] + }, + "valueJoiner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueJoinerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that joins two values" + } + }, + "required" : [ "table", "valueJoiner", "type" ], + "title" : "OuterJoinOperationWithTable", + "type" : "object" + }, + "ParameterDefinition" : { + "additionalProperties" : false, + "description" : "Defines a parameter for a user function", + "properties" : { + "defaultValue" : { + "description" : "*(optional)* The default value for the parameter", + "type" : "string" + }, + "name" : { + "description" : "The name of the parameter", + "type" : "string" + }, + "type" : { + "description" : "The type of the parameter", + "type" : "string" + } + }, + "required" : [ "name", "type" ], + "title" : "ParameterDefinition", + "type" : "object" + }, + "PeekOperation" : { + "additionalProperties" : false, + "description" : "Operation to peek into a stream, without modifying the stream contents", + "properties" : { + "forEach" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ForEachActionDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that gets called for every message in the stream" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "peek" ] + } + }, + "required" : [ "forEach", "type" ], + "title" : "PeekOperation", + "type" : "object" + }, + "PipelineDefinition" : { + "additionalProperties" : false, + "description" : "Defines a pipeline through a source, a series of operations to perform on it and a sink operation to close the stream with", + "properties" : { + "as" : { + "description" : "*(optional)* The name to register the pipeline result under, which can be used as source by follow-up pipelines", + "type" : "string" + }, + "branch" : { + "description" : "*(optional)* Defines a single branch, consisting of a condition and a pipeline to execute for messages that fulfil the predicate", + "items" : { + "$ref" : "#/definitions/BranchDefinitionWithPipeline", + "type" : "object" + }, + "type" : "array" + }, + "forEach" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ForEachActionDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that gets called for every message in the stream" + }, + "from" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TopicDefinitionSource", + "type" : "object" + } ], + "description" : "Pipeline source" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "print" : { + "$ref" : "#/definitions/PrintOperation", + "description" : "*(optional)* The specification of where to print messages to", + "type" : "object" + }, + "to" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ToTopicDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Ends the pipeline by sending all messages to a stream, table or globalTable, or to an inline defined output topic and optional partitioner" + }, + "toTopicNameExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ToTopicNameExtractorDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Ends the pipeline by sending all messages to a topic provided by a pre-defined topic name extractor function, or to a topic provided by an inline defined topic name extractor and optional partitioner" + }, + "via" : { + "description" : "*(optional)* A series of operations performed on the input stream", + "items" : { + "anyOf" : [ { + "$ref" : "#/definitions/AggregateOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/CogroupOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ConvertKeyOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ConvertKeyValueOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ConvertValueOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/CountOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/FilterNotOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/FilterOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/GroupByKeyOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/GroupByOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/JoinOperationWithGlobalTable", + "type" : "object" + }, { + "$ref" : "#/definitions/JoinOperationWithStream", + "type" : "object" + }, { + "$ref" : "#/definitions/JoinOperationWithTable", + "type" : "object" + }, { + "$ref" : "#/definitions/LeftJoinOperationWithGlobalTable", + "type" : "object" + }, { + "$ref" : "#/definitions/LeftJoinOperationWithStream", + "type" : "object" + }, { + "$ref" : "#/definitions/LeftJoinOperationWithTable", + "type" : "object" + }, { + "$ref" : "#/definitions/MergeOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/OuterJoinOperationWithStream", + "type" : "object" + }, { + "$ref" : "#/definitions/OuterJoinOperationWithTable", + "type" : "object" + }, { + "$ref" : "#/definitions/PeekOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ReduceOperationWithAdderAndSubtractor", + "type" : "object" + }, { + "$ref" : "#/definitions/ReduceOperationWithReducer", + "type" : "object" + }, { + "$ref" : "#/definitions/RepartitionOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/SuppressOperationUntilTimeLimit", + "type" : "object" + }, { + "$ref" : "#/definitions/SuppressOperationUntilWindowCloses", + "type" : "object" + }, { + "$ref" : "#/definitions/ToStreamOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ToTableOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformKeyOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformKeyValueOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformKeyValueToKeyValueListOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformKeyValueToValueListOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformMetadataOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformValueOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowBySessionOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowByTimeOperationWithHoppingWindow", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowByTimeOperationWithSlidingWindow", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowByTimeOperationWithTumblingWindow", + "type" : "object" + } ] + }, + "type" : "array" + } + }, + "required" : [ "from" ], + "title" : "PipelineDefinition", + "type" : "object" + }, + "PredicateDefinition" : { + "additionalProperties" : false, + "description" : "Defines a predicate function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the predicate" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the predicate. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the predicate. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the predicate. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the predicate", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the predicate. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the predicate uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "predicate" ] + } + }, + "required" : [ "type" ], + "title" : "PredicateDefinition", + "type" : "object" + }, + "PredicateDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a predicate function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the predicate" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the predicate. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the predicate. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the predicate. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the predicate", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the predicate. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the predicate uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + } + }, + "title" : "PredicateDefinitionWithImplicitType", + "type" : "object" + }, + "PrintOperation" : { + "additionalProperties" : false, + "description" : "Operation to print the contents of a pipeline on the screen or to write them to a file", + "properties" : { + "filename" : { + "description" : "*(optional)* The filename to output records to. If nothing is specified, then messages will be printed on stdout.", + "type" : "string" + }, + "label" : { + "description" : "*(optional)* A label to attach to the output records", + "type" : "string" + }, + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValuePrinterDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function to convert record into a string for output" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + } + }, + "title" : "PrintOperation", + "type" : "object" + }, + "ProducerDefinition" : { + "additionalProperties" : false, + "description" : "Definition of a Producer that regularly generates messages for a topic", + "properties" : { + "batchSize" : { + "description" : "*(optional)* The size of batches", + "type" : "number" + }, + "condition" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/PredicateDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that validates the generator's result message. Returns \"true\" when the message may be produced on the topic, \"false\" otherwise." + }, + "count" : { + "description" : "*(optional)* The number of messages to produce.", + "type" : "number" + }, + "generator" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/GeneratorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "The function that generates records" + }, + "interval" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The interval with which the generator is called" + }, + "to" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TopicDefinition", + "type" : "object" + } ], + "description" : "The topic to produce to" + }, + "until" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/PredicateDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A predicate that returns true to indicate producing should stop." + } + }, + "required" : [ "generator", "to" ], + "title" : "ProducerDefinition", + "type" : "object" + }, + "ReduceOperationWithAdderAndSubtractor" : { + "additionalProperties" : false, + "description" : "Operation to reduce a series of records into a single aggregate result", + "properties" : { + "adder" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ReducerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that adds a record to the aggregate result" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the aggregation" + }, + "subtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ReducerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that removes a record from the aggregate result" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "reduce" ] + } + }, + "required" : [ "adder", "subtractor", "type" ], + "title" : "ReduceOperationWithAdderAndSubtractor", + "type" : "object" + }, + "ReduceOperationWithReducer" : { + "additionalProperties" : false, + "description" : "Operation to reduce a series of records into a single aggregate result", + "properties" : { + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "reducer" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ReducerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that computes a new aggregate result" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the aggregation" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "reduce" ] + } + }, + "required" : [ "reducer", "type" ], + "title" : "ReduceOperationWithReducer", + "type" : "object" + }, + "ReducerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a reducer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the reducer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the reducer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the reducer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the reducer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the reducer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the reducer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "reducer" ] + } + }, + "required" : [ "type" ], + "title" : "ReducerDefinition", + "type" : "object" + }, + "ReducerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a reducer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the reducer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the reducer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the reducer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the reducer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the reducer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the reducer. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "ReducerDefinitionWithImplicitType", + "type" : "object" + }, + "RepartitionOperation" : { + "additionalProperties" : false, + "description" : "Operation to (re)partition a stream", + "properties" : { + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "numberOfPartitions" : { + "description" : "*(optional)* The target number of partitions" + }, + "partitioner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamPartitionerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that partitions stream records" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "repartition" ] + } + }, + "required" : [ "type" ], + "title" : "RepartitionOperation", + "type" : "object" + }, + "SessionStateStoreDefinition" : { + "additionalProperties" : false, + "description" : "Definition of a session state store", + "properties" : { + "caching" : { + "description" : "*(optional)* \"true\" if changed to the session store need to be buffered and periodically released, \"false\" to emit all changes directly", + "type" : "boolean" + }, + "keyType" : { + "description" : "*(optional)* The key type of the session store", + "type" : "string" + }, + "logging" : { + "description" : "*(optional)* \"true\" if a changelog topic should be set up on Kafka for this session store, \"false\" otherwise", + "type" : "boolean" + }, + "name" : { + "description" : "*(optional)* The name of the session store. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "persistent" : { + "description" : "*(optional)* \"true\" if this session store needs to be stored on disk, \"false\" otherwise", + "type" : "boolean" + }, + "retention" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The duration for which elements in the session store are retained" + }, + "timestamped" : { + "description" : "*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise", + "type" : "boolean" + }, + "type" : { + "description" : "The type of the state store", + "enum" : [ "session" ] + }, + "valueType" : { + "description" : "*(optional)* The value type of the session store", + "type" : "string" + } + }, + "required" : [ "type" ], + "title" : "SessionStateStoreDefinition", + "type" : "object" + }, + "StreamDefinition" : { + "additionalProperties" : false, + "description" : "Contains a definition of a Stream, which can be referenced by producers and pipelines", + "properties" : { + "keyType" : { + "description" : "*(optional)* The key type of the stream", + "type" : "string" + }, + "topic" : { + "description" : "The name of the Kafka topic for this stream", + "type" : "string" + }, + "valueType" : { + "description" : "*(optional)* The value type of the stream", + "type" : "string" + } + }, + "required" : [ "topic" ], + "title" : "StreamDefinition", + "type" : "object" + }, + "StreamDefinitionSource" : { + "additionalProperties" : false, + "description" : "Contains a definition of a Stream, which can be referenced by producers and pipelines", + "properties" : { + "keyType" : { + "description" : "The key type of the stream", + "type" : "string" + }, + "offsetResetPolicy" : { + "description" : "*(optional)* Policy that determines what to do when there is no initial offset in Kafka, or if the current offset does not exist any more on the server (e.g. because that data has been deleted)", + "type" : "string" + }, + "timestampExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TimestampExtractorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function extracts the event time from a consumed record" + }, + "topic" : { + "description" : "The name of the Kafka topic for this stream", + "type" : "string" + }, + "valueType" : { + "description" : "The value type of the stream", + "type" : "string" + } + }, + "required" : [ "topic", "keyType", "valueType" ], + "title" : "StreamDefinitionSource", + "type" : "object" + }, + "StreamPartitionerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a stream partitioner function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the stream partitioner" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the stream partitioner. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the stream partitioner. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the stream partitioner. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the stream partitioner", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the stream partitioner. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "streamPartitioner" ] + } + }, + "required" : [ "type" ], + "title" : "StreamPartitionerDefinition", + "type" : "object" + }, + "StreamPartitionerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a stream partitioner function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the stream partitioner" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the stream partitioner. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the stream partitioner. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the stream partitioner. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the stream partitioner", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the stream partitioner. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "StreamPartitionerDefinitionWithImplicitType", + "type" : "object" + }, + "StringOrInlinePredicateDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines the condition under which messages get sent down this branch", + "properties" : { + "if" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/PredicateDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Defines the condition under which messages get sent down this branch" + } + }, + "title" : "StringOrInlinePredicateDefinitionWithImplicitType", + "type" : "object" + }, + "SuppressOperationUntilTimeLimit" : { + "additionalProperties" : false, + "description" : "Operation to suppress messages in the source stream until a time limit is reached", + "properties" : { + "bufferFullStrategy" : { + "description" : "*(optional)* What to do when the buffer is full", + "enum" : [ "emitEarlyWhenFull", "shutdownWhenFull" ] + }, + "duration" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The duration for which messages are suppressed" + }, + "maxBytes" : { + "description" : "*(optional)* The maximum number of bytes in the buffer", + "type" : "string" + }, + "maxRecords" : { + "description" : "*(optional)* The maximum number of records in the buffer", + "type" : "string" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "suppress" ] + }, + "until" : { + "description" : "The until of the Operation to suppress messages in the source stream until a certain limit is reached", + "enum" : [ "timeLimit" ] + } + }, + "required" : [ "duration", "until", "type" ], + "title" : "SuppressOperationUntilTimeLimit", + "type" : "object" + }, + "SuppressOperationUntilWindowCloses" : { + "additionalProperties" : false, + "description" : "Operation to suppress messages in the source stream until a window limit is reached", + "properties" : { + "bufferFullStrategy" : { + "description" : "*(optional)* What to do when the buffer is full", + "enum" : [ "emitEarlyWhenFull", "shutdownWhenFull" ] + }, + "maxBytes" : { + "description" : "*(optional)* The maximum number of bytes in the buffer", + "type" : "string" + }, + "maxRecords" : { + "description" : "*(optional)* The maximum number of records in the buffer", + "type" : "string" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "suppress" ] + }, + "until" : { + "description" : "The until of the Operation to suppress messages in the source stream until a certain limit is reached", + "enum" : [ "windowCloses" ] + } + }, + "required" : [ "until", "type" ], + "title" : "SuppressOperationUntilWindowCloses", + "type" : "object" + }, + "TableDefinition" : { + "additionalProperties" : false, + "description" : "Contains a definition of a Table, which can be referenced by producers and pipelines", + "properties" : { + "keyType" : { + "description" : "*(optional)* The key type of the table", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* KeyValue state store definition" + }, + "topic" : { + "description" : "The name of the Kafka topic for this table", + "type" : "string" + }, + "valueType" : { + "description" : "*(optional)* The value type of the table", + "type" : "string" + } + }, + "required" : [ "topic" ], + "title" : "TableDefinition", + "type" : "object" + }, + "TableDefinitionSource" : { + "additionalProperties" : false, + "description" : "Contains a definition of a Table, which can be referenced by producers and pipelines", + "properties" : { + "keyType" : { + "description" : "The key type of the table", + "type" : "string" + }, + "offsetResetPolicy" : { + "description" : "*(optional)* Policy that determines what to do when there is no initial offset in Kafka, or if the current offset does not exist any more on the server (e.g. because that data has been deleted)", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* KeyValue state store definition" + }, + "timestampExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TimestampExtractorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function extracts the event time from a consumed record" + }, + "topic" : { + "description" : "The name of the Kafka topic for this table", + "type" : "string" + }, + "valueType" : { + "description" : "The value type of the table", + "type" : "string" + } + }, + "required" : [ "topic", "keyType", "valueType" ], + "title" : "TableDefinitionSource", + "type" : "object" + }, + "TimestampExtractorDefinition" : { + "additionalProperties" : false, + "description" : "Defines a timestamp extractor function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the timestamp extractor" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the timestamp extractor. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the timestamp extractor. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the timestamp extractor. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the timestamp extractor", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the timestamp extractor. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "timestampExtractor" ] + } + }, + "required" : [ "type" ], + "title" : "TimestampExtractorDefinition", + "type" : "object" + }, + "TimestampExtractorDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a timestamp extractor function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the timestamp extractor" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the timestamp extractor. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the timestamp extractor. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the timestamp extractor. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the timestamp extractor", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the timestamp extractor. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "TimestampExtractorDefinitionWithImplicitType", + "type" : "object" + }, + "ToStreamOperation" : { + "additionalProperties" : false, + "description" : "Convert a Table into a Stream, optionally through a custom key transformer", + "properties" : { + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyTransformerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that computes the output key for every record" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "toStream" ] + } + }, + "required" : [ "type" ], + "title" : "ToStreamOperation", + "type" : "object" + }, + "ToTableOperation" : { + "additionalProperties" : false, + "description" : "Convert a Stream into a Table", + "properties" : { + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the result table" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "toTable" ] + } + }, + "required" : [ "type" ], + "title" : "ToTableOperation", + "type" : "object" + }, + "ToTopicDefinition" : { + "additionalProperties" : false, + "description" : "Writes out pipeline messages to a topic", + "properties" : { + "keyType" : { + "description" : "*(optional)* The key type of the topic", + "type" : "string" + }, + "partitioner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamPartitionerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that partitions the records in the output topic" + }, + "topic" : { + "description" : "The name of the Kafka topic", + "type" : "string" + }, + "valueType" : { + "description" : "*(optional)* The value type of the topic", + "type" : "string" + } + }, + "required" : [ "topic" ], + "title" : "ToTopicDefinition", + "type" : "object" + }, + "ToTopicNameExtractorDefinition" : { + "additionalProperties" : false, + "description" : "Writes out pipeline messages to a topic as given by a topic name extractor", + "properties" : { + "partitioner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamPartitionerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that partitions the records in the output topic" + }, + "topicNameExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TopicNameExtractorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "Reference to a pre-defined topic name extractor, or an inline definition of a topic name extractor" + } + }, + "required" : [ "topicNameExtractor" ], + "title" : "ToTopicNameExtractorDefinition", + "type" : "object" + }, + "TopicDefinition" : { + "additionalProperties" : false, + "description" : "Contains a definition of a Kafka topic, to be used by producers and pipelines", + "properties" : { + "keyType" : { + "description" : "*(optional)* The key type of the topic", + "type" : "string" + }, + "topic" : { + "description" : "The name of the Kafka topic", + "type" : "string" + }, + "valueType" : { + "description" : "*(optional)* The value type of the topic", + "type" : "string" + } + }, + "required" : [ "topic" ], + "title" : "TopicDefinition", + "type" : "object" + }, + "TopicDefinitionSource" : { + "additionalProperties" : false, + "description" : "Contains a definition of a Kafka topic, to be used by producers and pipelines", + "properties" : { + "keyType" : { + "description" : "The key type of the topic", + "type" : "string" + }, + "offsetResetPolicy" : { + "description" : "*(optional)* Policy that determines what to do when there is no initial offset in Kafka, or if the current offset does not exist any more on the server (e.g. because that data has been deleted)", + "type" : "string" + }, + "timestampExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TimestampExtractorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function extracts the event time from a consumed record" + }, + "topic" : { + "description" : "The name of the Kafka topic", + "type" : "string" + }, + "valueType" : { + "description" : "The value type of the topic", + "type" : "string" + } + }, + "required" : [ "topic", "keyType", "valueType" ], + "title" : "TopicDefinitionSource", + "type" : "object" + }, + "TopicNameExtractorDefinition" : { + "additionalProperties" : false, + "description" : "Defines a topic name extractor function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the topic name extractor" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the topic name extractor. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the topic name extractor. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the topic name extractor. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the topic name extractor", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the topic name extractor. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "topicNameExtractor" ] + } + }, + "required" : [ "type" ], + "title" : "TopicNameExtractorDefinition", + "type" : "object" + }, + "TopicNameExtractorDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a topic name extractor function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the topic name extractor" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the topic name extractor. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the topic name extractor. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the topic name extractor. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the topic name extractor", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the topic name extractor. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "TopicNameExtractorDefinitionWithImplicitType", + "type" : "object" + }, + "TransformKeyOperation" : { + "additionalProperties" : false, + "description" : "Convert the key of every record in the stream to another key", + "properties" : { + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyTransformerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that computes a new key for each record" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "transformKey", "mapKey", "selectKey" ] + } + }, + "required" : [ "mapper", "type" ], + "title" : "TransformKeyOperation", + "type" : "object" + }, + "TransformKeyValueOperation" : { + "additionalProperties" : false, + "description" : "Convert the key/value of every record in the stream to another key/value", + "properties" : { + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueTransformerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that computes a new key/value for each record" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "mapKeyValue", "map", "transformKeyValue" ] + } + }, + "required" : [ "mapper", "type" ], + "title" : "TransformKeyValueOperation", + "type" : "object" + }, + "TransformKeyValueToKeyValueListOperation" : { + "additionalProperties" : false, + "description" : "Convert a stream by transforming every record into a list of derived records", + "properties" : { + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueToKeyValueListTransformerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that converts every record of a stream to a list of output records." + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "transformKeyValueToKeyValueList", "flatMap" ] + } + }, + "required" : [ "mapper", "type" ], + "title" : "TransformKeyValueToKeyValueListOperation", + "type" : "object" + }, + "TransformKeyValueToValueListOperation" : { + "additionalProperties" : false, + "description" : "Convert every record in the stream to a list of output records with the same key", + "properties" : { + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueToValueListTransformerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that converts every key/value into a list of result values, each of which will be combined with the original key to form a new message in the output stream" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "transformKeyValueToValueList", "flatMapValues" ] + } + }, + "required" : [ "mapper", "type" ], + "title" : "TransformKeyValueToValueListOperation", + "type" : "object" + }, + "TransformMetadataOperation" : { + "additionalProperties" : false, + "description" : "Convert the metadata of every record in the stream", + "properties" : { + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/MetadataTransformerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that converts the metadata (Kafka headers, timestamp) of every record in the stream" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "transformMetadata" ] + } + }, + "required" : [ "mapper", "type" ], + "title" : "TransformMetadataOperation", + "type" : "object" + }, + "TransformValueOperation" : { + "additionalProperties" : false, + "description" : "Convert the value of every record in the stream to another value", + "properties" : { + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueTransformerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that converts the value of every record into another value" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the transformed table (only applies to tables, ignored for streams)" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "mapValue", "transformValue", "mapValues" ] + } + }, + "required" : [ "mapper", "type" ], + "title" : "TransformValueOperation", + "type" : "object" + }, + "ValueJoinerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a value joiner function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the value joiner" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the value joiner. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the value joiner. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the value joiner. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the value joiner", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the value joiner. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "valueJoiner" ] + } + }, + "required" : [ "type" ], + "title" : "ValueJoinerDefinition", + "type" : "object" + }, + "ValueJoinerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a value joiner function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the value joiner" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the value joiner. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the value joiner. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the value joiner. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the value joiner", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the value joiner. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "ValueJoinerDefinitionWithImplicitType", + "type" : "object" + }, + "ValueTransformerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a value transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the value transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the value transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the value transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the value transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the value transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the value transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the value transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "valueTransformer" ] + } + }, + "required" : [ "type" ], + "title" : "ValueTransformerDefinition", + "type" : "object" + }, + "ValueTransformerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a value transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the value transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the value transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the value transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the value transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the value transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the value transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the value transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + } + }, + "title" : "ValueTransformerDefinitionWithImplicitType", + "type" : "object" + }, + "WindowBySessionOperation" : { + "additionalProperties" : false, + "description" : "Operation to window messages by session, configured by an inactivity gap", + "properties" : { + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* (Tumbling, Hopping) The grace period, during which out-of-order records can still be processed" + }, + "inactivityGap" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The inactivity gap, below which two messages are considered to be of the same session" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "windowBySession" ] + } + }, + "required" : [ "inactivityGap", "type" ], + "title" : "WindowBySessionOperation", + "type" : "object" + }, + "WindowByTimeOperationWithHoppingWindow" : { + "additionalProperties" : false, + "description" : "Operation to window records based on time criteria", + "properties" : { + "advanceBy" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The amount of time to increase time windows by" + }, + "duration" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The duration of time windows" + }, + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The grace period, during which out-of-order records can still be processed" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "windowByTime" ] + }, + "windowType" : { + "description" : "The windowType of the time window", + "enum" : [ "hopping" ] + } + }, + "required" : [ "duration", "advanceBy", "windowType", "type" ], + "title" : "WindowByTimeOperationWithHoppingWindow", + "type" : "object" + }, + "WindowByTimeOperationWithSlidingWindow" : { + "additionalProperties" : false, + "description" : "Operation to window records based on time criteria", + "properties" : { + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The grace period, during which out-of-order records can still be processed" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "timeDifference" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The maximum amount of time difference between two records" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "windowByTime" ] + }, + "windowType" : { + "description" : "The windowType of the time window", + "enum" : [ "sliding" ] + } + }, + "required" : [ "timeDifference", "windowType", "type" ], + "title" : "WindowByTimeOperationWithSlidingWindow", + "type" : "object" + }, + "WindowByTimeOperationWithTumblingWindow" : { + "additionalProperties" : false, + "description" : "Operation to window records based on time criteria", + "properties" : { + "duration" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The duration of time windows" + }, + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The grace period, during which out-of-order records can still be processed" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "windowByTime" ] + }, + "windowType" : { + "description" : "The windowType of the time window", + "enum" : [ "tumbling" ] + } + }, + "required" : [ "duration", "windowType", "type" ], + "title" : "WindowByTimeOperationWithTumblingWindow", + "type" : "object" + }, + "WindowStateStoreDefinition" : { + "additionalProperties" : false, + "description" : "Definition of a window state store", + "properties" : { + "caching" : { + "description" : "*(optional)* \"true\" if changed to the window store need to be buffered and periodically released, \"false\" to emit all changes directly", + "type" : "boolean" + }, + "keyType" : { + "description" : "*(optional)* The key type of the window store", + "type" : "string" + }, + "logging" : { + "description" : "*(optional)* \"true\" if a changelog topic should be set up on Kafka for this window store, \"false\" otherwise", + "type" : "boolean" + }, + "name" : { + "description" : "*(optional)* The name of the window store. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "persistent" : { + "description" : "*(optional)* \"true\" if this window store needs to be stored on disk, \"false\" otherwise", + "type" : "boolean" + }, + "retainDuplicates" : { + "description" : "*(optional)* Whether or not to retain duplicates", + "type" : "boolean" + }, + "retention" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The duration for which elements in the window store are retained" + }, + "timestamped" : { + "description" : "*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise", + "type" : "boolean" + }, + "type" : { + "description" : "The type of the state store", + "enum" : [ "window" ] + }, + "valueType" : { + "description" : "*(optional)* The value type of the window store", + "type" : "string" + }, + "windowSize" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Size of the windows (cannot be negative)" + } + }, + "required" : [ "type" ], + "title" : "WindowStateStoreDefinition", + "type" : "object" + }, + "WindowStateStoreDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Definition of a window state store", + "properties" : { + "caching" : { + "description" : "*(optional)* \"true\" if changed to the window store need to be buffered and periodically released, \"false\" to emit all changes directly", + "type" : "boolean" + }, + "keyType" : { + "description" : "*(optional)* The key type of the window store", + "type" : "string" + }, + "logging" : { + "description" : "*(optional)* \"true\" if a changelog topic should be set up on Kafka for this window store, \"false\" otherwise", + "type" : "boolean" + }, + "name" : { + "description" : "*(optional)* The name of the window store. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "persistent" : { + "description" : "*(optional)* \"true\" if this window store needs to be stored on disk, \"false\" otherwise", + "type" : "boolean" + }, + "retainDuplicates" : { + "description" : "*(optional)* Whether or not to retain duplicates", + "type" : "boolean" + }, + "retention" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The duration for which elements in the window store are retained" + }, + "timestamped" : { + "description" : "*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise", + "type" : "boolean" + }, + "type" : { + "description" : "The type of the state store", + "enum" : [ "window" ] + }, + "valueType" : { + "description" : "*(optional)* The value type of the window store", + "type" : "string" + }, + "windowSize" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Size of the windows (cannot be negative)" + } + }, + "title" : "WindowStateStoreDefinitionWithImplicitType", + "type" : "object" + } + }, + "description" : "KSML definition", + "properties" : { + "functions" : { + "description" : "*(optional)* Functions that can be referenced in producers and pipelines", + "patternProperties" : { + "^[a-zA-Z0-9_]+$" : { + "anyOf" : [ { + "$ref" : "#/definitions/AggregatorDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/ForEachActionDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/ForeignKeyExtractorDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/GeneratorDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/GenericFunctionDefinitionWithImplicitType", + "type" : "object" + }, { + "$ref" : "#/definitions/InitializerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/KeyTransformerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/KeyValueMapperDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/KeyValuePrinterDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/KeyValueToKeyValueListTransformerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/KeyValueToValueListTransformerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/KeyValueTransformerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/MergerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/MetadataTransformerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/PredicateDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/ReducerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/StreamPartitionerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/TimestampExtractorDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/TopicNameExtractorDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/ValueJoinerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/ValueTransformerDefinition", + "type" : "object" + } ] + } + }, + "type" : "object" + }, + "globalTables" : { + "description" : "*(optional)* GlobalTables that can be referenced in producers and pipelines", + "patternProperties" : { + "^[a-zA-Z0-9_]+$" : { + "$ref" : "#/definitions/GlobalTableDefinitionSource", + "type" : "object" + } + }, + "type" : "object" + }, + "pipelines" : { + "description" : "*(optional)* Collection of named pipelines", + "patternProperties" : { + "^[a-zA-Z0-9_]+$" : { + "$ref" : "#/definitions/PipelineDefinition", + "type" : "object" + } + }, + "type" : "object" + }, + "producers" : { + "description" : "*(optional)* Collection of named producers", + "patternProperties" : { + "^[a-zA-Z0-9_]+$" : { + "$ref" : "#/definitions/ProducerDefinition", + "type" : "object" + } + }, + "type" : "object" + }, + "stores" : { + "description" : "*(optional)* State stores that can be referenced in producers and pipelines", + "patternProperties" : { + "^[a-zA-Z0-9_]+$" : { + "anyOf" : [ { + "$ref" : "#/definitions/KeyValueStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/SessionStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinition", + "type" : "object" + } ] + } + }, + "type" : "object" + }, + "streams" : { + "description" : "*(optional)* Streams that can be referenced in producers and pipelines", + "patternProperties" : { + "^[a-zA-Z0-9_]+$" : { + "$ref" : "#/definitions/StreamDefinitionSource", + "type" : "object" + } + }, + "type" : "object" + }, + "tables" : { + "description" : "*(optional)* Tables that can be referenced in producers and pipelines", + "patternProperties" : { + "^[a-zA-Z0-9_]+$" : { + "$ref" : "#/definitions/TableDefinitionSource", + "type" : "object" + } + }, + "type" : "object" + } + }, + "title" : "TopologyDefinition", + "type" : "object" +} diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonDataObjectConverter.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonDataObjectConverter.java index 726cdc0c..d3def555 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonDataObjectConverter.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonDataObjectConverter.java @@ -31,7 +31,7 @@ import io.axual.ksml.data.type.UnionType; public class JsonDataObjectConverter implements NotationConverter { - private static final JsonDataObjectMapper DATA_OBJECT_MAPPER = new JsonDataObjectMapper(); + private static final JsonDataObjectMapper DATA_OBJECT_MAPPER = new JsonDataObjectMapper(false); @Override public DataObject convert(DataObject value, UserType targetType) { diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonDataObjectMapper.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonDataObjectMapper.java index bf880467..81a9ea96 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonDataObjectMapper.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonDataObjectMapper.java @@ -28,17 +28,21 @@ public class JsonDataObjectMapper implements DataObjectMapper { private static final NativeDataObjectMapper NATIVE_MAPPER = new NativeDataObjectMapper(); - private static final StringMapper STRING_MAPPER = new JsonStringMapper(); + private final StringMapper stringMapper; + + public JsonDataObjectMapper(boolean prettyPrint) { + stringMapper = new JsonStringMapper(prettyPrint); + } @Override public DataObject toDataObject(DataType expected, String value) { - var object = STRING_MAPPER.fromString(value); + var object = stringMapper.fromString(value); return NATIVE_MAPPER.toDataObject(expected, object); } @Override public String fromDataObject(DataObject value) { var object = NATIVE_MAPPER.fromDataObject(value); - return STRING_MAPPER.toString(object); + return stringMapper.toString(object); } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonSchemaLoader.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonSchemaLoader.java index 8e68bad3..e068d51e 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonSchemaLoader.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonSchemaLoader.java @@ -26,7 +26,7 @@ @Slf4j public class JsonSchemaLoader extends SchemaLoader { - private static final JsonSchemaMapper MAPPER = new JsonSchemaMapper(); + private static final JsonSchemaMapper MAPPER = new JsonSchemaMapper(false); public JsonSchemaLoader(String schemaDirectory) { super("JSON", schemaDirectory, ".json"); diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonSchemaMapper.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonSchemaMapper.java index 70830f7d..fbd39a43 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonSchemaMapper.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonSchemaMapper.java @@ -35,7 +35,6 @@ import static io.axual.ksml.data.schema.DataField.NO_INDEX; public class JsonSchemaMapper implements DataSchemaMapper { - private static final JsonDataObjectMapper MAPPER = new JsonDataObjectMapper(); private static final String TITLE_NAME = "title"; private static final String DESCRIPTION_NAME = "description"; private static final String TYPE_NAME = "type"; @@ -55,11 +54,16 @@ public class JsonSchemaMapper implements DataSchemaMapper { private static final String NUMBER_TYPE = "number"; private static final String OBJECT_TYPE = "object"; private static final String STRING_TYPE = "string"; + private final JsonDataObjectMapper mapper; + + public JsonSchemaMapper(boolean prettyPrint) { + mapper = new JsonDataObjectMapper(prettyPrint); + } @Override public DataSchema toDataSchema(String namespace, String name, String value) { // Convert JSON to internal DataObject format - var schema = MAPPER.toDataObject(value); + var schema = mapper.toDataObject(value); if (schema instanceof DataStruct schemaStruct) { return toDataSchema(namespace, name, schemaStruct); } @@ -135,7 +139,7 @@ public String fromDataSchema(DataSchema schema) { final var result = fromDataSchema(structSchema); // First translate the schema into DataObjects // The use the mapper to convert it into JSON - return MAPPER.fromDataObject(result); + return mapper.fromDataObject(result); } return null; } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonStringMapper.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonStringMapper.java index 920db9e9..aa52406b 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonStringMapper.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonStringMapper.java @@ -29,8 +29,13 @@ import java.io.StringWriter; public class JsonStringMapper implements StringMapper { - protected final ObjectMapper mapper = new ObjectMapper(); private static final JsonNodeNativeMapper NATIVE_MAPPER = new JsonNodeNativeMapper(); + protected final ObjectMapper mapper = new ObjectMapper(); + private final boolean prettyPrint; + + public JsonStringMapper(boolean prettyPrint) { + this.prettyPrint = prettyPrint; + } @Override public Object fromString(String value) { @@ -48,7 +53,10 @@ public String toString(Object value) { if (value == null) return null; // Allow null as native input, return null string as output try { final var writer = new StringWriter(); - mapper.writeTree(mapper.createGenerator(writer), NATIVE_MAPPER.fromNative(value)); + final var generator = prettyPrint + ? mapper.writerWithDefaultPrettyPrinter().createGenerator(writer) + : mapper.createGenerator(writer); + mapper.writeTree(generator, NATIVE_MAPPER.fromNative(value)); return writer.toString(); } catch (IOException e) { throw new DataException("Can not convert object to JSON string: " + value, e); diff --git a/ksml-data/src/main/java/io/axual/ksml/data/serde/JsonSerde.java b/ksml-data/src/main/java/io/axual/ksml/data/serde/JsonSerde.java index 9613ef4b..28d79c92 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/serde/JsonSerde.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/serde/JsonSerde.java @@ -26,7 +26,7 @@ import io.axual.ksml.data.type.DataType; public class JsonSerde extends StringSerde { - private static final DataObjectMapper STRING_MAPPER = new JsonDataObjectMapper(); + private static final DataObjectMapper STRING_MAPPER = new JsonDataObjectMapper(false); public JsonSerde(NativeDataObjectMapper nativeMapper, DataType expectedType) { super(nativeMapper, STRING_MAPPER, expectedType); diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java index 6dbd3c32..dbaa549f 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java @@ -337,7 +337,7 @@ private static void checkForSchemaOutput(String[] args) { // KSML definitions with on stdout and exit if (args.length >= 1 && WRITE_KSML_SCHEMA_ARGUMENT.equals(args[0])) { final var parser = new TopologyDefinitionParser("dummy"); - final var schema = new JsonSchemaMapper().fromDataSchema(parser.schema()); + final var schema = new JsonSchemaMapper(true).fromDataSchema(parser.schema()); final var filename = args.length >= 2 ? args[1] : null; if (filename != null) { diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/AlwaysReschedule.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/AlwaysReschedule.java deleted file mode 100644 index c12f1192..00000000 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/AlwaysReschedule.java +++ /dev/null @@ -1,56 +0,0 @@ -package io.axual.ksml.runner.backend; - -/*- - * ========================LICENSE_START================================= - * KSML Runner - * %% - * Copyright (C) 2021 - 2024 Axual B.V. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * =========================LICENSE_END================================== - */ - -import io.axual.ksml.data.object.DataObject; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; - -public class AlwaysReschedule implements RescheduleStrategy { - - private final List others = new ArrayList<>(); - - private final Duration interval; - - public AlwaysReschedule(Duration interval) { - this.interval = interval; - } - - public void combine(RescheduleStrategy other) { - others.add(other); - } - - @Override - public boolean shouldReschedule(DataObject key, DataObject value) { - var reschedule = true; - for (var other: others) { - reschedule = reschedule && other.shouldReschedule(key, value); - } - return reschedule; - } - - @Override - public Duration interval() { - return interval; - } -} diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/CountingReschedule.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/CountingReschedule.java deleted file mode 100644 index e69addcf..00000000 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/CountingReschedule.java +++ /dev/null @@ -1,41 +0,0 @@ -package io.axual.ksml.runner.backend; - -/*- - * ========================LICENSE_START================================= - * KSML Runner - * %% - * Copyright (C) 2021 - 2024 Axual B.V. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * =========================LICENSE_END================================== - */ - -import io.axual.ksml.data.object.DataObject; - -public class CountingReschedule implements RescheduleStrategy { - - private int count; - - public CountingReschedule(int count) { - this.count = count; - } - - @Override - public boolean shouldReschedule(DataObject key, DataObject value) { - if (count > 0) { - count--; - return true; - } - return false; - } -} diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaProducerRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaProducerRunner.java index 74f512a8..df783141 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaProducerRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaProducerRunner.java @@ -20,6 +20,14 @@ * =========================LICENSE_END================================== */ +import io.axual.ksml.client.producer.ResolvingProducer; +import io.axual.ksml.generator.TopologyDefinition; +import io.axual.ksml.python.PythonContext; +import io.axual.ksml.python.PythonFunction; +import io.axual.ksml.runner.exception.RunnerException; +import io.axual.ksml.runner.producer.ExecutableProducer; +import io.axual.ksml.runner.producer.IntervalSchedule; +import lombok.Builder; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.common.serialization.ByteArraySerializer; import org.slf4j.Logger; @@ -29,13 +37,6 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; -import io.axual.ksml.client.producer.ResolvingProducer; -import io.axual.ksml.generator.TopologyDefinition; -import io.axual.ksml.python.PythonContext; -import io.axual.ksml.python.PythonFunction; -import io.axual.ksml.runner.exception.RunnerException; -import lombok.Builder; - import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; @@ -95,9 +96,12 @@ public void run() { while (!stopRunning.get() && !hasFailed.get() && scheduler.hasScheduledItems()) { var scheduledGenerator = scheduler.getScheduledItem(); if (scheduledGenerator != null) { - scheduledGenerator.producer().produceMessage(producer); + scheduledGenerator.producer().produceMessages(producer); if (scheduledGenerator.producer().shouldReschedule()) { - final long nextTime = scheduledGenerator.startTime() + scheduledGenerator.producer().interval().toMillis(); + final var interval = scheduledGenerator.producer().interval() != null + ? scheduledGenerator.producer().interval().toMillis() + : 0L; + final long nextTime = scheduledGenerator.startTime() + interval; scheduler.schedule(scheduledGenerator.producer(), nextTime); } } diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/RescheduleStrategy.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/RescheduleStrategy.java deleted file mode 100644 index 4f5c972c..00000000 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/RescheduleStrategy.java +++ /dev/null @@ -1,63 +0,0 @@ -package io.axual.ksml.runner.backend; - -/*- - * ========================LICENSE_START================================= - * KSML Runner - * %% - * Copyright (C) 2021 - 2024 Axual B.V. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * =========================LICENSE_END================================== - */ - -import io.axual.ksml.data.object.DataObject; -import io.axual.ksml.user.UserFunction; - -import java.time.Duration; - -/** - * Interface to indicate if an {@link ExecutableProducer} should be rescheduled. - */ -public interface RescheduleStrategy { - - /** - * Implementations indicate if the producer should be scheduled again. - * @return - */ - boolean shouldReschedule(DataObject key, DataObject value); - - /** - * The only implementation keeping track of interval is {@link AlwaysReschedule}; all others just return {@link Duration#ZERO}. - * @return the interval until the next reschedule. - */ - default Duration interval() { - return Duration.ZERO; - } - - static AlwaysReschedule always(Duration interval) { - return new AlwaysReschedule(interval); - } - - static RescheduleStrategy once() { - return new SingleShotReschedule(); - } - - static RescheduleStrategy counting(int count) { - return new CountingReschedule(count); - } - - static RescheduleStrategy until(UserFunction userFunction) { - return new UntilReschedule(userFunction); - } - -} diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/SingleShotReschedule.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/SingleShotReschedule.java deleted file mode 100644 index bf4c998f..00000000 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/SingleShotReschedule.java +++ /dev/null @@ -1,34 +0,0 @@ -package io.axual.ksml.runner.backend; - -/*- - * ========================LICENSE_START================================= - * KSML Runner - * %% - * Copyright (C) 2021 - 2024 Axual B.V. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * =========================LICENSE_END================================== - */ - -import io.axual.ksml.data.object.DataObject; - -/** - * Reschedule strategy which runs the function only once. - */ -public class SingleShotReschedule implements RescheduleStrategy { - - @Override - public boolean shouldReschedule(DataObject key, DataObject value) { - return false; - } -} diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/UntilReschedule.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/UntilReschedule.java deleted file mode 100644 index 1d410e00..00000000 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/UntilReschedule.java +++ /dev/null @@ -1,43 +0,0 @@ -package io.axual.ksml.runner.backend; - -/*- - * ========================LICENSE_START================================= - * KSML Runner - * %% - * Copyright (C) 2021 - 2024 Axual B.V. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * =========================LICENSE_END================================== - */ - -import io.axual.ksml.data.object.DataBoolean; -import io.axual.ksml.data.object.DataObject; -import io.axual.ksml.user.UserFunction; - -public class UntilReschedule implements RescheduleStrategy { - - private final UserFunction condition; - - public UntilReschedule(UserFunction condition) { - this.condition = condition; - } - - @Override - public boolean shouldReschedule(DataObject key, DataObject value) { - DataObject result = condition.call(key, value); - - // Note: "until" should NOT reschedule if the predicate became true, negate the result! - if (result instanceof DataBoolean resultBoolean) return !(resultBoolean.value()); - throw new io.axual.ksml.data.exception.ExecutionException("Producer condition did not return a boolean value: " + condition.name); - } -} diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java index e72f4ab2..c0268692 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java @@ -52,7 +52,7 @@ public class KSMLConfig { @JsonProperty("applicationServer") @Builder.Default - private ApplicationServerConfig applicationServer = DEFAULT_APPSERVER_CONFIG; + private ApplicationServerConfig applicationServerConfig = DEFAULT_APPSERVER_CONFIG; @JsonProperty("prometheus") @Builder.Default @Getter @@ -68,7 +68,7 @@ public class KSMLConfig { private boolean enablePipelines = true; @JsonProperty("errorHandling") - private KSMLErrorHandlingConfig errorHandling; + private KSMLErrorHandlingConfig errorHandlingConfig; @JsonProperty("definitions") private Map definitions; @JsonProperty("schemas") @@ -95,12 +95,12 @@ public String storageDirectory() { } public ApplicationServerConfig applicationServerConfig() { - return applicationServer; + return applicationServerConfig; } public KSMLErrorHandlingConfig errorHandlingConfig() { - if (errorHandling == null) return KSMLErrorHandlingConfig.builder().build(); - return errorHandling; + if (errorHandlingConfig == null) return KSMLErrorHandlingConfig.builder().build(); + return errorHandlingConfig; } public Map definitions() { diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java index 6c01c5f3..126a232b 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLErrorHandlingConfig.java @@ -22,18 +22,18 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Builder; -import lombok.Getter; -import lombok.Setter; +import lombok.Data; import lombok.extern.jackson.Jacksonized; @JsonIgnoreProperties(ignoreUnknown = true) @Builder @Jacksonized public class KSMLErrorHandlingConfig { - private ErrorHandlingConfig consume; - private ErrorHandlingConfig produce; - private ErrorHandlingConfig process; + private final ErrorHandlingConfig consume; + private final ErrorHandlingConfig produce; + private final ErrorHandlingConfig process; public ErrorHandlingConfig consumerErrorHandlingConfig() { if (consume == null) { @@ -71,12 +71,15 @@ private ErrorHandlingConfig defaultErrorHandlingConfig(String logger) { return errorHandlingConfig; } - @Setter - @Getter + @Data public static class ErrorHandlingConfig { + @JsonProperty("log") private boolean log = true; + @JsonProperty("logPayload") private boolean logPayload = false; + @JsonProperty("loggerName") private String loggerName; + @JsonProperty("handler") private Handler handler = Handler.STOP; public enum Handler { diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/ExecutableProducer.java b/ksml-runner/src/main/java/io/axual/ksml/runner/producer/ExecutableProducer.java similarity index 54% rename from ksml-runner/src/main/java/io/axual/ksml/runner/backend/ExecutableProducer.java rename to ksml-runner/src/main/java/io/axual/ksml/runner/producer/ExecutableProducer.java index 8ff4da8b..f6449c9a 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/ExecutableProducer.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/producer/ExecutableProducer.java @@ -1,4 +1,4 @@ -package io.axual.ksml.runner.backend; +package io.axual.ksml.runner.producer; /*- * ========================LICENSE_START================================= @@ -29,22 +29,26 @@ import io.axual.ksml.data.object.DataObject; import io.axual.ksml.data.object.DataTuple; import io.axual.ksml.data.tag.ContextTags; -import io.axual.ksml.definition.FunctionDefinition; +import io.axual.ksml.data.value.Pair; import io.axual.ksml.definition.ProducerDefinition; +import io.axual.ksml.exception.TopologyException; import io.axual.ksml.python.PythonContext; import io.axual.ksml.python.PythonFunction; import io.axual.ksml.user.UserFunction; import io.axual.ksml.user.UserGenerator; -import io.axual.ksml.user.UserPredicate; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.serialization.Serializer; import java.time.Duration; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; import static io.axual.ksml.data.notation.UserType.DEFAULT_NOTATION; @@ -56,35 +60,31 @@ public class ExecutableProducer { @Getter private final String name; private final UserGenerator generator; - private final UserPredicate condition; + private final ProducerStrategy producerStrategy; private final String topic; private final UserType keyType; private final UserType valueType; private final Serializer keySerializer; private final Serializer valueSerializer; - private final RescheduleStrategy rescheduleStrategy; - - private DataObject lastKey = DataNull.INSTANCE; - private DataObject lastValue = DataNull.INSTANCE; + private long batchCount = 0; + private boolean stopProducing = false; private ExecutableProducer(UserFunction generator, - UserFunction condition, + ProducerStrategy producerStrategy, ContextTags tags, String topic, UserType keyType, UserType valueType, Serializer keySerializer, - Serializer valueSerializer, - RescheduleStrategy rescheduleStrategy) { + Serializer valueSerializer) { this.name = generator.name; this.generator = new UserGenerator(generator, tags); - this.condition = condition != null ? new UserPredicate(condition, tags) : null; + this.producerStrategy = producerStrategy; this.topic = topic; this.keyType = keyType; this.valueType = valueType; this.keySerializer = keySerializer; this.valueSerializer = valueSerializer; - this.rescheduleStrategy = rescheduleStrategy; } /** @@ -99,90 +99,137 @@ private ExecutableProducer(UserFunction generator, */ public static ExecutableProducer forProducer(PythonContext context, String namespace, String name, ProducerDefinition producerDefinition, Map kafkaConfig) { final var target = producerDefinition.target(); - final var gen = producerDefinition.generator(); final var tags = new ContextTags().append("namespace", namespace); + + // Initialize the message generator + final var gen = producerDefinition.generator(); + if (gen == null) { + throw new TopologyException("Missing generator function for producer \"" + name + "\""); + } final var generator = gen.name() != null ? PythonFunction.forGenerator(context, namespace, gen.name(), gen) : PythonFunction.forGenerator(context, namespace, name, gen); - final var cond = producerDefinition.condition(); - final var condition = cond != null - ? cond.name() != null - ? PythonFunction.forPredicate(context, namespace, cond.name(), cond) - : PythonFunction.forPredicate(context, namespace, name, cond) - : null; + + // Initialize the producer strategy + final var producerStrategy = new ProducerStrategy(context, namespace, name, tags, producerDefinition); + + // Initialize serializers final var keySerde = NotationLibrary.get(target.keyType().notation()).serde(target.keyType().dataType(), true); final var keySerializer = new ResolvingSerializer<>(keySerde.serializer(), kafkaConfig); final var valueSerde = NotationLibrary.get(target.valueType().notation()).serde(target.valueType().dataType(), false); final var valueSerializer = new ResolvingSerializer<>(valueSerde.serializer(), kafkaConfig); - final var reschedulingStrategy = setupRescheduling(producerDefinition, context, namespace, name); - return new ExecutableProducer(generator, condition, tags, target.topic(), target.keyType(), target.valueType(), keySerializer, valueSerializer, reschedulingStrategy); + // Set up the producer + return new ExecutableProducer(generator, producerStrategy, tags, target.topic(), target.keyType(), target.valueType(), keySerializer, valueSerializer); + } + + public void produceMessages(Producer producer) { + final var messages = generateBatch(); + final var futures = new ArrayList>(); + try { + for (var message : messages) { + ProducerRecord record = new ProducerRecord<>( + topic, + message.left(), + message.right() + ); + futures.add(producer.send(record)); + } + + batchCount++; + + for (var future : futures) { + var metadata = future.get(); + if (metadata != null && metadata.hasOffset()) { + producerStrategy.successfullyProducedOneMessage(); + log.info("Produced message: producer={}, batch #{}, message #{}, topic={}, partition={}, offset={}", name, batchCount, producerStrategy.messagesProduced(), metadata.topic(), metadata.partition(), metadata.offset()); + } else { + log.error("Error producing message to topic {}", topic); + } + } + } catch (InterruptedException | ExecutionException e) { + throw new io.axual.ksml.data.exception.ExecutionException("Could not produce to topic " + topic, e); + } } - public void produceMessage(Producer producer) { + private List> generateBatch() { + final var result = new ArrayList>(); + for (int index = 0; index < producerStrategy.batchSize(); index++) { + Pair message = null; + for (int t = 0; t < 10; t++) { + message = generateMessage(); + if (message != null) break; + } + + if (message != null) { + final var key = message.left(); + final var value = message.right(); + + // Log the generated messages + final var keyStr = key != null ? key.toString(DataObject.Printer.EXTERNAL_TOP_SCHEMA).replace("\n", "\\\\n") : "null"; + final var valueStr = value != null ? value.toString(DataObject.Printer.EXTERNAL_TOP_SCHEMA).replace("\n", "\\\\n") : "null"; + log.info("Message: key={}, value={}", keyStr, valueStr); + + // Serialize the message + var serializedKey = keySerializer.serialize(topic, NATIVE_MAPPER.fromDataObject(key)); + var serializedValue = valueSerializer.serialize(topic, NATIVE_MAPPER.fromDataObject(value)); + + // Add the serialized message to the batch + result.add(new Pair<>(serializedKey, serializedValue)); + + // Check if this should be the last message produced + if (!producerStrategy.continueAfterMessage(key, value)) { + stopProducing = true; + break; + } + } else { + log.warn("Could not generate a valid message after 10 tries, skipping..."); + } + } + + // Return the batch of messages + return result; + } + + private Pair generateMessage() { DataObject result = generator.apply(); if (result instanceof DataTuple tuple && tuple.size() == 2) { var key = tuple.get(0); var value = tuple.get(1); - if (condition == null || condition.test(key, value)) { + if (producerStrategy.validateMessage(key, value)) { // keep produced key and value to determine rescheduling later - lastKey = key; - lastValue = value; - key = DATA_OBJECT_CONVERTER.convert(DEFAULT_NOTATION, key, keyType); value = DATA_OBJECT_CONVERTER.convert(DEFAULT_NOTATION, value, valueType); + var okay = true; if (key != null && !keyType.dataType().isAssignableFrom(key.type())) { - log.error("Can not convert {} to topic key type {}", key.type(), keyType); + log.error("Wrong topic key type: expected={} key={}", keyType, key.type()); okay = false; } if (value != null && !valueType.dataType().isAssignableFrom(value.type())) { - log.error("Can not convert {} to topic value type {}", value.type(), valueType); + log.error("Wrong topic value type: expected={} value={}", valueType, value.type()); okay = false; } if (okay) { - var keyStr = key != null ? key.toString() : "null"; - var valueStr = value != null ? value.toString() : "null"; - - keyStr = keyStr.replace("\n", "\\\\n"); - valueStr = valueStr.replace("\n", "\\\\n"); - log.info("Message: key={}, value={}", keyStr, valueStr); - - var serializedKey = keySerializer.serialize(topic, NATIVE_MAPPER.fromDataObject(key)); - var serializedValue = valueSerializer.serialize(topic, NATIVE_MAPPER.fromDataObject(value)); - ProducerRecord message = new ProducerRecord<>( - topic, - serializedKey, - serializedValue - ); - var future = producer.send(message); - try { - var metadata = future.get(); - if (metadata != null && metadata.hasOffset()) { - log.info("Produced message to {}, partition {}, offset {}", metadata.topic(), metadata.partition(), metadata.offset()); - } else { - log.error("Error producing message to topic {}", topic); - } - } catch (InterruptedException | ExecutionException e) { - throw new io.axual.ksml.data.exception.ExecutionException("Could not produce to topic " + topic, e); - } + return new Pair<>(key, value); } } else { - log.info("Condition FALSE, skipping message"); + log.warn("Skipping invalid message: key={} value={}", key, value); } } + return null; } /** - * Indicate if this instance wants to be rescheduled after its most recent run. + * Indicate if this producer wants to be rescheduled after its most recent run. * * @return true if should reschedule. */ public boolean shouldReschedule() { - return rescheduleStrategy.shouldReschedule(lastKey, lastValue); + return !stopProducing && producerStrategy.shouldReschedule(); } /** @@ -191,29 +238,6 @@ public boolean shouldReschedule() { * @return the desired wait until next run. */ public Duration interval() { - return rescheduleStrategy.interval(); - } - - private static RescheduleStrategy setupRescheduling(ProducerDefinition definition, PythonContext context, String namespace, String name) { - if (definition.interval() == null) { - return RescheduleStrategy.once(); - } - - AlwaysReschedule strategy = RescheduleStrategy.always(definition.interval()); - - if (definition.count() != null) { - // since the producer is always called first before calling the strategy, decrement count by 1 - strategy.combine(RescheduleStrategy.counting(definition.count() - 1)); - } - - if (definition.until() != null) { - FunctionDefinition untilDefinition = definition.until(); - final var untilFunction = untilDefinition.name() != null - ? PythonFunction.forPredicate(context, namespace, untilDefinition.name(), untilDefinition) - : PythonFunction.forPredicate(context, namespace, name, untilDefinition); - strategy.combine(RescheduleStrategy.until(untilFunction)); - } - - return strategy; + return producerStrategy.interval(); } } diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/IntervalSchedule.java b/ksml-runner/src/main/java/io/axual/ksml/runner/producer/IntervalSchedule.java similarity index 98% rename from ksml-runner/src/main/java/io/axual/ksml/runner/backend/IntervalSchedule.java rename to ksml-runner/src/main/java/io/axual/ksml/runner/producer/IntervalSchedule.java index 746345c4..87df0396 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/IntervalSchedule.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/producer/IntervalSchedule.java @@ -1,4 +1,4 @@ -package io.axual.ksml.runner.backend; +package io.axual.ksml.runner.producer; /*- * ========================LICENSE_START================================= diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/producer/ProducerStrategy.java b/ksml-runner/src/main/java/io/axual/ksml/runner/producer/ProducerStrategy.java new file mode 100644 index 00000000..2ff552aa --- /dev/null +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/producer/ProducerStrategy.java @@ -0,0 +1,126 @@ +package io.axual.ksml.runner.producer; + +/*- + * ========================LICENSE_START================================= + * KSML Runner + * %% + * Copyright (C) 2021 - 2024 Axual B.V. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * =========================LICENSE_END================================== + */ + +import io.axual.ksml.data.object.DataObject; +import io.axual.ksml.data.tag.ContextTags; +import io.axual.ksml.definition.FunctionDefinition; +import io.axual.ksml.definition.ProducerDefinition; +import io.axual.ksml.python.PythonContext; +import io.axual.ksml.python.PythonFunction; +import io.axual.ksml.user.UserPredicate; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.time.Duration; + +@Slf4j +public class ProducerStrategy { + private static final int INFINITE = -1; + + private final boolean once; + private final long messageCount; + private final long batchSize; + @Getter + private final Duration interval; + private final UserPredicate condition; + private final UserPredicate until; + @Getter + private long messagesProduced = 0; + private boolean untilTriggered = false; + + public ProducerStrategy(PythonContext context, String namespace, String name, ContextTags tags, ProducerDefinition definition) { + // If interval, messageCount and until are not defined, then only produce one message + once = (definition.interval() == null && definition.messageCount() == null && definition.until() == null); + if (once) { + messageCount = 1; + batchSize = 1; + interval = Duration.ofMillis(0); + condition = null; + until = null; + return; + } + + // Set the message count + messageCount = definition.messageCount() != null + ? definition.messageCount() >= 1 + ? definition.messageCount() + : 1 + : INFINITE; + + // Set the batch size + if (definition.batchSize() != null) { + if (definition.batchSize() >= 1 && definition.batchSize() <= 1000) { + batchSize = definition.batchSize(); + } else { + log.warn("Batch size must be between 1 and 1000. Using 1 as default."); + batchSize = 1; + } + } else { + batchSize = 1; + } + + // Set the interval + interval = definition.interval() != null ? definition.interval() : Duration.ofMillis(0); + + // Set up the message validator + condition = userPredicateFrom(definition.condition(), context, namespace, name, tags); + + // Set up the user function that - when returns true - halts the producer + until = userPredicateFrom(definition.until(), context, namespace, name, tags); + } + + private UserPredicate userPredicateFrom(FunctionDefinition function, PythonContext context, String namespace, String name, ContextTags tags) { + return function != null + ? function.name() != null + ? new UserPredicate(PythonFunction.forPredicate(context, namespace, function.name(), function), tags) + : new UserPredicate(PythonFunction.forPredicate(context, namespace, name, function), tags) + : null; + } + + public boolean shouldReschedule() { + // Reschedule if not once, not all messages were produced yet and until never returned true + return !once && (messageCount == INFINITE || messagesProduced < messageCount) && !untilTriggered; + } + + public boolean validateMessage(DataObject key, DataObject value) { + return (condition == null || condition.test(key, value)); + } + + public boolean continueAfterMessage(DataObject key, DataObject value) { + if (until != null) { + if (until.test(key, value)) { + untilTriggered = true; + return false; + } + } + return true; + } + + public long batchSize() { + if (messageCount == INFINITE) return batchSize; + return Math.min(batchSize, messageCount - messagesProduced); + } + + public void successfullyProducedOneMessage() { + messagesProduced++; + } +} diff --git a/ksml-runner/src/test/java/io/axual/ksml/runner/backend/IntervalScheduleTest.java b/ksml-runner/src/test/java/io/axual/ksml/runner/backend/IntervalScheduleTest.java index 7c2451ca..81990b29 100644 --- a/ksml-runner/src/test/java/io/axual/ksml/runner/backend/IntervalScheduleTest.java +++ b/ksml-runner/src/test/java/io/axual/ksml/runner/backend/IntervalScheduleTest.java @@ -20,6 +20,8 @@ * =========================LICENSE_END================================== */ +import io.axual.ksml.runner.producer.ExecutableProducer; +import io.axual.ksml.runner.producer.IntervalSchedule; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/ksml-runner/src/test/java/io/axual/ksml/runner/backend/KafkaProducerRunnerTest.java b/ksml-runner/src/test/java/io/axual/ksml/runner/backend/KafkaProducerRunnerTest.java index 535b0dcd..09c9705b 100644 --- a/ksml-runner/src/test/java/io/axual/ksml/runner/backend/KafkaProducerRunnerTest.java +++ b/ksml-runner/src/test/java/io/axual/ksml/runner/backend/KafkaProducerRunnerTest.java @@ -142,7 +142,7 @@ private Map loadDefinitions(String filename) throws /** * Create a KafkaProducerRunner from the given config, but with a mock Kafka producer. * - * @param config a {@link io.axual.ksml.runner.backend.KafkaProducerRunner.Config}. + * @param config a {@link KafkaProducerRunner.Config}. * @return a {@link KafkaProducerRunner} with a mocked producer. */ private KafkaProducerRunner runnerUnderTest(KafkaProducerRunner.Config config) { diff --git a/ksml/src/main/java/io/axual/ksml/definition/ProducerDefinition.java b/ksml/src/main/java/io/axual/ksml/definition/ProducerDefinition.java index b1ef21b2..d06deb4e 100644 --- a/ksml/src/main/java/io/axual/ksml/definition/ProducerDefinition.java +++ b/ksml/src/main/java/io/axual/ksml/definition/ProducerDefinition.java @@ -22,9 +22,13 @@ import java.time.Duration; -public record ProducerDefinition(FunctionDefinition generator, Duration interval, FunctionDefinition condition, - TopicDefinition target, Integer count, - FunctionDefinition until) implements Definition { +public record ProducerDefinition(FunctionDefinition generator, + FunctionDefinition condition, + FunctionDefinition until, + TopicDefinition target, + Long messageCount, + Long batchSize, + Duration interval) implements Definition { @Override public String toString() { return definitionType(); diff --git a/ksml/src/main/java/io/axual/ksml/definition/parser/ProducerDefinitionParser.java b/ksml/src/main/java/io/axual/ksml/definition/parser/ProducerDefinitionParser.java index af66320d..ca311c58 100644 --- a/ksml/src/main/java/io/axual/ksml/definition/parser/ProducerDefinitionParser.java +++ b/ksml/src/main/java/io/axual/ksml/definition/parser/ProducerDefinitionParser.java @@ -40,11 +40,12 @@ public StructsParser parser() { "", "Definition of a Producer that regularly generates messages for a topic", functionField(Producers.GENERATOR, "The function that generates records", new GeneratorDefinitionParser(false)), - durationField(Producers.INTERVAL, "The interval with which the generator is called"), optional(functionField(Producers.CONDITION, "A function that validates the generator's result message. Returns \"true\" when the message may be produced on the topic, \"false\" otherwise.", new PredicateDefinitionParser(false))), - topicField(Producers.TARGET, "The topic to produce to", new TopicDefinitionParser(resources(), false)), - optional(integerField(Producers.COUNT, "The number of messages to produce.")), optional(functionField(Producers.UNTIL, "A predicate that returns true to indicate producing should stop.", new PredicateDefinitionParser(false))), - (generator, interval, condition, target, count, until, tags) -> new ProducerDefinition(generator, interval, condition, target, count, until)); + topicField(Producers.TARGET, "The topic to produce to", new TopicDefinitionParser(resources(), false)), + optional(longField(Producers.COUNT, "The number of messages to produce.")), + optional(longField(Producers.BATCH_SIZE, 1L, "The size of batches")), + optional(durationField(Producers.INTERVAL, "The interval with which the generator is called")), + (generator, condition, until, target, count, batchSize, interval, tags) -> new ProducerDefinition(generator, condition, until, target, count, batchSize, interval)); } } diff --git a/ksml/src/main/java/io/axual/ksml/dsl/KSMLDSL.java b/ksml/src/main/java/io/axual/ksml/dsl/KSMLDSL.java index 596e3ecb..d0b5721c 100644 --- a/ksml/src/main/java/io/axual/ksml/dsl/KSMLDSL.java +++ b/ksml/src/main/java/io/axual/ksml/dsl/KSMLDSL.java @@ -74,6 +74,7 @@ public static class Parameters { @NoArgsConstructor(access = AccessLevel.PRIVATE) public static class Producers { public static final String GENERATOR = "generator"; + public static final String BATCH_SIZE = "batchSize"; public static final String INTERVAL = "interval"; public static final String CONDITION = "condition"; public static final String TARGET = "to"; diff --git a/ksml/src/main/java/io/axual/ksml/parser/DefinitionParser.java b/ksml/src/main/java/io/axual/ksml/parser/DefinitionParser.java index 2f7b47b9..cac26947 100644 --- a/ksml/src/main/java/io/axual/ksml/parser/DefinitionParser.java +++ b/ksml/src/main/java/io/axual/ksml/parser/DefinitionParser.java @@ -172,6 +172,14 @@ protected StructsParser integerField(String childName, Integer valueIfN return freeField(childName, valueIfNull, doc, ParserWithSchemas.of(ParseNode::asInt, DataSchema.integerSchema())); } + protected StructsParser longField(String childName, String doc) { + return longField(childName, null, doc); + } + + protected StructsParser longField(String childName, Long valueIfNull, String doc) { + return freeField(childName, valueIfNull, doc, ParserWithSchemas.of(ParseNode::asLong, DataSchema.longSchema())); + } + protected StructsParser stringField(String childName, String doc) { return stringField(childName, null, doc); } From 1325c9d4353b10288fbeb3683059bbccadacde93 Mon Sep 17 00:00:00 2001 From: Richard Bosch Date: Fri, 1 Nov 2024 10:07:47 +0100 Subject: [PATCH 16/55] Add header support to serialisation/deserialisation to allow Apicurio notation to use headers --- docker-compose.yml | 23 +++-- examples/00-example-generate-sensordata.yaml | 6 +- examples/01-example-inspect.yaml | 8 ++ examples/ksml-data-generator.yaml | 58 ++++++++---- examples/ksml-runner.yaml | 54 +++++++---- .../ksml/data/notation/avro/AvroNotation.java | 91 +++++++++++++++---- .../runner/backend/ExecutableProducer.java | 26 ++++-- 7 files changed, 187 insertions(+), 79 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 29f3bf96..8e185377 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ networks: services: broker: - image: bitnami/kafka:3.7.0 + image: bitnami/kafka:3.8.0 hostname: broker ports: - "9092:9092" @@ -40,7 +40,7 @@ services: start_period: 20s schema_registry: - image: confluentinc/cp-schema-registry:7.3.6 + image: apicurio/apicurio-registry:3.0.2 hostname: schema_registry depends_on: - broker @@ -50,12 +50,14 @@ services: - ksml restart: always environment: - SCHEMA_REGISTRY_HOST_NAME: schema_registry - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'PLAINTEXT://broker:9093' - SCHEMA_REGISTRY_KAFKASTORE_TOPIC_REPLICATION_FACTOR: 1 - SCHEMA_REGISTRY_ZK_NAMESPACE: 'mySRNS' + QUARKUS_HTTP_PORT: 8081 + QUARKUS_HTTP_CORS_ORIGINS: '*' + QUARKUS_PROFILE: "prod" + APICURIO_STORAGE_KIND: kafkasql + APICURIO_KAFKASQL_BOOTSTRAP_SERVERS: 'broker:9093' + APICURIO_KAFKASQL_TOPIC: '_apciurio-kafkasql-store' healthcheck: - test: curl http://localhost:8081 | fgrep -q "{}" + test: curl http://localhost:8081/apis interval: 30s timeout: 10s retries: 5 @@ -63,7 +65,7 @@ services: # This "container" is a workaround to pre-create topics kafka-setup: - image: bitnami/kafka:3.7.0 + image: bitnami/kafka:3.8.0 hostname: kafka-setup networks: - ksml @@ -72,6 +74,7 @@ services: restart: on-failure command: "bash -c 'echo Trying to create topics... && \ kafka-topics.sh --create --if-not-exists --bootstrap-server broker:9093 --partitions 1 --replication-factor 1 --topic ksml_sensordata_avro && \ + kafka-topics.sh --create --if-not-exists --bootstrap-server broker:9093 --partitions 1 --replication-factor 1 --topic ksml_sensordata_avro_binary && \ kafka-topics.sh --create --if-not-exists --bootstrap-server broker:9093 --partitions 1 --replication-factor 1 --topic ksml_sensordata_csv && \ kafka-topics.sh --create --if-not-exists --bootstrap-server broker:9093 --partitions 1 --replication-factor 1 --topic ksml_sensordata_json && \ kafka-topics.sh --create --if-not-exists --bootstrap-server broker:9093 --partitions 1 --replication-factor 1 --topic ksml_sensordata_xml && \ @@ -91,9 +94,9 @@ services: kafka-topics.sh --create --if-not-exists --bootstrap-server broker:9093 --partitions 1 --replication-factor 1 --topic io.ksml.example.processor-ownerCount-changelog'" example-producer: - image: axual/ksml:latest + image: axual/ksml:local hostname: kafka-data-generator - restart: always + restart: no networks: - ksml working_dir: /ksml diff --git a/examples/00-example-generate-sensordata.yaml b/examples/00-example-generate-sensordata.yaml index 75d400aa..838c5904 100644 --- a/examples/00-example-generate-sensordata.yaml +++ b/examples/00-example-generate-sensordata.yaml @@ -45,12 +45,12 @@ producers: keyType: string valueType: avro:SensorData - # The following producer generates messages for 12-example-byte-manipulation.yaml. Enable this producer if you - # want to run that specific example. + # This example uses the otherAvro notation, using the Apicurio Registry API and serializers. + # See the ksml-data-generator.yaml for the notation definition sensordata_avro_producer_binary: generator: generate_sensordata_message interval: 3s to: topic: ksml_sensordata_avro_binary keyType: string - valueType: avro:SensorData + valueType: otherAvro:SensorData diff --git a/examples/01-example-inspect.yaml b/examples/01-example-inspect.yaml index 235a9f94..ea0e3622 100644 --- a/examples/01-example-inspect.yaml +++ b/examples/01-example-inspect.yaml @@ -8,6 +8,10 @@ streams: keyType: string valueType: avro:SensorData offsetResetPolicy: latest + sensor_source_avro_binary: + topic: ksml_sensordata_avro_binary + keyType: string + valueType: otherAvro:SensorData sensor_source_csv: topic: ksml_sensordata_csv keyType: string @@ -36,6 +40,10 @@ pipelines: from: sensor_source_avro forEach: code: log_message(key, value, format="AVRO") + consume_avro_binary: + from: sensor_source_avro_binary + forEach: + code: log_message(key, value, format="OTHER_AVRO") consume_csv: from: sensor_source_csv forEach: diff --git a/examples/ksml-data-generator.yaml b/examples/ksml-data-generator.yaml index 76270c86..f4ac4b80 100644 --- a/examples/ksml-data-generator.yaml +++ b/examples/ksml-data-generator.yaml @@ -1,8 +1,8 @@ ksml: # The examples directory is mounted to /ksml in the docker compose definition - configDirectory: /ksml # When not set defaults to the working directory - schemaDirectory: /ksml # When not set defaults to the config directory - storageDirectory: /tmp # When not set defaults to the default JVM temp directory +# configDirectory: /ksml # When not set defaults to the working directory +# schemaDirectory: /ksml # When not set defaults to the config directory +# storageDirectory: /tmp # When not set defaults to the default JVM temp directory # This section defines if a REST endpoint is opened on the KSML runner, through which # state stores and/or readiness / liveness probes can be accessed. @@ -13,7 +13,7 @@ ksml: # This section defines whether a Prometheus endpoint is opened to allow metric scraping. prometheus: - enabled: true # Set to true to enable, or false to disable + enabled: false # Set to true to enable, or false to disable host: 0.0.0.0 # IP address to bind the Prometheus agent server to port: 9999 # Port number to listen on @@ -41,27 +41,46 @@ ksml: type: confluent_avro # For AVRO there are two implementations: apicurio_avro and confluent_avro ## Below this line, specify properties to be passed into Confluent's KafkaAvroSerializer and KafkaAvroDeserializer config: - schema.registry.url: https://url.to.schena.registry.org + # The example uses Apicurio, using the Apicurio Confluent Compatibility URL + schema.registry.url: http://localhost:8081/apis/ccompat/v7 + auto.register.schemas: true normalize.schemas: true - schema.registry.ssl.protocol: TLSv1.3 - schema.registry.ssl.enabled.protocols: TLSv1.3,TLSv1.2 - schema.registry.ssl.endpoint.identification.algorithm: "" - schema.registry.ssl.keystore.type: JKS - schema.registry.ssl.truststore.type: JKS - schema.registry.ssl.key.password: password - schema.registry.ssl.keystore.password: password - schema.registry.ssl.keystore.location: /path/to/keystore.jks - schema.registry.ssl.truststore.password: password - schema.registry.ssl.truststore.location: /path/to/truststore.jks - auto.register.schemas: false + # Below is an example SSL configuration for Confluent Serialization library + # schema.registry.ssl.protocol: TLSv1.3 + # schema.registry.ssl.enabled.protocols: TLSv1.3,TLSv1.2 + # schema.registry.ssl.endpoint.identification.algorithm: "" + # schema.registry.ssl.keystore.location: /path/to/keystore.jks + # schema.registry.ssl.keystore.type: JKS + # schema.registry.ssl.key.password: password + # schema.registry.ssl.keystore.password: password + # schema.registry.ssl.truststore.location: /path/to/truststore.jks + # schema.registry.ssl.truststore.type: JKS + # schema.registry.ssl.truststore.password: password + # Definition for "other_avro" notation. With this config, you can use a type like otherAvro:SchenaName in your # KSML definition, which then uses the Apicurio implementation for AVRO. otherAvro: type: apicurio_avro # For AVRO there are two implementations: apicurio_avro and confluent_avro ## Below this line, specify properties to be passed into Apicurio's AvroKafkaSerializer and AvroKafkaDeserializer config: - apicurio.registry.url: https://sr2 - apicurio.registry.schema-resolver: CustomResolver + apicurio.registry.url: http://schema_registry:8081/apis/registry/v3 + # Register schema if it does not exist + apicurio.registry.auto-register: "true" + # Apicurio Avro serialisation settings + # apicurio.registry.avro.encoding: "BINARY" + # apicurio.registry.headers.enabled: "true" + # apicurio.registry.serde.IdHandler: "io.apicurio.registry.serde.Default4ByteIdHandler" + # apicurio.registry.schema-resolver: "io.apicurio.registry.resolver.DefaultSchemaResolver" + + # Below is an example SSL configuration for Apicurio Serialization library from Apicurio documentation + # apicurio.registry.request.ssl.keystore.location: /path/to/keystore.jks + # apicurio.registry.request.ssl.keystore.type: JKS + # apicurio.registry.request.ssl.keystore.password: password + # apicurio.registry.request.ssl.key.password: password + # apicurio.registry.request.ssl.truststore.location: /path/to/truststore.jks + # apicurio.registry.request.ssl.truststore.type: JKS + # apicurio.registry.request.ssl.truststore.password: password + enableProducers: true # Set to true to allow producer definitions to be parsed in the KSML definitions and be executed. enablePipelines: false # Set to true to allow pipeline definitions to be parsed in the KSML definitions and be executed. @@ -69,7 +88,7 @@ ksml: # Section where you specify which KSML definitions to load, parse and execute. definitions: # Format is : - generate_alert_setting: 00-example-generate-alertsettings.yaml +# generate_alert_setting: 00-example-generate-alertsettings.yaml generate_sensor_data: 00-example-generate-sensordata.yaml # This setup connects to the Kafka broker and schema registry started with the example docker-compose file @@ -77,7 +96,6 @@ ksml: kafka: bootstrap.servers: broker:9093 application.id: io.ksml.example.producer - schema.registry.url: http://schema_registry:8081 security.protocol: PLAINTEXT acks: all diff --git a/examples/ksml-runner.yaml b/examples/ksml-runner.yaml index dc23ae90..fee44b9b 100644 --- a/examples/ksml-runner.yaml +++ b/examples/ksml-runner.yaml @@ -28,7 +28,7 @@ ksml: log: true # Log errors true/false logPayload: true # Upon error, should the payload of the message be dumped to the log file. loggerName: ProcessError # Definition of the error logger name. - handler: continueOnFail # How to proceed after encountering the error. Either continueOnFail or stopOnFail. + handler: stopOnFail # How to proceed after encountering the error. Either continueOnFail or stopOnFail. produce: log: true # Log errors true/false logPayload: true # Upon error, should the payload of the message be dumped to the log file. @@ -41,27 +41,45 @@ ksml: type: confluent_avro # For AVRO there are two implementations: apicurio_avro and confluent_avro ## Below this line, specify properties to be passed into Confluent's KafkaAvroSerializer and KafkaAvroDeserializer config: - schema.registry.url: https://url.to.schena.registry.org + # Link to Apicurio Confluent Compatibility URL + schema.registry.url: http://schema_registry:8081/apis/ccompat/v88 + auto.register.schemas: true normalize.schemas: true - schema.registry.ssl.protocol: TLSv1.3 - schema.registry.ssl.enabled.protocols: TLSv1.3,TLSv1.2 - schema.registry.ssl.endpoint.identification.algorithm: "" - schema.registry.ssl.keystore.type: JKS - schema.registry.ssl.truststore.type: JKS - schema.registry.ssl.key.password: password - schema.registry.ssl.keystore.password: password - schema.registry.ssl.keystore.location: /path/to/keystore.jks - schema.registry.ssl.truststore.password: password - schema.registry.ssl.truststore.location: /path/to/truststore.jks - auto.register.schemas: false + # Below is an example SSL configuration for Confluent Serialization library + # schema.registry.ssl.protocol: TLSv1.3 + # schema.registry.ssl.enabled.protocols: TLSv1.3,TLSv1.2 + # schema.registry.ssl.endpoint.identification.algorithm: "" + # schema.registry.ssl.keystore.location: /path/to/keystore.jks + # schema.registry.ssl.keystore.type: JKS + # schema.registry.ssl.key.password: password + # schema.registry.ssl.keystore.password: password + # schema.registry.ssl.truststore.location: /path/to/truststore.jks + # schema.registry.ssl.truststore.type: JKS + # schema.registry.ssl.truststore.password: password + # Definition for "other_avro" notation. With this config, you can use a type like otherAvro:SchenaName in your # KSML definition, which then uses the Apicurio implementation for AVRO. otherAvro: type: apicurio_avro # For AVRO there are two implementations: apicurio_avro and confluent_avro ## Below this line, specify properties to be passed into Apicurio's AvroKafkaSerializer and AvroKafkaDeserializer config: - apicurio.registry.url: https://sr2 - apicurio.registry.schema-resolver: CustomResolver + apicurio.registry.url: http://schema_registry:8081/apis/registry/v3 + # Register schema if it does not exist + apicurio.registry.auto-register: "true" + # Apicurio Avro serialisation settings + # apicurio.registry.avro.encoding: "BINARY" + # apicurio.registry.headers.enabled: "true" + # apicurio.registry.serde.IdHandler: "io.apicurio.registry.serde.Default4ByteIdHandler" + # apicurio.registry.schema-resolver: "io.apicurio.registry.resolver.DefaultSchemaResolver" + + # Below is an example SSL configuration for Apicurio Serialization library from Apicurio documentation + # apicurio.registry.request.ssl.keystore.location: /path/to/keystore.jks + # apicurio.registry.request.ssl.keystore.type: JKS + # apicurio.registry.request.ssl.keystore.password: password + # apicurio.registry.request.ssl.key.password: password + # apicurio.registry.request.ssl.truststore.location: /path/to/truststore.jks + # apicurio.registry.request.ssl.truststore.type: JKS + # apicurio.registry.request.ssl.truststore.password: password enableProducers: true # Set to true to allow producer definitions to be parsed in the KSML definitions and be executed. enablePipelines: true # Set to true to allow pipeline definitions to be parsed in the KSML definitions and be executed. @@ -92,12 +110,16 @@ ksml: # These examples are intended to run from a inside a container on the same network kafka: application.id: io.ksml.example.streaming + # The group instance id is used to identify a single instance of the application, allowing for + # faster rebalances and less partition reassignments. The name must be unique for each member in the group. + group.instance.id: example-instance + bootstrap.servers: broker:9093 - schema.registry.url: http://schema_registry:8081 security.protocol: PLAINTEXT auto.offset.reset: earliest acks: all + # These are Kafka SSL configuration properties. Check the documentation at1 # Check the documentation at https://kafka.apache.org/documentation/#producerconfigs for more properties diff --git a/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/AvroNotation.java b/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/AvroNotation.java index 93a9da6f..9f42f349 100644 --- a/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/AvroNotation.java +++ b/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/AvroNotation.java @@ -21,6 +21,15 @@ */ import com.google.common.collect.ImmutableMap; + +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serializer; + +import java.nio.ByteBuffer; +import java.util.Map; + import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; import io.apicurio.registry.serde.avro.AvroKafkaSerializer; import io.axual.ksml.data.exception.DataException; @@ -35,11 +44,6 @@ import io.confluent.kafka.serializers.KafkaAvroDeserializer; import io.confluent.kafka.serializers.KafkaAvroSerializer; import lombok.Getter; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.common.serialization.Serializer; - -import java.util.Map; public class AvroNotation implements Notation { public static final DataType DEFAULT_TYPE = new StructType(); @@ -106,21 +110,9 @@ public AvroSerde(SerdeType type) { case CONFLUENT -> new KafkaAvroDeserializer(); }; - wrapSerializer = (topic, data) -> { - try { - return serializer.serialize(topic, mapper.fromDataObject(nativeMapper.toDataObject(data))); - } catch (Exception e) { - throw new ExecutionException("Error serializing AVRO message to topic " + topic, e); - } - }; - - wrapDeserializer = (topic, data) -> { - try { - return mapper.toDataObject(deserializer.deserialize(topic, data)); - } catch (Exception e) { - throw new ExecutionException("Error deserializing AVRO message from topic " + topic, e); - } - }; + var wrappedSerde = new WrappedSerde(serializer,deserializer,nativeMapper); + wrapSerializer = wrappedSerde; + wrapDeserializer = wrappedSerde; } @Override @@ -139,4 +131,63 @@ public Deserializer deserializer() { return wrapDeserializer; } } + + private record WrappedSerde(Serializer serializer, Deserializer deserializer, NativeDataObjectMapper nativeMapper) implements Serializer, Deserializer { + @Override + public Object deserialize(final String topic, final byte[] data) { + try { + return mapper.toDataObject(deserializer.deserialize(topic, data)); + } catch (Exception e) { + throw new ExecutionException("Error deserializing AVRO message from topic " + topic, e); + } + } + + @Override + public Object deserialize(final String topic, final Headers headers, final byte[] data) { + try { + return mapper.toDataObject(deserializer.deserialize(topic, headers, data)); + } catch (Exception e) { + throw new ExecutionException("Error deserializing AVRO message from topic " + topic, e); + } + } + + @Override + public Object deserialize(final String topic, final Headers headers, final ByteBuffer data) { + try { + return mapper.toDataObject(deserializer.deserialize(topic, headers, data)); + } catch (Exception e) { + throw new ExecutionException("Error deserializing AVRO message from topic " + topic, e); + } + } + + @Override + public void configure(final Map configs, final boolean isKey) { + serializer.configure(configs, isKey); + deserializer.configure(configs, isKey); + } + + @Override + public byte[] serialize(final String topic, final Object data) { + try { + return serializer.serialize(topic, mapper.fromDataObject(nativeMapper.toDataObject(data))); + } catch (Exception e) { + throw new ExecutionException("Error serializing AVRO message to topic " + topic, e); + } + } + + @Override + public byte[] serialize(final String topic, final Headers headers, final Object data) { + try { + return serializer.serialize(topic, headers, mapper.fromDataObject(nativeMapper.toDataObject(data))); + } catch (Exception e) { + throw new ExecutionException("Error serializing AVRO message to topic " + topic, e); + } + } + + @Override + public void close() { + serializer.close(); + deserializer.close(); + } + } } diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/ExecutableProducer.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/ExecutableProducer.java index 9bcd056d..64803d41 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/ExecutableProducer.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/ExecutableProducer.java @@ -20,6 +20,15 @@ * =========================LICENSE_END================================== */ +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.header.internals.RecordHeaders; +import org.apache.kafka.common.serialization.Serializer; + +import java.time.Duration; +import java.util.Map; +import java.util.concurrent.ExecutionException; + import io.axual.ksml.client.serde.ResolvingSerializer; import io.axual.ksml.data.mapper.DataObjectConverter; import io.axual.ksml.data.mapper.NativeDataObjectMapper; @@ -38,13 +47,6 @@ import io.axual.ksml.user.UserPredicate; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.Serializer; - -import java.time.Duration; -import java.util.Map; -import java.util.concurrent.ExecutionException; import static io.axual.ksml.data.notation.UserType.DEFAULT_NOTATION; @@ -151,12 +153,16 @@ public void produceMessage(Producer producer) { valueStr = valueStr.replace("\n", "\\\\n"); log.info("Message: key={}, value={}", keyStr, valueStr); - var serializedKey = keySerializer.serialize(topic, NATIVE_MAPPER.fromDataObject(key)); - var serializedValue = valueSerializer.serialize(topic, NATIVE_MAPPER.fromDataObject(value)); + // Headers are sometimes needed for Apicurio serialisation + var headers = new RecordHeaders(); + var serializedKey = keySerializer.serialize(topic, headers, NATIVE_MAPPER.fromDataObject(key)); + var serializedValue = valueSerializer.serialize(topic, headers, NATIVE_MAPPER.fromDataObject(value)); ProducerRecord message = new ProducerRecord<>( topic, + null, serializedKey, - serializedValue + serializedValue, + headers ); var future = producer.send(message); try { From 069a10364248eb0bfd4c766bc5f89579d6dbf6fa Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Fri, 1 Nov 2024 10:33:32 +0100 Subject: [PATCH 17/55] Improve detection of input, intermediate, output and internal topics --- .../java/io/axual/ksml/TopologyGenerator.java | 50 +++++++++---------- .../ksml/generator/TopologyAnalyzer.java | 47 +++++++++-------- 2 files changed, 49 insertions(+), 48 deletions(-) diff --git a/ksml/src/main/java/io/axual/ksml/TopologyGenerator.java b/ksml/src/main/java/io/axual/ksml/TopologyGenerator.java index 4665c39d..401f02a7 100644 --- a/ksml/src/main/java/io/axual/ksml/TopologyGenerator.java +++ b/ksml/src/main/java/io/axual/ksml/TopologyGenerator.java @@ -21,17 +21,6 @@ */ -import org.apache.kafka.streams.StreamsBuilder; -import org.apache.kafka.streams.StreamsConfig; -import org.apache.kafka.streams.Topology; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashSet; -import java.util.Map; -import java.util.Properties; -import java.util.Set; - import io.axual.ksml.definition.GlobalTableDefinition; import io.axual.ksml.definition.StateStoreDefinition; import io.axual.ksml.definition.TableDefinition; @@ -42,6 +31,13 @@ import io.axual.ksml.operation.StreamOperation; import io.axual.ksml.operation.ToOperation; import io.axual.ksml.stream.StreamWrapper; +import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.Topology; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; public class TopologyGenerator { private static final Logger LOG = LoggerFactory.getLogger(TopologyGenerator.class); @@ -56,7 +52,7 @@ public TopologyGenerator(String applicationId, String optimization) { // Parse configuration this.applicationId = applicationId; this.optimization = new Properties(); - if(optimization != null) { + if (optimization != null) { this.optimization.put(StreamsConfig.TOPOLOGY_OPTIMIZATION_CONFIG, optimization); } } @@ -64,16 +60,16 @@ public TopologyGenerator(String applicationId, String optimization) { public Topology create(StreamsBuilder streamsBuilder, Map definitions) { if (definitions.isEmpty()) return null; - final var knownTopics = new HashSet(); + final var stores = new TreeMap(); definitions.forEach((name, definition) -> { final var context = new TopologyBuildContext(streamsBuilder, definition); generate(definition, context); + stores.putAll(definition.stateStores()); }); - var topology = streamsBuilder.build(optimization); - var analysis = TopologyAnalyzer.analyze(topology, applicationId, knownTopics); + var analysis = TopologyAnalyzer.analyze(topology, applicationId); StringBuilder summary = new StringBuilder("\n\n"); summary.append(topology != null ? topology.describe() : "null"); @@ -82,10 +78,11 @@ public Topology create(StreamsBuilder streamsBuilder, Map "keyValue"; case SESSION_STORE -> "session"; case WINDOW_STORE -> "window"; }) .append("/") - .append(entry.getKey()) - .append("/"); + .append(storeEntry.getKey()) + .append("/") + .append("\n"); } } @@ -213,7 +211,7 @@ private void generate(TopologyDefinition specification, TopologyBuildContext con LOG.info(""" Generating Kafka Streams topology for pipeline {}: {} - """,name,tsBuilder); + """, name, tsBuilder); }); } diff --git a/ksml/src/main/java/io/axual/ksml/generator/TopologyAnalyzer.java b/ksml/src/main/java/io/axual/ksml/generator/TopologyAnalyzer.java index 87b882f9..cda3a760 100644 --- a/ksml/src/main/java/io/axual/ksml/generator/TopologyAnalyzer.java +++ b/ksml/src/main/java/io/axual/ksml/generator/TopologyAnalyzer.java @@ -24,46 +24,49 @@ import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.TopologyDescription; -import java.util.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; public class TopologyAnalyzer { public record TopologyAnalysis(Set inputTopics, Set intermediateTopics, - Set outputTopics, Map stores) { + Set outputTopics, Set internalTopics) { } - public static TopologyAnalysis analyze(Topology topology, String applicationId, Collection knownTopics) { + public static TopologyAnalysis analyze(Topology topology, String applicationId) { // Derive all input, intermediate and output topics - final var topologyInputs = new HashSet(); - final var topologyOutputs = new HashSet(); - final var inputTopics = new HashSet(); - final var outputTopics = new HashSet(); - final var intermediateTopics = new HashSet(); + final var inputTopics = new TreeSet(); + final var intermediateTopics = new TreeSet(); + final var outputTopics = new TreeSet(); + final var internalTopics = new TreeSet(); - analyzeTopology(topology, topologyInputs, topologyOutputs); + analyzeTopology(topology, inputTopics, outputTopics); - for (String topic : topologyInputs) { - if (knownTopics.contains(topic)) { - inputTopics.add(topic); - } else { - intermediateTopics.add(applicationId + "-" + topic); + // Intermediate topics are both input and output topics, so sort them into a separate set + for (final var topic : inputTopics) { + if (outputTopics.contains(topic)) { + intermediateTopics.add(topic); + outputTopics.remove(topic); } } + inputTopics.removeAll(intermediateTopics); - for (String topic : topologyOutputs) { - if (knownTopics.contains(topic)) { - outputTopics.add(topic); - } else { - intermediateTopics.add(applicationId + "-" + topic); + // Internal topics end in "changelog" or "repartition", so sort them into a separate set + for (final var topic : intermediateTopics) { + if (topic.endsWith("-changelog") || topic.endsWith("-repartition")) { + internalTopics.add(topic); } } + intermediateTopics.removeAll(internalTopics); // Return the built topology and its context return new TopologyAnalysis( inputTopics, intermediateTopics, outputTopics, - new HashMap<>()); -// context.getStoreDefinitions()); + internalTopics.stream().map(topic -> applicationId + "-" + topic).collect(Collectors.toSet())); } private static void analyzeTopology(Topology topology, Set inputTopics, Set outputTopics) { @@ -77,7 +80,7 @@ private static void analyzeTopology(Topology topology, Set inputTopics, inputTopics.add(sourceNode.topicPattern().pattern()); } if (node instanceof TopologyDescription.Processor processorNode) { -// stores.addAll(processorNode.stores()); + // Ignore store names here } if (node instanceof TopologyDescription.Sink sinkNode && sinkNode.topic() != null) { outputTopics.add(sinkNode.topic()); From a73889eeb4c24565ec8e664bdb5965032001d61f Mon Sep 17 00:00:00 2001 From: Richard Bosch Date: Fri, 1 Nov 2024 10:33:50 +0100 Subject: [PATCH 18/55] Fix example configuration Fix docker compose producer version and depends_on statements --- docker-compose.yml | 36 +++++++++++++++++++------------ examples/ksml-data-generator.yaml | 8 +++---- examples/ksml-runner.yaml | 2 +- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8e185377..c30f161d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,17 +33,19 @@ services: KAFKA_CFG_MIN_INSYNC_REPLICAS: 1 KAFKA_CFG_NUM_PARTITIONS: 1 healthcheck: - test: kafka-topics.sh --bootstrap-server broker:9093 --list | fgrep -q ksml_sensordata_table - interval: 30s + # If the kafka topics can list data, the broker is healthy + test: kafka-topics.sh --bootstrap-server broker:9093 --list + interval: 5s timeout: 10s - retries: 5 - start_period: 20s + retries: 10 + start_period: 5s schema_registry: image: apicurio/apicurio-registry:3.0.2 hostname: schema_registry depends_on: - - broker + broker: + condition: service_healthy ports: - "8081:8081" networks: @@ -57,11 +59,12 @@ services: APICURIO_KAFKASQL_BOOTSTRAP_SERVERS: 'broker:9093' APICURIO_KAFKASQL_TOPIC: '_apciurio-kafkasql-store' healthcheck: + # If the api endpoint is available, the service is considered healthy test: curl http://localhost:8081/apis - interval: 30s + interval: 15s timeout: 10s - retries: 5 - start_period: 20s + retries: 10 + start_period: 10s # This "container" is a workaround to pre-create topics kafka-setup: @@ -70,7 +73,8 @@ services: networks: - ksml depends_on: - - broker + broker: + condition: service_healthy restart: on-failure command: "bash -c 'echo Trying to create topics... && \ kafka-topics.sh --create --if-not-exists --bootstrap-server broker:9093 --partitions 1 --replication-factor 1 --topic ksml_sensordata_avro && \ @@ -94,9 +98,9 @@ services: kafka-topics.sh --create --if-not-exists --bootstrap-server broker:9093 --partitions 1 --replication-factor 1 --topic io.ksml.example.processor-ownerCount-changelog'" example-producer: - image: axual/ksml:local + image: axual/ksml:latest hostname: kafka-data-generator - restart: no + restart: on-failure networks: - ksml working_dir: /ksml @@ -105,6 +109,10 @@ services: volumes: - ./examples:/ksml depends_on: - - broker - - schema_registry - - kafka-setup + broker: + condition: service_healthy + schema_registry: + condition: service_healthy + kafka-setup: + condition: service_completed_successfully + diff --git a/examples/ksml-data-generator.yaml b/examples/ksml-data-generator.yaml index f4ac4b80..b5c63cfd 100644 --- a/examples/ksml-data-generator.yaml +++ b/examples/ksml-data-generator.yaml @@ -1,8 +1,8 @@ ksml: # The examples directory is mounted to /ksml in the docker compose definition -# configDirectory: /ksml # When not set defaults to the working directory -# schemaDirectory: /ksml # When not set defaults to the config directory -# storageDirectory: /tmp # When not set defaults to the default JVM temp directory + configDirectory: . # When not set defaults to the working directory + schemaDirectory: . # When not set defaults to the config directory + storageDirectory: /tmp # When not set defaults to the default JVM temp directory # This section defines if a REST endpoint is opened on the KSML runner, through which # state stores and/or readiness / liveness probes can be accessed. @@ -42,7 +42,7 @@ ksml: ## Below this line, specify properties to be passed into Confluent's KafkaAvroSerializer and KafkaAvroDeserializer config: # The example uses Apicurio, using the Apicurio Confluent Compatibility URL - schema.registry.url: http://localhost:8081/apis/ccompat/v7 + schema.registry.url: http://schema_registry:8081/apis/ccompat/v7 auto.register.schemas: true normalize.schemas: true # Below is an example SSL configuration for Confluent Serialization library diff --git a/examples/ksml-runner.yaml b/examples/ksml-runner.yaml index fee44b9b..b4214969 100644 --- a/examples/ksml-runner.yaml +++ b/examples/ksml-runner.yaml @@ -42,7 +42,7 @@ ksml: ## Below this line, specify properties to be passed into Confluent's KafkaAvroSerializer and KafkaAvroDeserializer config: # Link to Apicurio Confluent Compatibility URL - schema.registry.url: http://schema_registry:8081/apis/ccompat/v88 + schema.registry.url: http://schema_registry:8081/apis/ccompat/v7 auto.register.schemas: true normalize.schemas: true # Below is an example SSL configuration for Confluent Serialization library From 8daf79d4ae7cc530e53a66809982196d0e9a875c Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 19:22:27 +0100 Subject: [PATCH 19/55] Introduce performance measurement example * Split up example message generator into a normal one (interval), a binary one (intended to run example 12) and a batch one (intended for performance measurements) * Introduce example 19 to perform generic performance measurements on any KSML pipeline * Remove reading back from binary topic in example 1 to allow running all examples in parallel (ie no conflict with example 12) * Remove dead code from TopologyGenerator * Improve error message in UserTypeParser --- .../00-example-generate-sensordata-batch.yaml | 49 +++++++++++++++++++ ...00-example-generate-sensordata-binary.yaml | 48 ++++++++++++++++++ examples/00-example-generate-sensordata.yaml | 10 ---- examples/01-example-inspect.yaml | 8 --- .../19-example-performance-measurement.yaml | 44 +++++++++++++++++ examples/ksml-data-generator.yaml | 7 ++- examples/ksml-runner.yaml | 1 + .../java/io/axual/ksml/TopologyGenerator.java | 14 ------ .../io/axual/ksml/parser/UserTypeParser.java | 2 +- 9 files changed, 146 insertions(+), 37 deletions(-) create mode 100644 examples/00-example-generate-sensordata-batch.yaml create mode 100644 examples/00-example-generate-sensordata-binary.yaml create mode 100644 examples/19-example-performance-measurement.yaml diff --git a/examples/00-example-generate-sensordata-batch.yaml b/examples/00-example-generate-sensordata-batch.yaml new file mode 100644 index 00000000..645bccbc --- /dev/null +++ b/examples/00-example-generate-sensordata-batch.yaml @@ -0,0 +1,49 @@ +# $schema: https://raw.githubusercontent.com/Axual/ksml/refs/heads/main/docs/ksml-language-spec.json + +# This example shows how to generate data and have it sent to a target topic in a given format. + +functions: + generate_sensordata_message: + type: generator + globalCode: | + import time + import random + sensorCounter = 0 + code: | + global sensorCounter + + key = "sensor"+str(sensorCounter) # Set the key to return ("sensor0" to "sensor9") + sensorCounter = (sensorCounter+1) % 10 # Increase the counter for next iteration + + # Generate some random sensor measurement data + types = { 0: { "type": "AREA", "unit": random.choice([ "m2", "ft2" ]), "value": str(random.randrange(1000)) }, + 1: { "type": "HUMIDITY", "unit": random.choice([ "g/m3", "%" ]), "value": str(random.randrange(100)) }, + 2: { "type": "LENGTH", "unit": random.choice([ "m", "ft" ]), "value": str(random.randrange(1000)) }, + 3: { "type": "STATE", "unit": "state", "value": random.choice([ "off", "on" ]) }, + 4: { "type": "TEMPERATURE", "unit": random.choice([ "C", "F" ]), "value": str(random.randrange(-100, 100)) } + } + + # Build the result value using any of the above measurement types + value = { "name": key, "timestamp": str(round(time.time()*1000)), **random.choice(types) } + value["color"] = random.choice([ "black", "blue", "red", "yellow", "white" ]) + value["owner"] = random.choice([ "Alice", "Bob", "Charlie", "Dave", "Evan" ]) + value["city"] = random.choice([ "Amsterdam", "Xanten", "Utrecht", "Alkmaar", "Leiden" ]) + + if random.randrange(10) == 0: + key = None + if random.randrange(10) == 0: + value = None + expression: (key, value) # Return a message tuple with the key and value + resultType: (string, json) # Indicate the type of key and value + +producers: + # Produce 10k messages in batches of 100 messages with a 1ms interval + sensordata_avro_producer: + generator: generate_sensordata_message + interval: 1 + count: 10000 + batchSize: 100 + to: + topic: ksml_sensordata_avro + keyType: string + valueType: avro:SensorData diff --git a/examples/00-example-generate-sensordata-binary.yaml b/examples/00-example-generate-sensordata-binary.yaml new file mode 100644 index 00000000..7205d23a --- /dev/null +++ b/examples/00-example-generate-sensordata-binary.yaml @@ -0,0 +1,48 @@ +# $schema: https://raw.githubusercontent.com/Axual/ksml/refs/heads/main/docs/ksml-language-spec.json + +# This example shows how to generate data and have it sent to a target topic in a given format. + +functions: + generate_sensordata_message: + type: generator + globalCode: | + import time + import random + sensorCounter = 0 + code: | + global sensorCounter + + key = "sensor"+str(sensorCounter) # Set the key to return ("sensor0" to "sensor9") + sensorCounter = (sensorCounter+1) % 10 # Increase the counter for next iteration + + # Generate some random sensor measurement data + types = { 0: { "type": "AREA", "unit": random.choice([ "m2", "ft2" ]), "value": str(random.randrange(1000)) }, + 1: { "type": "HUMIDITY", "unit": random.choice([ "g/m3", "%" ]), "value": str(random.randrange(100)) }, + 2: { "type": "LENGTH", "unit": random.choice([ "m", "ft" ]), "value": str(random.randrange(1000)) }, + 3: { "type": "STATE", "unit": "state", "value": random.choice([ "off", "on" ]) }, + 4: { "type": "TEMPERATURE", "unit": random.choice([ "C", "F" ]), "value": str(random.randrange(-100, 100)) } + } + + # Build the result value using any of the above measurement types + value = { "name": key, "timestamp": str(round(time.time()*1000)), **random.choice(types) } + value["color"] = random.choice([ "black", "blue", "red", "yellow", "white" ]) + value["owner"] = random.choice([ "Alice", "Bob", "Charlie", "Dave", "Evan" ]) + value["city"] = random.choice([ "Amsterdam", "Xanten", "Utrecht", "Alkmaar", "Leiden" ]) + + if random.randrange(10) == 0: + key = None + if random.randrange(10) == 0: + value = None + expression: (key, value) # Return a message tuple with the key and value + resultType: (string, json) # Indicate the type of key and value + +producers: + # This example uses the otherAvro notation, using the Apicurio Registry API and serializers. + # See the ksml-data-generator.yaml for the notation definition + sensordata_avro_producer_binary: + generator: generate_sensordata_message + interval: 3s + to: + topic: ksml_sensordata_avro_binary + keyType: string + valueType: otherAvro:SensorData diff --git a/examples/00-example-generate-sensordata.yaml b/examples/00-example-generate-sensordata.yaml index 838c5904..1e446bff 100644 --- a/examples/00-example-generate-sensordata.yaml +++ b/examples/00-example-generate-sensordata.yaml @@ -44,13 +44,3 @@ producers: topic: ksml_sensordata_avro keyType: string valueType: avro:SensorData - - # This example uses the otherAvro notation, using the Apicurio Registry API and serializers. - # See the ksml-data-generator.yaml for the notation definition - sensordata_avro_producer_binary: - generator: generate_sensordata_message - interval: 3s - to: - topic: ksml_sensordata_avro_binary - keyType: string - valueType: otherAvro:SensorData diff --git a/examples/01-example-inspect.yaml b/examples/01-example-inspect.yaml index ea0e3622..235a9f94 100644 --- a/examples/01-example-inspect.yaml +++ b/examples/01-example-inspect.yaml @@ -8,10 +8,6 @@ streams: keyType: string valueType: avro:SensorData offsetResetPolicy: latest - sensor_source_avro_binary: - topic: ksml_sensordata_avro_binary - keyType: string - valueType: otherAvro:SensorData sensor_source_csv: topic: ksml_sensordata_csv keyType: string @@ -40,10 +36,6 @@ pipelines: from: sensor_source_avro forEach: code: log_message(key, value, format="AVRO") - consume_avro_binary: - from: sensor_source_avro_binary - forEach: - code: log_message(key, value, format="OTHER_AVRO") consume_csv: from: sensor_source_csv forEach: diff --git a/examples/19-example-performance-measurement.yaml b/examples/19-example-performance-measurement.yaml new file mode 100644 index 00000000..bd69818d --- /dev/null +++ b/examples/19-example-performance-measurement.yaml @@ -0,0 +1,44 @@ +# $schema: https://raw.githubusercontent.com/Axual/ksml/refs/heads/main/docs/ksml-language-spec.json + +# This example behaves similarly to example 2 (copy) but includes Python code to measure KSML's performance. It does +# so by storing the message count and startup timestamp in global variables, and outputting log statements every 100 +# #messages, containing #messages processed, #seconds running since first message came in, and average #msg/sec. + +streams: + sensor_source: + topic: ksml_sensordata_avro + keyType: string + valueType: avro:SensorData + offsetResetPolicy: latest + sensor_copy: + topic: ksml_sensordata_copy + keyType: string + valueType: avro:SensorData + +pipelines: + main: + from: sensor_source + via: + # Use a PEEK operation to initialize the global messageCount and startTime + - type: peek + forEach: + globalCode: | + from datetime import datetime + messageCount, startTime = 0, 0 + code: | + # Declare global variables, since we are updating them below + global messageCount, startTime + if messageCount == 0: + startTime = datetime.now() + messageCount += 1 + # Output performance thus far, done in separate PEEK to allow easy insertion of other operations above + - type: peek + forEach: + code: | + # No need to include the global statement here, since we only read and don't update the global variables + # For every 100 messages that we process, we output a log statement with performance indication so far + if messageCount % 100 == 0: + # Prevent division by zero by using 1 second as minimum + runtime = max(1, (datetime.now() - startTime).total_seconds()) + log.warn("Processed {} messages in {} seconds = {} msg/sec", messageCount, runtime, round(messageCount / runtime, 2)) + to: sensor_copy diff --git a/examples/ksml-data-generator.yaml b/examples/ksml-data-generator.yaml index b5c63cfd..5d09c3f2 100644 --- a/examples/ksml-data-generator.yaml +++ b/examples/ksml-data-generator.yaml @@ -81,15 +81,16 @@ ksml: # apicurio.registry.request.ssl.truststore.type: JKS # apicurio.registry.request.ssl.truststore.password: password - enableProducers: true # Set to true to allow producer definitions to be parsed in the KSML definitions and be executed. enablePipelines: false # Set to true to allow pipeline definitions to be parsed in the KSML definitions and be executed. # Section where you specify which KSML definitions to load, parse and execute. definitions: # Format is : -# generate_alert_setting: 00-example-generate-alertsettings.yaml + generate_alert_setting: 00-example-generate-alertsettings.yaml generate_sensor_data: 00-example-generate-sensordata.yaml +# generate_sensor_data_batch: 00-example-generate-sensordata-batch.yaml +# generate_sensor_data_binary: 00-example-generate-sensordata-binary.yaml # This setup connects to the Kafka broker and schema registry started with the example docker-compose file # These examples are intended to run from a inside a container on the same network @@ -118,7 +119,6 @@ kafka: # These patterns are resolved into the actual name used on Kafka using the values in this configuration map # and the topic names specified in the definition YAML files - # tenant: "ksmldemo" # instance: "dta" # environment: "dev" @@ -126,4 +126,3 @@ kafka: # # Results in Kafka topic ksmldemo-dta-dev- # group.id.pattern: "{tenant}-{instance}-{environment}-{group.id}" # transactional.id.pattern: "{tenant}-{instance}-{environment}-{transactional.id}" - diff --git a/examples/ksml-runner.yaml b/examples/ksml-runner.yaml index b4214969..e0dfd5f8 100644 --- a/examples/ksml-runner.yaml +++ b/examples/ksml-runner.yaml @@ -105,6 +105,7 @@ ksml: # transform_metadata: 16-example-transform-metadata.yaml # inspect_with_metrics: 17-example-inspect-with-metrics.yaml # timestamp_extractor: 18-example-timestamp-extractor.yaml +# performance-measurement: 19-example-performance-measurement.yaml # This setup connects to the Kafka broker and schema registry started with the example docker-compose file # These examples are intended to run from a inside a container on the same network diff --git a/ksml/src/main/java/io/axual/ksml/TopologyGenerator.java b/ksml/src/main/java/io/axual/ksml/TopologyGenerator.java index 401f02a7..3f256406 100644 --- a/ksml/src/main/java/io/axual/ksml/TopologyGenerator.java +++ b/ksml/src/main/java/io/axual/ksml/TopologyGenerator.java @@ -29,7 +29,6 @@ import io.axual.ksml.generator.TopologyDefinition; import io.axual.ksml.operation.StoreOperation; import io.axual.ksml.operation.StreamOperation; -import io.axual.ksml.operation.ToOperation; import io.axual.ksml.stream.StreamWrapper; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsConfig; @@ -137,19 +136,6 @@ private String getPrefix(String source) { } private void generate(TopologyDefinition specification, TopologyBuildContext context) { - // Register all topics - final var knownTopics = new HashSet(); - specification.topics().forEach((name, def) -> knownTopics.add(def.topic())); - - // Add source and target topics to the set of known topics - specification.pipelines().forEach((name, def) -> { - if (def.source() != null && def.source().definition() != null) - knownTopics.add(def.source().definition().topic()); - if (def.sink() instanceof ToOperation toOperation && toOperation.topic != null) { - knownTopics.add(toOperation.topic.topic()); - } - }); - // Preload the function into the Python context specification.functions().forEach((name, func) -> context.createUserFunction(func)); diff --git a/ksml/src/main/java/io/axual/ksml/parser/UserTypeParser.java b/ksml/src/main/java/io/axual/ksml/parser/UserTypeParser.java index 2392f7bf..12f56d89 100644 --- a/ksml/src/main/java/io/axual/ksml/parser/UserTypeParser.java +++ b/ksml/src/main/java/io/axual/ksml/parser/UserTypeParser.java @@ -163,7 +163,7 @@ private static UserType parseTypeAndNotation(String composedType, String default return new UserType(datatype, NotationLibrary.get(datatype).defaultType()); } - throw new TopologyException("Unknown data type: " + datatype); + throw new TopologyException("Unknown type: " + notation + ":" + datatype); } // This method decomposes a user type into its components. User types are always of the form "notation:datatype". From 33cf884a9ee947bd3467e9c4b9b20cd27d037820 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Mon, 23 Sep 2024 08:33:59 +0200 Subject: [PATCH 20/55] Update byte manipulation example - Generate example data for byte manipulation example - Have byte manipulation example read from alternative topic --- examples/00-example-generate-sensordata.yaml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/examples/00-example-generate-sensordata.yaml b/examples/00-example-generate-sensordata.yaml index 838c5904..c21822a1 100644 --- a/examples/00-example-generate-sensordata.yaml +++ b/examples/00-example-generate-sensordata.yaml @@ -1,4 +1,4 @@ -# $schema: https://raw.githubusercontent.com/Axual/ksml/refs/heads/main/docs/ksml-language-spec.json +# $schema: https://raw.githubusercontent.com/Axual/ksml/main/docs/ksml-language-spec.json # This example shows how to generate data and have it sent to a target topic in a given format. @@ -44,13 +44,3 @@ producers: topic: ksml_sensordata_avro keyType: string valueType: avro:SensorData - - # This example uses the otherAvro notation, using the Apicurio Registry API and serializers. - # See the ksml-data-generator.yaml for the notation definition - sensordata_avro_producer_binary: - generator: generate_sensordata_message - interval: 3s - to: - topic: ksml_sensordata_avro_binary - keyType: string - valueType: otherAvro:SensorData From 24414d9f8dd343aa22a75e940b675b9bdb683d88 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:12:17 +0100 Subject: [PATCH 21/55] Merge with upstream --- ksml-query/NOTICE.txt | 44 ++++-------------- ksml-reporting/NOTICE.txt | 45 ++++-------------- ksml-runner/NOTICE.txt | 44 ++++-------------- .../java/io/axual/ksml/runner/KSMLRunner.java | 41 ++++++++--------- .../runner/config/ErrorHandlingConfig.java | 2 - .../ksml/runner/config/KSMLRunnerConfig.java | 14 ++++-- .../ksml/runner/config/NotationConfig.java | 1 - .../java/io/axual/ksml/runner/lombok.config | 3 ++ .../runner/config/KSMLRunnerConfigTest.java | 4 +- ksml/NOTICE.txt | 46 ++++--------------- 10 files changed, 66 insertions(+), 178 deletions(-) create mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config diff --git a/ksml-query/NOTICE.txt b/ksml-query/NOTICE.txt index 8c27f39f..5613bbe6 100644 --- a/ksml-query/NOTICE.txt +++ b/ksml-query/NOTICE.txt @@ -1,5 +1,5 @@ -Lists of 136 third-party dependencies. +Lists of 108 third-party dependencies. (MIT License) minimal-json (com.eclipsesource.minimal-json:minimal-json:0.9.5 - https://github.com/ralfstx/minimal-json) (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.17.1 - https://github.com/FasterXML/jackson) (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.17.1 - https://github.com/FasterXML/jackson-core) @@ -19,18 +19,16 @@ Lists of 136 third-party dependencies. (The Apache Software License, Version 2.0) Guava ListenableFuture only (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava - https://github.com/google/guava/listenablefuture) (Apache License, Version 2.0) J2ObjC Annotations (com.google.j2objc:j2objc-annotations:3.0.0 - https://github.com/google/j2objc/) (Go License) RE2/J (com.google.re2j:re2j:1.6 - http://github.com/google/re2j) - (MIT License) Microsoft Kiota-Java Abstractions (com.microsoft.kiota:microsoft-kiota-abstractions:1.1.14 - https://github.com/microsoft/kiota-java) - (MIT License) Microsoft Kiota-Java Serialization-Form (com.microsoft.kiota:microsoft-kiota-serialization-form:1.1.14 - https://github.com/microsoft/kiota-java) - (MIT License) Microsoft Kiota-Java Serialization-Multipart (com.microsoft.kiota:microsoft-kiota-serialization-multipart:1.1.14 - https://github.com/microsoft/kiota-java) - (MIT License) Microsoft Kiota-Java Serialization-Text (com.microsoft.kiota:microsoft-kiota-serialization-text:1.1.14 - https://github.com/microsoft/kiota-java) (Eclipse Distribution License - v 1.0) Jakarta SOAP Implementation (com.sun.xml.messaging.saaj:saaj-impl:3.0.4 - https://projects.eclipse.org/projects/ee4j/metro-saaj/saaj-impl) (Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.17.0 - https://commons.apache.org/proper/commons-codec/) (Apache-2.0) Apache Commons IO (commons-io:commons-io:2.16.1 - https://commons.apache.org/proper/commons-io/) - (Apache License Version 2.0) apicurio-registry-common (io.apicurio:apicurio-registry-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-common/) - (Apache License Version 2.0) apicurio-registry-java-sdk (io.apicurio:apicurio-registry-java-sdk:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-java-sdk/) - (Apache License Version 2.0) apicurio-registry-schema-resolver (io.apicurio:apicurio-registry-schema-resolver:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-schema-resolver/) - (Apache License Version 2.0) apicurio-registry-serde-common (io.apicurio:apicurio-registry-serde-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serde-common/) - (Apache License Version 2.0) apicurio-registry-serdes-avro-serde (io.apicurio:apicurio-registry-serdes-avro-serde:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serdes-avro-serde/) + (Apache License Version 2.0) apicurio-common-rest-client-common (io.apicurio:apicurio-common-rest-client-common:0.1.18.Final - https://www.apicur.io/apicurio-common-rest-client-common/) + (Apache License Version 2.0) apicurio-common-rest-client-jdk (io.apicurio:apicurio-common-rest-client-jdk:0.1.18.Final - https://www.apicur.io/apicurio-common-rest-client-jdk/) + (Apache License Version 2.0) apicurio-registry-client (io.apicurio:apicurio-registry-client:2.6.5.Final - https://www.apicur.io/apicurio-registry-client/) + (Apache License Version 2.0) apicurio-registry-common (io.apicurio:apicurio-registry-common:2.6.5.Final - https://www.apicur.io/apicurio-registry-common/) + (Apache License Version 2.0) apicurio-registry-schema-resolver (io.apicurio:apicurio-registry-schema-resolver:2.6.5.Final - https://www.apicur.io/apicurio-registry-schema-resolver/) + (Apache License Version 2.0) apicurio-registry-serde-common (io.apicurio:apicurio-registry-serde-common:2.6.5.Final - https://www.apicur.io/apicurio-registry-serde-common/) + (Apache License Version 2.0) apicurio-registry-serdes-avro-serde (io.apicurio:apicurio-registry-serdes-avro-serde:2.6.5.Final - https://www.apicur.io/apicurio-registry-serdes-avro-serde/) (Apache 2.0) KSML (io.axual.ksml:ksml:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml) (Apache 2.0) KSML Data Library (io.axual.ksml:ksml-data:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-data) (Apache 2.0) KSML Data Library - AVRO (io.axual.ksml:ksml-data-avro:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-data-avro) @@ -45,31 +43,7 @@ Lists of 136 third-party dependencies. (The Apache Software License, Version 2.0) Log Redactor Metrics (io.confluent:logredactor-metrics:1.0.12 - https://github.com/confluentinc/logredactor) (Apache License 2.0) Metrics Core (io.dropwizard.metrics:metrics-core:4.2.25 - https://metrics.dropwizard.io/metrics-core) (Apache License 2.0) Metrics Integration with JMX (io.dropwizard.metrics:metrics-jmx:4.2.25 - https://metrics.dropwizard.io/metrics-jmx) - (Apache License, Version 2.0) Standard Uri Template (io.github.std-uritemplate:std-uritemplate:0.0.59 - https://std-uritemplate.github.io/) - (Apache-2.0) Kiota - Http - Vert.X (io.kiota:kiota-http-vertx:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-http-vertx) - (Apache-2.0) Kiota - Serialization - Jackson (io.kiota:kiota-serialization-jackson:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-serialization-jackson-parent/kiota-serialization-jackson) - (Apache License, Version 2.0) Netty/Buffer (io.netty:netty-buffer:4.1.108.Final - https://netty.io/netty-buffer/) - (Apache License, Version 2.0) Netty/Codec (io.netty:netty-codec:4.1.108.Final - https://netty.io/netty-codec/) - (Apache License, Version 2.0) Netty/Codec/DNS (io.netty:netty-codec-dns:4.1.108.Final - https://netty.io/netty-codec-dns/) - (Apache License, Version 2.0) Netty/Codec/HTTP (io.netty:netty-codec-http:4.1.108.Final - https://netty.io/netty-codec-http/) - (Apache License, Version 2.0) Netty/Codec/HTTP2 (io.netty:netty-codec-http2:4.1.108.Final - https://netty.io/netty-codec-http2/) - (Apache License, Version 2.0) Netty/Codec/Socks (io.netty:netty-codec-socks:4.1.108.Final - https://netty.io/netty-codec-socks/) - (Apache License, Version 2.0) Netty/Common (io.netty:netty-common:4.1.108.Final - https://netty.io/netty-common/) - (Apache License, Version 2.0) Netty/Handler (io.netty:netty-handler:4.1.108.Final - https://netty.io/netty-handler/) - (Apache License, Version 2.0) Netty/Handler/Proxy (io.netty:netty-handler-proxy:4.1.108.Final - https://netty.io/netty-handler-proxy/) - (Apache License, Version 2.0) Netty/Resolver (io.netty:netty-resolver:4.1.108.Final - https://netty.io/netty-resolver/) - (Apache License, Version 2.0) Netty/Resolver/DNS (io.netty:netty-resolver-dns:4.1.108.Final - https://netty.io/netty-resolver-dns/) - (Apache License, Version 2.0) Netty/Transport (io.netty:netty-transport:4.1.108.Final - https://netty.io/netty-transport/) - (Apache License, Version 2.0) Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.108.Final - https://netty.io/netty-transport-native-unix-common/) - (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-api:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) - (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-context:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) (Apache License 2.0) swagger-annotations (io.swagger.core.v3:swagger-annotations:2.1.10 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-common (io.vertx:vertx-auth-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-common) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-oauth2 (io.vertx:vertx-auth-oauth2:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-oauth2) - (Eclipse Public License - v 2.0) (The Apache Software License, Version 2.0) Vert.x Core (io.vertx:vertx-core:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-core) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) Vert.x URI Template (io.vertx:vertx-uri-template:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-uri-template) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-client (io.vertx:vertx-web-client:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-client) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-common (io.vertx:vertx-web-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-common) (EDL 1.0) Jakarta Activation API (jakarta.activation:jakarta.activation-api:2.1.0 - https://github.com/eclipse-ee4j/jaf) (EPL 2.0) (GPL2 w/ CPE) Jakarta Annotations API (jakarta.annotation:jakarta.annotation-api:2.1.1 - https://projects.eclipse.org/projects/ee4j.ca) (The Apache Software License, Version 2.0) Jakarta Dependency Injection (jakarta.inject:jakarta.inject-api:2.0.1 - https://github.com/eclipse-ee4j/injection-api) @@ -126,8 +100,6 @@ Lists of 136 third-party dependencies. (Universal Permissive License, Version 1.0) Truffle Nfi Libffi (org.graalvm.truffle:truffle-nfi-libffi:23.1.2 - http://openjdk.java.net/projects/graal) (Universal Permissive License, Version 1.0) Truffle Runtime (org.graalvm.truffle:truffle-runtime:23.1.2 - http://openjdk.java.net/projects/graal) (Apache License 2.0) (LGPL 2.1) (MPL 1.1) Javassist (org.javassist:javassist:3.30.2-GA - https://www.javassist.org/) - (Apache License, version 2.0) JBoss Logging 3 (org.jboss.logging:jboss-logging:3.4.1.Final - http://www.jboss.org) - (Apache License 2.0) slf4j to JBoss Logging Adapter (org.jboss.slf4j:slf4j-jboss-logging:1.2.1.Final - http://www.jboss.org) (Eclipse Distribution License - v 1.0) Extended StAX API (org.jvnet.staxex:stax-ex:2.1.0 - https://projects.eclipse.org/projects/ee4j/stax-ex) (The Apache Software License, Version 2.0) LZ4 and xxHash (org.lz4:lz4-java:1.8.0 - https://github.com/lz4/lz4-java) (The MIT License) Project Lombok (org.projectlombok:lombok:1.18.32 - https://projectlombok.org) diff --git a/ksml-reporting/NOTICE.txt b/ksml-reporting/NOTICE.txt index 4a705eba..8aceb42f 100644 --- a/ksml-reporting/NOTICE.txt +++ b/ksml-reporting/NOTICE.txt @@ -1,5 +1,5 @@ -Lists of 159 third-party dependencies. +Lists of 130 third-party dependencies. (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Classic Module (ch.qos.logback:logback-classic:1.5.6 - http://logback.qos.ch/logback-classic) (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Core Module (ch.qos.logback:logback-core:1.5.6 - http://logback.qos.ch/logback-core) (MIT License) minimal-json (com.eclipsesource.minimal-json:minimal-json:0.9.5 - https://github.com/ralfstx/minimal-json) @@ -21,18 +21,16 @@ Lists of 159 third-party dependencies. (The Apache Software License, Version 2.0) Guava ListenableFuture only (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava - https://github.com/google/guava/listenablefuture) (Apache License, Version 2.0) J2ObjC Annotations (com.google.j2objc:j2objc-annotations:3.0.0 - https://github.com/google/j2objc/) (Go License) RE2/J (com.google.re2j:re2j:1.6 - http://github.com/google/re2j) - (MIT License) Microsoft Kiota-Java Abstractions (com.microsoft.kiota:microsoft-kiota-abstractions:1.1.14 - https://github.com/microsoft/kiota-java) - (MIT License) Microsoft Kiota-Java Serialization-Form (com.microsoft.kiota:microsoft-kiota-serialization-form:1.1.14 - https://github.com/microsoft/kiota-java) - (MIT License) Microsoft Kiota-Java Serialization-Multipart (com.microsoft.kiota:microsoft-kiota-serialization-multipart:1.1.14 - https://github.com/microsoft/kiota-java) - (MIT License) Microsoft Kiota-Java Serialization-Text (com.microsoft.kiota:microsoft-kiota-serialization-text:1.1.14 - https://github.com/microsoft/kiota-java) (Eclipse Distribution License - v 1.0) Jakarta SOAP Implementation (com.sun.xml.messaging.saaj:saaj-impl:3.0.4 - https://projects.eclipse.org/projects/ee4j/metro-saaj/saaj-impl) (Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.17.0 - https://commons.apache.org/proper/commons-codec/) (Apache-2.0) Apache Commons IO (commons-io:commons-io:2.16.1 - https://commons.apache.org/proper/commons-io/) - (Apache License Version 2.0) apicurio-registry-common (io.apicurio:apicurio-registry-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-common/) - (Apache License Version 2.0) apicurio-registry-java-sdk (io.apicurio:apicurio-registry-java-sdk:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-java-sdk/) - (Apache License Version 2.0) apicurio-registry-schema-resolver (io.apicurio:apicurio-registry-schema-resolver:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-schema-resolver/) - (Apache License Version 2.0) apicurio-registry-serde-common (io.apicurio:apicurio-registry-serde-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serde-common/) - (Apache License Version 2.0) apicurio-registry-serdes-avro-serde (io.apicurio:apicurio-registry-serdes-avro-serde:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serdes-avro-serde/) + (Apache License Version 2.0) apicurio-common-rest-client-common (io.apicurio:apicurio-common-rest-client-common:0.1.18.Final - https://www.apicur.io/apicurio-common-rest-client-common/) + (Apache License Version 2.0) apicurio-common-rest-client-jdk (io.apicurio:apicurio-common-rest-client-jdk:0.1.18.Final - https://www.apicur.io/apicurio-common-rest-client-jdk/) + (Apache License Version 2.0) apicurio-registry-client (io.apicurio:apicurio-registry-client:2.6.5.Final - https://www.apicur.io/apicurio-registry-client/) + (Apache License Version 2.0) apicurio-registry-common (io.apicurio:apicurio-registry-common:2.6.5.Final - https://www.apicur.io/apicurio-registry-common/) + (Apache License Version 2.0) apicurio-registry-schema-resolver (io.apicurio:apicurio-registry-schema-resolver:2.6.5.Final - https://www.apicur.io/apicurio-registry-schema-resolver/) + (Apache License Version 2.0) apicurio-registry-serde-common (io.apicurio:apicurio-registry-serde-common:2.6.5.Final - https://www.apicur.io/apicurio-registry-serde-common/) + (Apache License Version 2.0) apicurio-registry-serdes-avro-serde (io.apicurio:apicurio-registry-serdes-avro-serde:2.6.5.Final - https://www.apicur.io/apicurio-registry-serdes-avro-serde/) (Apache 2.0) KSML (io.axual.ksml:ksml:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml) (Apache 2.0) KSML Data Library (io.axual.ksml:ksml-data:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-data) (Apache 2.0) KSML Data Library - AVRO (io.axual.ksml:ksml-data-avro:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-data-avro) @@ -50,24 +48,6 @@ Lists of 159 third-party dependencies. (The Apache Software License, Version 2.0) Log Redactor Metrics (io.confluent:logredactor-metrics:1.0.12 - https://github.com/confluentinc/logredactor) (Apache License 2.0) Metrics Core (io.dropwizard.metrics:metrics-core:4.2.25 - https://metrics.dropwizard.io/metrics-core) (Apache License 2.0) Metrics Integration with JMX (io.dropwizard.metrics:metrics-jmx:4.2.25 - https://metrics.dropwizard.io/metrics-jmx) - (Apache License, Version 2.0) Standard Uri Template (io.github.std-uritemplate:std-uritemplate:0.0.59 - https://std-uritemplate.github.io/) - (Apache-2.0) Kiota - Http - Vert.X (io.kiota:kiota-http-vertx:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-http-vertx) - (Apache-2.0) Kiota - Serialization - Jackson (io.kiota:kiota-serialization-jackson:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-serialization-jackson-parent/kiota-serialization-jackson) - (Apache License, Version 2.0) Netty/Buffer (io.netty:netty-buffer:4.1.108.Final - https://netty.io/netty-buffer/) - (Apache License, Version 2.0) Netty/Codec (io.netty:netty-codec:4.1.108.Final - https://netty.io/netty-codec/) - (Apache License, Version 2.0) Netty/Codec/DNS (io.netty:netty-codec-dns:4.1.108.Final - https://netty.io/netty-codec-dns/) - (Apache License, Version 2.0) Netty/Codec/HTTP (io.netty:netty-codec-http:4.1.108.Final - https://netty.io/netty-codec-http/) - (Apache License, Version 2.0) Netty/Codec/HTTP2 (io.netty:netty-codec-http2:4.1.108.Final - https://netty.io/netty-codec-http2/) - (Apache License, Version 2.0) Netty/Codec/Socks (io.netty:netty-codec-socks:4.1.108.Final - https://netty.io/netty-codec-socks/) - (Apache License, Version 2.0) Netty/Common (io.netty:netty-common:4.1.108.Final - https://netty.io/netty-common/) - (Apache License, Version 2.0) Netty/Handler (io.netty:netty-handler:4.1.108.Final - https://netty.io/netty-handler/) - (Apache License, Version 2.0) Netty/Handler/Proxy (io.netty:netty-handler-proxy:4.1.108.Final - https://netty.io/netty-handler-proxy/) - (Apache License, Version 2.0) Netty/Resolver (io.netty:netty-resolver:4.1.108.Final - https://netty.io/netty-resolver/) - (Apache License, Version 2.0) Netty/Resolver/DNS (io.netty:netty-resolver-dns:4.1.108.Final - https://netty.io/netty-resolver-dns/) - (Apache License, Version 2.0) Netty/Transport (io.netty:netty-transport:4.1.108.Final - https://netty.io/netty-transport/) - (Apache License, Version 2.0) Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.108.Final - https://netty.io/netty-transport-native-unix-common/) - (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-api:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) - (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-context:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) (The Apache Software License, Version 2.0) Prometheus Metrics Config (io.prometheus:prometheus-metrics-config:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-config) (The Apache Software License, Version 2.0) Prometheus Metrics Core (io.prometheus:prometheus-metrics-core:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-core) (The Apache Software License, Version 2.0) Prometheus Metrics Exporter - Common (io.prometheus:prometheus-metrics-exporter-common:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-exporter-common) @@ -83,12 +63,6 @@ Lists of 159 third-party dependencies. (The Apache Software License, Version 2.0) Prometheus JMX Exporter - Collector (io.prometheus.jmx:collector:1.0.1 - http://github.com/prometheus/jmx_exporter) (The Apache Software License, Version 2.0) Prometheus JMX Exporter - Common (io.prometheus.jmx:jmx_prometheus_common:1.0.1 - http://github.com/prometheus/jmx_exporter) (Apache License 2.0) swagger-annotations (io.swagger.core.v3:swagger-annotations:2.1.10 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-common (io.vertx:vertx-auth-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-common) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-oauth2 (io.vertx:vertx-auth-oauth2:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-oauth2) - (Eclipse Public License - v 2.0) (The Apache Software License, Version 2.0) Vert.x Core (io.vertx:vertx-core:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-core) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) Vert.x URI Template (io.vertx:vertx-uri-template:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-uri-template) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-client (io.vertx:vertx-web-client:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-client) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-common (io.vertx:vertx-web-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-common) (EDL 1.0) Jakarta Activation API (jakarta.activation:jakarta.activation-api:2.1.3 - https://github.com/jakartaee/jaf-api) (EPL 2.0) (GPL2 w/ CPE) Jakarta Annotations API (jakarta.annotation:jakarta.annotation-api:2.1.1 - https://projects.eclipse.org/projects/ee4j.ca) (The Apache Software License, Version 2.0) Jakarta Dependency Injection (jakarta.inject:jakarta.inject-api:2.0.1 - https://github.com/eclipse-ee4j/injection-api) @@ -149,11 +123,8 @@ Lists of 159 third-party dependencies. (Universal Permissive License, Version 1.0) Truffle Nfi Libffi (org.graalvm.truffle:truffle-nfi-libffi:23.1.2 - http://openjdk.java.net/projects/graal) (Universal Permissive License, Version 1.0) Truffle Runtime (org.graalvm.truffle:truffle-runtime:23.1.2 - http://openjdk.java.net/projects/graal) (Apache License 2.0) (LGPL 2.1) (MPL 1.1) Javassist (org.javassist:javassist:3.30.2-GA - https://www.javassist.org/) - (Apache License, version 2.0) JBoss Logging 3 (org.jboss.logging:jboss-logging:3.4.1.Final - http://www.jboss.org) - (Apache License 2.0) slf4j to JBoss Logging Adapter (org.jboss.slf4j:slf4j-jboss-logging:1.2.1.Final - http://www.jboss.org) (Eclipse Distribution License - v 1.0) Extended StAX API (org.jvnet.staxex:stax-ex:2.1.0 - https://projects.eclipse.org/projects/ee4j/stax-ex) (The Apache Software License, Version 2.0) LZ4 and xxHash (org.lz4:lz4-java:1.8.0 - https://github.com/lz4/lz4-java) - (The MIT License) Project Lombok (org.projectlombok:lombok:1.18.32 - https://projectlombok.org) (Apache License 2.0) (GNU General Public License, version 2) RocksDB JNI (org.rocksdb:rocksdbjni:7.9.2 - https://rocksdb.org) (MIT License) SLF4J API Module (org.slf4j:slf4j-api:2.0.13 - http://www.slf4j.org) (Public Domain) XZ for Java (org.tukaani:xz:1.9 - https://tukaani.org/xz/java.html) diff --git a/ksml-runner/NOTICE.txt b/ksml-runner/NOTICE.txt index 7d2dfe5f..3e3865d7 100644 --- a/ksml-runner/NOTICE.txt +++ b/ksml-runner/NOTICE.txt @@ -1,5 +1,5 @@ -Lists of 158 third-party dependencies. +Lists of 130 third-party dependencies. (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Classic Module (ch.qos.logback:logback-classic:1.5.6 - http://logback.qos.ch/logback-classic) (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Core Module (ch.qos.logback:logback-core:1.5.6 - http://logback.qos.ch/logback-core) (MIT License) minimal-json (com.eclipsesource.minimal-json:minimal-json:0.9.5 - https://github.com/ralfstx/minimal-json) @@ -21,18 +21,16 @@ Lists of 158 third-party dependencies. (The Apache Software License, Version 2.0) Guava ListenableFuture only (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava - https://github.com/google/guava/listenablefuture) (Apache License, Version 2.0) J2ObjC Annotations (com.google.j2objc:j2objc-annotations:3.0.0 - https://github.com/google/j2objc/) (Go License) RE2/J (com.google.re2j:re2j:1.6 - http://github.com/google/re2j) - (MIT License) Microsoft Kiota-Java Abstractions (com.microsoft.kiota:microsoft-kiota-abstractions:1.1.14 - https://github.com/microsoft/kiota-java) - (MIT License) Microsoft Kiota-Java Serialization-Form (com.microsoft.kiota:microsoft-kiota-serialization-form:1.1.14 - https://github.com/microsoft/kiota-java) - (MIT License) Microsoft Kiota-Java Serialization-Multipart (com.microsoft.kiota:microsoft-kiota-serialization-multipart:1.1.14 - https://github.com/microsoft/kiota-java) - (MIT License) Microsoft Kiota-Java Serialization-Text (com.microsoft.kiota:microsoft-kiota-serialization-text:1.1.14 - https://github.com/microsoft/kiota-java) (Eclipse Distribution License - v 1.0) Jakarta SOAP Implementation (com.sun.xml.messaging.saaj:saaj-impl:3.0.4 - https://projects.eclipse.org/projects/ee4j/metro-saaj/saaj-impl) (Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.17.0 - https://commons.apache.org/proper/commons-codec/) (Apache-2.0) Apache Commons IO (commons-io:commons-io:2.16.1 - https://commons.apache.org/proper/commons-io/) - (Apache License Version 2.0) apicurio-registry-common (io.apicurio:apicurio-registry-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-common/) - (Apache License Version 2.0) apicurio-registry-java-sdk (io.apicurio:apicurio-registry-java-sdk:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-java-sdk/) - (Apache License Version 2.0) apicurio-registry-schema-resolver (io.apicurio:apicurio-registry-schema-resolver:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-schema-resolver/) - (Apache License Version 2.0) apicurio-registry-serde-common (io.apicurio:apicurio-registry-serde-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serde-common/) - (Apache License Version 2.0) apicurio-registry-serdes-avro-serde (io.apicurio:apicurio-registry-serdes-avro-serde:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serdes-avro-serde/) + (Apache License Version 2.0) apicurio-common-rest-client-common (io.apicurio:apicurio-common-rest-client-common:0.1.18.Final - https://www.apicur.io/apicurio-common-rest-client-common/) + (Apache License Version 2.0) apicurio-common-rest-client-jdk (io.apicurio:apicurio-common-rest-client-jdk:0.1.18.Final - https://www.apicur.io/apicurio-common-rest-client-jdk/) + (Apache License Version 2.0) apicurio-registry-client (io.apicurio:apicurio-registry-client:2.6.5.Final - https://www.apicur.io/apicurio-registry-client/) + (Apache License Version 2.0) apicurio-registry-common (io.apicurio:apicurio-registry-common:2.6.5.Final - https://www.apicur.io/apicurio-registry-common/) + (Apache License Version 2.0) apicurio-registry-schema-resolver (io.apicurio:apicurio-registry-schema-resolver:2.6.5.Final - https://www.apicur.io/apicurio-registry-schema-resolver/) + (Apache License Version 2.0) apicurio-registry-serde-common (io.apicurio:apicurio-registry-serde-common:2.6.5.Final - https://www.apicur.io/apicurio-registry-serde-common/) + (Apache License Version 2.0) apicurio-registry-serdes-avro-serde (io.apicurio:apicurio-registry-serdes-avro-serde:2.6.5.Final - https://www.apicur.io/apicurio-registry-serdes-avro-serde/) (Apache 2.0) KSML (io.axual.ksml:ksml:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml) (Apache 2.0) KSML Data Library (io.axual.ksml:ksml-data:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-data) (Apache 2.0) KSML Data Library - AVRO (io.axual.ksml:ksml-data-avro:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-data-avro) @@ -49,24 +47,6 @@ Lists of 158 third-party dependencies. (The Apache Software License, Version 2.0) Log Redactor Metrics (io.confluent:logredactor-metrics:1.0.12 - https://github.com/confluentinc/logredactor) (Apache License 2.0) Metrics Core (io.dropwizard.metrics:metrics-core:4.2.25 - https://metrics.dropwizard.io/metrics-core) (Apache License 2.0) Metrics Integration with JMX (io.dropwizard.metrics:metrics-jmx:4.2.25 - https://metrics.dropwizard.io/metrics-jmx) - (Apache License, Version 2.0) Standard Uri Template (io.github.std-uritemplate:std-uritemplate:0.0.59 - https://std-uritemplate.github.io/) - (Apache-2.0) Kiota - Http - Vert.X (io.kiota:kiota-http-vertx:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-http-vertx) - (Apache-2.0) Kiota - Serialization - Jackson (io.kiota:kiota-serialization-jackson:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-serialization-jackson-parent/kiota-serialization-jackson) - (Apache License, Version 2.0) Netty/Buffer (io.netty:netty-buffer:4.1.108.Final - https://netty.io/netty-buffer/) - (Apache License, Version 2.0) Netty/Codec (io.netty:netty-codec:4.1.108.Final - https://netty.io/netty-codec/) - (Apache License, Version 2.0) Netty/Codec/DNS (io.netty:netty-codec-dns:4.1.108.Final - https://netty.io/netty-codec-dns/) - (Apache License, Version 2.0) Netty/Codec/HTTP (io.netty:netty-codec-http:4.1.108.Final - https://netty.io/netty-codec-http/) - (Apache License, Version 2.0) Netty/Codec/HTTP2 (io.netty:netty-codec-http2:4.1.108.Final - https://netty.io/netty-codec-http2/) - (Apache License, Version 2.0) Netty/Codec/Socks (io.netty:netty-codec-socks:4.1.108.Final - https://netty.io/netty-codec-socks/) - (Apache License, Version 2.0) Netty/Common (io.netty:netty-common:4.1.108.Final - https://netty.io/netty-common/) - (Apache License, Version 2.0) Netty/Handler (io.netty:netty-handler:4.1.108.Final - https://netty.io/netty-handler/) - (Apache License, Version 2.0) Netty/Handler/Proxy (io.netty:netty-handler-proxy:4.1.108.Final - https://netty.io/netty-handler-proxy/) - (Apache License, Version 2.0) Netty/Resolver (io.netty:netty-resolver:4.1.108.Final - https://netty.io/netty-resolver/) - (Apache License, Version 2.0) Netty/Resolver/DNS (io.netty:netty-resolver-dns:4.1.108.Final - https://netty.io/netty-resolver-dns/) - (Apache License, Version 2.0) Netty/Transport (io.netty:netty-transport:4.1.108.Final - https://netty.io/netty-transport/) - (Apache License, Version 2.0) Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.108.Final - https://netty.io/netty-transport-native-unix-common/) - (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-api:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) - (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-context:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) (The Apache Software License, Version 2.0) Prometheus Metrics Config (io.prometheus:prometheus-metrics-config:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-config) (The Apache Software License, Version 2.0) Prometheus Metrics Core (io.prometheus:prometheus-metrics-core:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-core) (The Apache Software License, Version 2.0) Prometheus Metrics Exporter - Common (io.prometheus:prometheus-metrics-exporter-common:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-exporter-common) @@ -82,12 +62,6 @@ Lists of 158 third-party dependencies. (The Apache Software License, Version 2.0) Prometheus JMX Exporter - Collector (io.prometheus.jmx:collector:1.0.1 - http://github.com/prometheus/jmx_exporter) (The Apache Software License, Version 2.0) Prometheus JMX Exporter - Common (io.prometheus.jmx:jmx_prometheus_common:1.0.1 - http://github.com/prometheus/jmx_exporter) (Apache License 2.0) swagger-annotations (io.swagger.core.v3:swagger-annotations:2.1.10 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-common (io.vertx:vertx-auth-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-common) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-oauth2 (io.vertx:vertx-auth-oauth2:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-oauth2) - (Eclipse Public License - v 2.0) (The Apache Software License, Version 2.0) Vert.x Core (io.vertx:vertx-core:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-core) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) Vert.x URI Template (io.vertx:vertx-uri-template:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-uri-template) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-client (io.vertx:vertx-web-client:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-client) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-common (io.vertx:vertx-web-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-common) (EDL 1.0) Jakarta Activation API (jakarta.activation:jakarta.activation-api:2.1.3 - https://github.com/jakartaee/jaf-api) (EPL 2.0) (GPL2 w/ CPE) Jakarta Annotations API (jakarta.annotation:jakarta.annotation-api:2.1.1 - https://projects.eclipse.org/projects/ee4j.ca) (The Apache Software License, Version 2.0) Jakarta Dependency Injection (jakarta.inject:jakarta.inject-api:2.0.1 - https://github.com/eclipse-ee4j/injection-api) @@ -148,8 +122,6 @@ Lists of 158 third-party dependencies. (Universal Permissive License, Version 1.0) Truffle Nfi Libffi (org.graalvm.truffle:truffle-nfi-libffi:23.1.2 - http://openjdk.java.net/projects/graal) (Universal Permissive License, Version 1.0) Truffle Runtime (org.graalvm.truffle:truffle-runtime:23.1.2 - http://openjdk.java.net/projects/graal) (Apache License 2.0) (LGPL 2.1) (MPL 1.1) Javassist (org.javassist:javassist:3.30.2-GA - https://www.javassist.org/) - (Apache License, version 2.0) JBoss Logging 3 (org.jboss.logging:jboss-logging:3.4.1.Final - http://www.jboss.org) - (Apache License 2.0) slf4j to JBoss Logging Adapter (org.jboss.slf4j:slf4j-jboss-logging:1.2.1.Final - http://www.jboss.org) (Eclipse Distribution License - v 1.0) Extended StAX API (org.jvnet.staxex:stax-ex:2.1.0 - https://projects.eclipse.org/projects/ee4j/stax-ex) (The Apache Software License, Version 2.0) LZ4 and xxHash (org.lz4:lz4-java:1.8.0 - https://github.com/lz4/lz4-java) (The MIT License) Project Lombok (org.projectlombok:lombok:1.18.32 - https://projectlombok.org) diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java index d5aa9fcd..3b22817e 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java @@ -84,7 +84,7 @@ public static void main(String[] args) { } final var config = readConfiguration(configFile); - final var ksmlConfig = config.ksml(); + final var ksmlConfig = config.ksmlConfig(); log.info("Using directories: config: {}, schema: {}, storage: {}", ksmlConfig.getConfigDirectory(), ksmlConfig.getSchemaDirectory(), ksmlConfig.getStorageDirectory()); final var definitions = ksmlConfig.getDefinitions(); if (definitions == null || definitions.isEmpty()) { @@ -104,34 +104,31 @@ public static void main(String[] args) { } // Set up all default notations and register them in the NotationLibrary - final var notationFactories = new NotationFactories(config.kafka().kafkaConfig(), ksmlConfig.getSchemaDirectory()); + final var notationFactories = new NotationFactories(config.kafkaConfig(), ksmlConfig.getSchemaDirectory()); for (final var notation : notationFactories.notations().entrySet()) { NotationLibrary.register(notation.getKey(), notation.getValue().create(null)); } // Set up all notation overrides from the KSML config for (final var notationEntry : ksmlConfig.notations().entrySet()) { - final var notationStr = notationEntry.getKey() != null ? notationEntry.getKey() : "undefined"; + final var notation = notationEntry.getKey(); final var notationConfig = notationEntry.getValue(); - final var factoryName = notationConfig != null ? notationConfig.type() : "unknown"; - if (notationConfig != null && factoryName != null) { - final var factory = notationFactories.notations().get(factoryName); - if (factory == null) { - throw FatalError.reportAndExit(new ConfigException("Unknown notation type: " + factoryName)); + if (notation != null && notationConfig != null) { + final var n = notationFactories.notations().get(notationConfig.type()); + if (n == null) { + throw FatalError.reportAndExit(new ConfigException("Notation type '" + notationConfig.type() + "' not found")); } - NotationLibrary.register(notationStr, factory.create(notationConfig.config())); - } else { - log.warn("Notation configuration incomplete: notation=" + notationStr + ", type=" + factoryName); + NotationLibrary.register(notation, n.create(notationConfig.config())); } } - // Ensure typical defaults are used for AVRO - // WARNING: Defaults for notations will be deprecated in the future. Make sure you explicitly configure - // notations with multiple implementations (like AVRO) in your ksml-runner.yaml. + // Ensure typical defaults are used + // WARNING: Defaults for notations will be deprecated in the future. Make sure you explicitly define + // notations that have multiple implementations in your ksml-runner.yaml. if (!NotationLibrary.exists(NotationFactories.AVRO)) { final var defaultAvro = notationFactories.confluentAvro(); NotationLibrary.register(NotationFactories.AVRO, defaultAvro.create(null)); - log.warn("No implementation specified for AVRO notation. If you use AVRO in your KSML definition, add the required configuration to the ksml-runner.yaml"); + log.warn("No implementation specified for AVRO notation. If you plan to use AVRO, add the required implementation to the ksml-runner.yaml"); } final var errorHandling = ksmlConfig.getErrorHandlingConfig(); @@ -142,8 +139,8 @@ public static void main(String[] args) { } ExecutionContext.INSTANCE.serdeWrapper( serde -> new Serdes.WrapperSerde<>( - new ResolvingSerializer<>(serde.serializer(), config.kafka().kafkaConfig()), - new ResolvingDeserializer<>(serde.deserializer(), config.kafka().kafkaConfig()))); + new ResolvingSerializer<>(serde.serializer(), config.kafkaConfig()), + new ResolvingDeserializer<>(serde.deserializer(), config.kafkaConfig()))); final Map producerSpecs = new HashMap<>(); final Map pipelineSpecs = new HashMap<>(); @@ -166,13 +163,13 @@ public static void main(String[] args) { final var producer = producerSpecs.isEmpty() ? null : new KafkaProducerRunner(KafkaProducerRunner.Config.builder() .definitions(producerSpecs) - .kafkaConfig(config.kafka().kafkaConfig()) + .kafkaConfig(config.kafkaConfig()) .build()); final var streams = pipelineSpecs.isEmpty() ? null : new KafkaStreamsRunner(KafkaStreamsRunner.Config.builder() .storageDirectory(ksmlConfig.getStorageDirectory()) .appServer(ksmlConfig.getApplicationServerConfig()) .definitions(pipelineSpecs) - .kafkaConfig(config.kafka().kafkaConfig()) + .kafkaConfig(config.kafkaConfig()) .build()); if (producer != null || streams != null) { @@ -188,7 +185,7 @@ public static void main(String[] args) { Runtime.getRuntime().addShutdownHook(shutdownHook); - try (var prometheusExport = new PrometheusExport(config.ksml().prometheusConfig())) { + try (var prometheusExport = new PrometheusExport(config.ksmlConfig().prometheusConfig())) { prometheusExport.start(); final var executorService = Executors.newFixedThreadPool(2); @@ -361,10 +358,10 @@ private static KSMLRunnerConfig readConfiguration(File configFile) { try { final var config = mapper.readValue(configFile, KSMLRunnerConfig.class); if (config != null) { - if (config.ksml() == null) { + if (config.ksmlConfig() == null) { throw new ConfigException("Section \"ksml\" is missing in configuration"); } - if (config.kafka() == null) { + if (config.kafkaConfig() == null) { throw new ConfigException("Section \"kafka\" is missing in configuration"); } return config; diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/ErrorHandlingConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/ErrorHandlingConfig.java index ea4815e2..dc3590bb 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/ErrorHandlingConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/ErrorHandlingConfig.java @@ -22,9 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.Builder; import lombok.Data; -import lombok.extern.jackson.Jacksonized; @JsonIgnoreProperties(ignoreUnknown = true) @Data diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java index df8c4292..e694c53b 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java @@ -41,20 +41,26 @@ public class KSMLRunnerConfig { private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); @JsonProperty("ksml") - private KSMLConfig ksml; + private KSMLConfig ksmlConfig; @JsonProperty("kafka") - private KafkaConfig kafka; + private KafkaConfig kafkaConfig; public String applicationId() { - return kafka.applicationId; + return kafkaConfig.applicationId; + } + + public Map kafkaConfig() { + var newConfig = new HashMap<>(kafkaConfig.kafkaConfig()); + newConfig.put("application.id", kafkaConfig.applicationId()); + return newConfig; } @Data public static class KafkaConfig { @JsonProperty("app.id") @JsonAlias({"applicationId", "application.id"}) - private String applicationId; + public String applicationId; @JsonIgnore private Map kafkaConfig = new HashMap<>(); diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/NotationConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/NotationConfig.java index ef2d040e..01091bc1 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/NotationConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/NotationConfig.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Builder; -import lombok.Data; import lombok.extern.jackson.Jacksonized; import java.util.Map; diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config b/ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config new file mode 100644 index 00000000..881a8aa5 --- /dev/null +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config @@ -0,0 +1,3 @@ +lombok.accessors.fluent=true +lombok.accessors.chain=false +lombok.equalsAndHashCode.callSuper=call \ No newline at end of file diff --git a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java index b536e067..90da3a51 100644 --- a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java +++ b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java @@ -45,7 +45,7 @@ void shouldLoadWithoutExceptions() throws IOException { final var yaml = getClass().getClassLoader().getResourceAsStream("ksml-runner-config.yaml"); final var ksmlRunnerConfig = objectMapper.readValue(yaml, KSMLRunnerConfig.class); - assertNotNull(ksmlRunnerConfig.ksml()); - assertNotNull(ksmlRunnerConfig.kafka()); + assertNotNull(ksmlRunnerConfig.ksmlConfig()); + assertNotNull(ksmlRunnerConfig.kafkaConfig()); } } diff --git a/ksml/NOTICE.txt b/ksml/NOTICE.txt index f7b17223..0ee289c8 100644 --- a/ksml/NOTICE.txt +++ b/ksml/NOTICE.txt @@ -1,5 +1,5 @@ -Lists of 109 third-party dependencies. +Lists of 79 third-party dependencies. (MIT License) minimal-json (com.eclipsesource.minimal-json:minimal-json:0.9.5 - https://github.com/ralfstx/minimal-json) (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.17.1 - https://github.com/FasterXML/jackson) (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.17.1 - https://github.com/FasterXML/jackson-core) @@ -16,18 +16,16 @@ Lists of 109 third-party dependencies. (The Apache Software License, Version 2.0) Guava ListenableFuture only (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava - https://github.com/google/guava/listenablefuture) (Apache License, Version 2.0) J2ObjC Annotations (com.google.j2objc:j2objc-annotations:3.0.0 - https://github.com/google/j2objc/) (Go License) RE2/J (com.google.re2j:re2j:1.6 - http://github.com/google/re2j) - (MIT License) Microsoft Kiota-Java Abstractions (com.microsoft.kiota:microsoft-kiota-abstractions:1.1.14 - https://github.com/microsoft/kiota-java) - (MIT License) Microsoft Kiota-Java Serialization-Form (com.microsoft.kiota:microsoft-kiota-serialization-form:1.1.14 - https://github.com/microsoft/kiota-java) - (MIT License) Microsoft Kiota-Java Serialization-Multipart (com.microsoft.kiota:microsoft-kiota-serialization-multipart:1.1.14 - https://github.com/microsoft/kiota-java) - (MIT License) Microsoft Kiota-Java Serialization-Text (com.microsoft.kiota:microsoft-kiota-serialization-text:1.1.14 - https://github.com/microsoft/kiota-java) (Eclipse Distribution License - v 1.0) Jakarta SOAP Implementation (com.sun.xml.messaging.saaj:saaj-impl:3.0.4 - https://projects.eclipse.org/projects/ee4j/metro-saaj/saaj-impl) (Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.17.0 - https://commons.apache.org/proper/commons-codec/) (Apache-2.0) Apache Commons IO (commons-io:commons-io:2.16.1 - https://commons.apache.org/proper/commons-io/) - (Apache License Version 2.0) apicurio-registry-common (io.apicurio:apicurio-registry-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-common/) - (Apache License Version 2.0) apicurio-registry-java-sdk (io.apicurio:apicurio-registry-java-sdk:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-java-sdk/) - (Apache License Version 2.0) apicurio-registry-schema-resolver (io.apicurio:apicurio-registry-schema-resolver:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-schema-resolver/) - (Apache License Version 2.0) apicurio-registry-serde-common (io.apicurio:apicurio-registry-serde-common:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serde-common/) - (Apache License Version 2.0) apicurio-registry-serdes-avro-serde (io.apicurio:apicurio-registry-serdes-avro-serde:3.0.0-SNAPSHOT - https://www.apicur.io/apicurio-registry-serdes-avro-serde/) + (Apache License Version 2.0) apicurio-common-rest-client-common (io.apicurio:apicurio-common-rest-client-common:0.1.18.Final - https://www.apicur.io/apicurio-common-rest-client-common/) + (Apache License Version 2.0) apicurio-common-rest-client-jdk (io.apicurio:apicurio-common-rest-client-jdk:0.1.18.Final - https://www.apicur.io/apicurio-common-rest-client-jdk/) + (Apache License Version 2.0) apicurio-registry-client (io.apicurio:apicurio-registry-client:2.6.5.Final - https://www.apicur.io/apicurio-registry-client/) + (Apache License Version 2.0) apicurio-registry-common (io.apicurio:apicurio-registry-common:2.6.5.Final - https://www.apicur.io/apicurio-registry-common/) + (Apache License Version 2.0) apicurio-registry-schema-resolver (io.apicurio:apicurio-registry-schema-resolver:2.6.5.Final - https://www.apicur.io/apicurio-registry-schema-resolver/) + (Apache License Version 2.0) apicurio-registry-serde-common (io.apicurio:apicurio-registry-serde-common:2.6.5.Final - https://www.apicur.io/apicurio-registry-serde-common/) + (Apache License Version 2.0) apicurio-registry-serdes-avro-serde (io.apicurio:apicurio-registry-serdes-avro-serde:2.6.5.Final - https://www.apicur.io/apicurio-registry-serdes-avro-serde/) (Apache 2.0) KSML Data Library (io.axual.ksml:ksml-data:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-data) (Apache 2.0) KSML Data Library - AVRO (io.axual.ksml:ksml-data-avro:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-data-avro) (Apache 2.0) KSML Data Library - CSV (io.axual.ksml:ksml-data-csv:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-data-csv) @@ -41,35 +39,9 @@ Lists of 109 third-party dependencies. (The Apache Software License, Version 2.0) Log Redactor Metrics (io.confluent:logredactor-metrics:1.0.12 - https://github.com/confluentinc/logredactor) (Apache License 2.0) Metrics Core (io.dropwizard.metrics:metrics-core:4.2.25 - https://metrics.dropwizard.io/metrics-core) (Apache License 2.0) Metrics Integration with JMX (io.dropwizard.metrics:metrics-jmx:4.2.25 - https://metrics.dropwizard.io/metrics-jmx) - (Apache License, Version 2.0) Standard Uri Template (io.github.std-uritemplate:std-uritemplate:0.0.59 - https://std-uritemplate.github.io/) - (Apache-2.0) Kiota - Http - Vert.X (io.kiota:kiota-http-vertx:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-http-vertx) - (Apache-2.0) Kiota - Serialization - Jackson (io.kiota:kiota-serialization-jackson:0.0.17 - https://github.com/kiota-community/kiota-java-extra/kiota-libraries-parent/kiota-serialization-jackson-parent/kiota-serialization-jackson) - (Apache License, Version 2.0) Netty/Buffer (io.netty:netty-buffer:4.1.108.Final - https://netty.io/netty-buffer/) - (Apache License, Version 2.0) Netty/Codec (io.netty:netty-codec:4.1.108.Final - https://netty.io/netty-codec/) - (Apache License, Version 2.0) Netty/Codec/DNS (io.netty:netty-codec-dns:4.1.108.Final - https://netty.io/netty-codec-dns/) - (Apache License, Version 2.0) Netty/Codec/HTTP (io.netty:netty-codec-http:4.1.108.Final - https://netty.io/netty-codec-http/) - (Apache License, Version 2.0) Netty/Codec/HTTP2 (io.netty:netty-codec-http2:4.1.108.Final - https://netty.io/netty-codec-http2/) - (Apache License, Version 2.0) Netty/Codec/Socks (io.netty:netty-codec-socks:4.1.108.Final - https://netty.io/netty-codec-socks/) - (Apache License, Version 2.0) Netty/Common (io.netty:netty-common:4.1.108.Final - https://netty.io/netty-common/) - (Apache License, Version 2.0) Netty/Handler (io.netty:netty-handler:4.1.108.Final - https://netty.io/netty-handler/) - (Apache License, Version 2.0) Netty/Handler/Proxy (io.netty:netty-handler-proxy:4.1.108.Final - https://netty.io/netty-handler-proxy/) - (Apache License, Version 2.0) Netty/Resolver (io.netty:netty-resolver:4.1.108.Final - https://netty.io/netty-resolver/) - (Apache License, Version 2.0) Netty/Resolver/DNS (io.netty:netty-resolver-dns:4.1.108.Final - https://netty.io/netty-resolver-dns/) - (Apache License, Version 2.0) Netty/Transport (io.netty:netty-transport:4.1.108.Final - https://netty.io/netty-transport/) - (Apache License, Version 2.0) Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.108.Final - https://netty.io/netty-transport-native-unix-common/) - (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-api:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) - (The Apache License, Version 2.0) OpenTelemetry Java (io.opentelemetry:opentelemetry-context:1.39.0 - https://github.com/open-telemetry/opentelemetry-java) (Apache License 2.0) swagger-annotations (io.swagger.core.v3:swagger-annotations:2.1.10 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-common (io.vertx:vertx-auth-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-common) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-auth-oauth2 (io.vertx:vertx-auth-oauth2:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-auth-parent/vertx-auth-oauth2) - (Eclipse Public License - v 2.0) (The Apache Software License, Version 2.0) Vert.x Core (io.vertx:vertx-core:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-core) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) Vert.x URI Template (io.vertx:vertx-uri-template:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-uri-template) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-client (io.vertx:vertx-web-client:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-client) - (Eclipse Public License - v 1.0) (The Apache Software License, Version 2.0) vertx-web-common (io.vertx:vertx-web-common:4.5.7 - http://nexus.sonatype.org/oss-repository-hosting.html/vertx-parent/vertx-ext/vertx-ext-parent/vertx-web-parent/vertx-web-common) (EDL 1.0) Jakarta Activation API (jakarta.activation:jakarta.activation-api:2.1.3 - https://github.com/jakartaee/jaf-api) - (EPL 2.0) (GPL2 w/ CPE) Jakarta Annotations API (jakarta.annotation:jakarta.annotation-api:3.0.0 - https://projects.eclipse.org/projects/ee4j.ca) (Apache License 2.0) Jakarta Bean Validation API (jakarta.validation:jakarta.validation-api:3.0.2 - https://beanvalidation.org) - (EPL-2.0) (GPL-2.0-with-classpath-exception) Jakarta RESTful WS API (jakarta.ws.rs:jakarta.ws.rs-api:3.1.0 - https://github.com/eclipse-ee4j/jaxrs-api) (Eclipse Distribution License - v 1.0) Jakarta SOAP with Attachments API (jakarta.xml.soap:jakarta.xml.soap-api:3.0.2 - https://github.com/jakartaee/saaj-api) (Apache-2.0) Apache Avro (org.apache.avro:avro:1.12.0 - https://avro.apache.org) (Apache-2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.26.2 - https://commons.apache.org/proper/commons-compress/) @@ -99,8 +71,6 @@ Lists of 109 third-party dependencies. (Universal Permissive License, Version 1.0) Truffle Nfi (org.graalvm.truffle:truffle-nfi:23.1.2 - http://openjdk.java.net/projects/graal) (Universal Permissive License, Version 1.0) Truffle Nfi Libffi (org.graalvm.truffle:truffle-nfi-libffi:23.1.2 - http://openjdk.java.net/projects/graal) (Universal Permissive License, Version 1.0) Truffle Runtime (org.graalvm.truffle:truffle-runtime:23.1.2 - http://openjdk.java.net/projects/graal) - (Apache License, version 2.0) JBoss Logging 3 (org.jboss.logging:jboss-logging:3.4.1.Final - http://www.jboss.org) - (Apache License 2.0) slf4j to JBoss Logging Adapter (org.jboss.slf4j:slf4j-jboss-logging:1.2.1.Final - http://www.jboss.org) (Eclipse Distribution License - v 1.0) Extended StAX API (org.jvnet.staxex:stax-ex:2.1.0 - https://projects.eclipse.org/projects/ee4j/stax-ex) (The Apache Software License, Version 2.0) LZ4 and xxHash (org.lz4:lz4-java:1.8.0 - https://github.com/lz4/lz4-java) (The MIT License) Project Lombok (org.projectlombok:lombok:1.18.32 - https://projectlombok.org) From f6e2abd6a76142bb70932f0173607f4d5049cb7b Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:15:14 +0100 Subject: [PATCH 22/55] Updates to the code # Conflicts: # ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java # ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java --- .../java/io/axual/ksml/runner/config/KSMLRunnerConfig.java | 2 +- .../io/axual/ksml/runner/config/KSMLRunnerConfigTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java index e694c53b..41c5bcf0 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLRunnerConfig.java @@ -60,7 +60,7 @@ public Map kafkaConfig() { public static class KafkaConfig { @JsonProperty("app.id") @JsonAlias({"applicationId", "application.id"}) - public String applicationId; + private String applicationId; @JsonIgnore private Map kafkaConfig = new HashMap<>(); diff --git a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java index 90da3a51..b536e067 100644 --- a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java +++ b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java @@ -45,7 +45,7 @@ void shouldLoadWithoutExceptions() throws IOException { final var yaml = getClass().getClassLoader().getResourceAsStream("ksml-runner-config.yaml"); final var ksmlRunnerConfig = objectMapper.readValue(yaml, KSMLRunnerConfig.class); - assertNotNull(ksmlRunnerConfig.ksmlConfig()); - assertNotNull(ksmlRunnerConfig.kafkaConfig()); + assertNotNull(ksmlRunnerConfig.ksml()); + assertNotNull(ksmlRunnerConfig.kafka()); } } From acce8ef3b9e564fdaa8ae0e962697016fac76206 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:17:53 +0100 Subject: [PATCH 23/55] Improved configuration syntax + introduced named notation factories # Conflicts: # ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java --- .../java/io/axual/ksml/runner/KSMLRunner.java | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java index 3b22817e..e0ee95af 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java @@ -111,24 +111,27 @@ public static void main(String[] args) { // Set up all notation overrides from the KSML config for (final var notationEntry : ksmlConfig.notations().entrySet()) { - final var notation = notationEntry.getKey(); + final var notationStr = notationEntry.getKey() != null ? notationEntry.getKey() : "undefined"; final var notationConfig = notationEntry.getValue(); - if (notation != null && notationConfig != null) { - final var n = notationFactories.notations().get(notationConfig.type()); - if (n == null) { - throw FatalError.reportAndExit(new ConfigException("Notation type '" + notationConfig.type() + "' not found")); + final var factoryName = notationConfig != null ? notationConfig.type() : "unknown"; + if (notationConfig != null && factoryName != null) { + final var factory = notationFactories.notations().get(factoryName); + if (factory == null) { + throw FatalError.reportAndExit(new ConfigException("Unknown notation type: " + factoryName)); } - NotationLibrary.register(notation, n.create(notationConfig.config())); + NotationLibrary.register(notationStr, factory.create(notationConfig.config())); + } else { + log.warn("Notation configuration incomplete: notation=" + notationStr + ", type=" + factoryName); } } - // Ensure typical defaults are used - // WARNING: Defaults for notations will be deprecated in the future. Make sure you explicitly define - // notations that have multiple implementations in your ksml-runner.yaml. + // Ensure typical defaults are used for AVRO + // WARNING: Defaults for notations will be deprecated in the future. Make sure you explicitly configure + // notations with multiple implementations (like AVRO) in your ksml-runner.yaml. if (!NotationLibrary.exists(NotationFactories.AVRO)) { final var defaultAvro = notationFactories.confluentAvro(); NotationLibrary.register(NotationFactories.AVRO, defaultAvro.create(null)); - log.warn("No implementation specified for AVRO notation. If you plan to use AVRO, add the required implementation to the ksml-runner.yaml"); + log.warn("No implementation specified for AVRO notation. If you use AVRO in your KSML definition, add the required configuration to the ksml-runner.yaml"); } final var errorHandling = ksmlConfig.getErrorHandlingConfig(); From b11e59f572d7fe4124ea9fb67f979515407a488e Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:21:22 +0100 Subject: [PATCH 24/55] Update documentation and example runner definitions # Conflicts: # examples/ksml-data-generator.yaml # examples/ksml-runner.yaml --- examples/ksml-data-generator.yaml | 2 -- examples/ksml-runner.yaml | 1 - 2 files changed, 3 deletions(-) diff --git a/examples/ksml-data-generator.yaml b/examples/ksml-data-generator.yaml index f7848e2f..f59ee570 100644 --- a/examples/ksml-data-generator.yaml +++ b/examples/ksml-data-generator.yaml @@ -81,7 +81,6 @@ ksml: # apicurio.registry.request.ssl.truststore.type: JKS # apicurio.registry.request.ssl.truststore.password: password - enableProducers: true # Set to true to allow producer definitions to be parsed in the KSML definitions and be executed. enablePipelines: false # Set to true to allow pipeline definitions to be parsed in the KSML definitions and be executed. @@ -118,7 +117,6 @@ kafka: # These patterns are resolved into the actual name used on Kafka using the values in this configuration map # and the topic names specified in the definition YAML files - # tenant: "ksmldemo" # instance: "dta" # environment: "dev" diff --git a/examples/ksml-runner.yaml b/examples/ksml-runner.yaml index 974beb24..46991fb2 100644 --- a/examples/ksml-runner.yaml +++ b/examples/ksml-runner.yaml @@ -119,7 +119,6 @@ kafka: auto.offset.reset: earliest acks: all - # These are Kafka SSL configuration properties. Check the documentation at1 # Check the documentation at https://kafka.apache.org/documentation/#producerconfigs for more properties From ed4c6aa0bd081b1719fe06b852407946e61a9ccd Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:21:55 +0100 Subject: [PATCH 25/55] Update apicurio serde version # Conflicts: # pom.xml From 13bc30cec0f0b34483c711cf39624d914e451695 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:25:45 +0100 Subject: [PATCH 26/55] Updated library for Apicurio serdes --- .../io/axual/ksml/runner/config/KSMLRunnerConfigTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java index b536e067..90da3a51 100644 --- a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java +++ b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java @@ -45,7 +45,7 @@ void shouldLoadWithoutExceptions() throws IOException { final var yaml = getClass().getClassLoader().getResourceAsStream("ksml-runner-config.yaml"); final var ksmlRunnerConfig = objectMapper.readValue(yaml, KSMLRunnerConfig.class); - assertNotNull(ksmlRunnerConfig.ksml()); - assertNotNull(ksmlRunnerConfig.kafka()); + assertNotNull(ksmlRunnerConfig.ksmlConfig()); + assertNotNull(ksmlRunnerConfig.kafkaConfig()); } } From a59e74ffdb111e8b997c87ce09ffb95fe5569a1d Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:26:50 +0100 Subject: [PATCH 27/55] Processed Richard's review comments for MR From f434d2189691b2c13f62c606c3f8ebc34fe5e9eb Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 30 Oct 2024 18:35:53 +0100 Subject: [PATCH 28/55] Move lombok config to parent project level, remove submodule definitions --- ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config b/ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config deleted file mode 100644 index 881a8aa5..00000000 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config +++ /dev/null @@ -1,3 +0,0 @@ -lombok.accessors.fluent=true -lombok.accessors.chain=false -lombok.equalsAndHashCode.callSuper=call \ No newline at end of file From 72f67031bbf3ef1214bea5b8a07012834e47125b Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:29:36 +0100 Subject: [PATCH 29/55] Add header support to serialisation/deserialisation to allow Apicurio notation to use headers From d3c248fa9e08cec97270a1a417af6d96918518c0 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Fri, 1 Nov 2024 10:33:32 +0100 Subject: [PATCH 30/55] Improve detection of input, intermediate, output and internal topics --- .../java/io/axual/ksml/TopologyGenerator.java | 50 +++++++++---------- .../ksml/generator/TopologyAnalyzer.java | 47 +++++++++-------- 2 files changed, 49 insertions(+), 48 deletions(-) diff --git a/ksml/src/main/java/io/axual/ksml/TopologyGenerator.java b/ksml/src/main/java/io/axual/ksml/TopologyGenerator.java index 4665c39d..401f02a7 100644 --- a/ksml/src/main/java/io/axual/ksml/TopologyGenerator.java +++ b/ksml/src/main/java/io/axual/ksml/TopologyGenerator.java @@ -21,17 +21,6 @@ */ -import org.apache.kafka.streams.StreamsBuilder; -import org.apache.kafka.streams.StreamsConfig; -import org.apache.kafka.streams.Topology; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashSet; -import java.util.Map; -import java.util.Properties; -import java.util.Set; - import io.axual.ksml.definition.GlobalTableDefinition; import io.axual.ksml.definition.StateStoreDefinition; import io.axual.ksml.definition.TableDefinition; @@ -42,6 +31,13 @@ import io.axual.ksml.operation.StreamOperation; import io.axual.ksml.operation.ToOperation; import io.axual.ksml.stream.StreamWrapper; +import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.Topology; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; public class TopologyGenerator { private static final Logger LOG = LoggerFactory.getLogger(TopologyGenerator.class); @@ -56,7 +52,7 @@ public TopologyGenerator(String applicationId, String optimization) { // Parse configuration this.applicationId = applicationId; this.optimization = new Properties(); - if(optimization != null) { + if (optimization != null) { this.optimization.put(StreamsConfig.TOPOLOGY_OPTIMIZATION_CONFIG, optimization); } } @@ -64,16 +60,16 @@ public TopologyGenerator(String applicationId, String optimization) { public Topology create(StreamsBuilder streamsBuilder, Map definitions) { if (definitions.isEmpty()) return null; - final var knownTopics = new HashSet(); + final var stores = new TreeMap(); definitions.forEach((name, definition) -> { final var context = new TopologyBuildContext(streamsBuilder, definition); generate(definition, context); + stores.putAll(definition.stateStores()); }); - var topology = streamsBuilder.build(optimization); - var analysis = TopologyAnalyzer.analyze(topology, applicationId, knownTopics); + var analysis = TopologyAnalyzer.analyze(topology, applicationId); StringBuilder summary = new StringBuilder("\n\n"); summary.append(topology != null ? topology.describe() : "null"); @@ -82,10 +78,11 @@ public Topology create(StreamsBuilder streamsBuilder, Map "keyValue"; case SESSION_STORE -> "session"; case WINDOW_STORE -> "window"; }) .append("/") - .append(entry.getKey()) - .append("/"); + .append(storeEntry.getKey()) + .append("/") + .append("\n"); } } @@ -213,7 +211,7 @@ private void generate(TopologyDefinition specification, TopologyBuildContext con LOG.info(""" Generating Kafka Streams topology for pipeline {}: {} - """,name,tsBuilder); + """, name, tsBuilder); }); } diff --git a/ksml/src/main/java/io/axual/ksml/generator/TopologyAnalyzer.java b/ksml/src/main/java/io/axual/ksml/generator/TopologyAnalyzer.java index 87b882f9..cda3a760 100644 --- a/ksml/src/main/java/io/axual/ksml/generator/TopologyAnalyzer.java +++ b/ksml/src/main/java/io/axual/ksml/generator/TopologyAnalyzer.java @@ -24,46 +24,49 @@ import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.TopologyDescription; -import java.util.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; public class TopologyAnalyzer { public record TopologyAnalysis(Set inputTopics, Set intermediateTopics, - Set outputTopics, Map stores) { + Set outputTopics, Set internalTopics) { } - public static TopologyAnalysis analyze(Topology topology, String applicationId, Collection knownTopics) { + public static TopologyAnalysis analyze(Topology topology, String applicationId) { // Derive all input, intermediate and output topics - final var topologyInputs = new HashSet(); - final var topologyOutputs = new HashSet(); - final var inputTopics = new HashSet(); - final var outputTopics = new HashSet(); - final var intermediateTopics = new HashSet(); + final var inputTopics = new TreeSet(); + final var intermediateTopics = new TreeSet(); + final var outputTopics = new TreeSet(); + final var internalTopics = new TreeSet(); - analyzeTopology(topology, topologyInputs, topologyOutputs); + analyzeTopology(topology, inputTopics, outputTopics); - for (String topic : topologyInputs) { - if (knownTopics.contains(topic)) { - inputTopics.add(topic); - } else { - intermediateTopics.add(applicationId + "-" + topic); + // Intermediate topics are both input and output topics, so sort them into a separate set + for (final var topic : inputTopics) { + if (outputTopics.contains(topic)) { + intermediateTopics.add(topic); + outputTopics.remove(topic); } } + inputTopics.removeAll(intermediateTopics); - for (String topic : topologyOutputs) { - if (knownTopics.contains(topic)) { - outputTopics.add(topic); - } else { - intermediateTopics.add(applicationId + "-" + topic); + // Internal topics end in "changelog" or "repartition", so sort them into a separate set + for (final var topic : intermediateTopics) { + if (topic.endsWith("-changelog") || topic.endsWith("-repartition")) { + internalTopics.add(topic); } } + intermediateTopics.removeAll(internalTopics); // Return the built topology and its context return new TopologyAnalysis( inputTopics, intermediateTopics, outputTopics, - new HashMap<>()); -// context.getStoreDefinitions()); + internalTopics.stream().map(topic -> applicationId + "-" + topic).collect(Collectors.toSet())); } private static void analyzeTopology(Topology topology, Set inputTopics, Set outputTopics) { @@ -77,7 +80,7 @@ private static void analyzeTopology(Topology topology, Set inputTopics, inputTopics.add(sourceNode.topicPattern().pattern()); } if (node instanceof TopologyDescription.Processor processorNode) { -// stores.addAll(processorNode.stores()); + // Ignore store names here } if (node instanceof TopologyDescription.Sink sinkNode && sinkNode.topic() != null) { outputTopics.add(sinkNode.topic()); From d263a43ca579e4a6c149b037813094ed8273c78f Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:32:00 +0100 Subject: [PATCH 31/55] Improve toString() function for DataObjects --- .../data/notation/csv/CsvSchemaMapper.java | 10 ++++- .../notation/xml/XmlDataObjectMapper.java | 2 +- .../data/notation/xml/XmlSchemaMapper.java | 11 ++++- .../io/axual/ksml/data/object/DataBytes.java | 16 +++---- .../io/axual/ksml/data/object/DataEnum.java | 5 --- .../io/axual/ksml/data/object/DataList.java | 11 ++++- .../io/axual/ksml/data/object/DataObject.java | 23 ++++++++++ .../axual/ksml/data/object/DataPrimitive.java | 10 ++++- .../io/axual/ksml/data/object/DataStruct.java | 42 ++++++++++--------- .../io/axual/ksml/data/object/DataTuple.java | 8 +++- .../io/axual/ksml/data/object/DataUnion.java | 4 +- .../io/axual/ksml/data/util/ValuePrinter.java | 30 +++++++++++++ .../runner/backend/KafkaStreamsRunner.java | 2 +- .../java/io/axual/ksml/user/UserFunction.java | 5 +-- 14 files changed, 133 insertions(+), 46 deletions(-) create mode 100644 ksml-data/src/main/java/io/axual/ksml/data/util/ValuePrinter.java diff --git a/ksml-data-csv/src/main/java/io/axual/ksml/data/notation/csv/CsvSchemaMapper.java b/ksml-data-csv/src/main/java/io/axual/ksml/data/notation/csv/CsvSchemaMapper.java index 6368072b..5b1fd669 100644 --- a/ksml-data-csv/src/main/java/io/axual/ksml/data/notation/csv/CsvSchemaMapper.java +++ b/ksml-data-csv/src/main/java/io/axual/ksml/data/notation/csv/CsvSchemaMapper.java @@ -22,6 +22,7 @@ import io.axual.ksml.data.mapper.DataSchemaMapper; import io.axual.ksml.data.object.DataList; +import io.axual.ksml.data.object.DataObject; import io.axual.ksml.data.schema.DataField; import io.axual.ksml.data.schema.DataSchema; import io.axual.ksml.data.schema.DataValue; @@ -48,7 +49,14 @@ public DataSchema toDataSchema(String namespace, String name, String value) { private DataSchema toDataSchema(String namespace, String name, DataList fieldNames) { List fields = new ArrayList<>(); for (var fieldName : fieldNames) { - fields.add(new DataField(fieldName.toString(), DataSchema.create(DataSchema.Type.STRING), fieldName.toString(), NO_INDEX, true, false, new DataValue(""))); + fields.add(new DataField( + fieldName.toString(DataObject.Printer.INTERNAL), + DataSchema.create(DataSchema.Type.STRING), + fieldName.toString(DataObject.Printer.INTERNAL), + NO_INDEX, + true, + false, + new DataValue(""))); } return new StructSchema(namespace, name, "CSV schema", fields); } diff --git a/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlDataObjectMapper.java b/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlDataObjectMapper.java index c31add53..c7f5bcf7 100644 --- a/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlDataObjectMapper.java +++ b/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlDataObjectMapper.java @@ -230,7 +230,7 @@ private void elementFromDataObject(ElementCreator elementCreator, Element elemen return; } if (value != null) { - element.setTextContent(value.toString()); + element.setTextContent(value.toString(DataObject.Printer.INTERNAL)); } } } diff --git a/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlSchemaMapper.java b/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlSchemaMapper.java index 684744bb..2191cf12 100644 --- a/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlSchemaMapper.java +++ b/ksml-data-xml/src/main/java/io/axual/ksml/data/notation/xml/XmlSchemaMapper.java @@ -170,7 +170,7 @@ private DataField parseField(DataStruct fieldStruct) { var fieldType = fieldStruct.get(ATTRIBUTES_ELEMENT_NAME) instanceof DataStruct attributeStruct ? attributeStruct.get(TYPE_NAME) : null; if (fieldType instanceof DataString fieldTypeString) { var type = fieldTypeString.value().contains(":") ? fieldTypeString.value().substring(fieldTypeString.value().indexOf(":") + 1) : fieldTypeString.value(); - return simpleField(fieldName.toString(), type); + return simpleField(fieldName.toString(DataObject.Printer.INTERNAL), type); } else { // Field type is not specified, so dig down into the elements below to find out the type var complexTypeElement = fieldStruct.get(COMPLEX_TYPE_NAME); @@ -178,7 +178,14 @@ private DataField parseField(DataStruct fieldStruct) { var sequenceElement = complexTypeStruct.get(SEQUENCE_NAME); if (sequenceElement instanceof DataStruct sequenceStruct) { var fields = parseFields(sequenceStruct); - return new DataField(fieldName.toString(), new StructSchema(null, fieldName.toString(), "Converted from XSD", fields), null); + return new DataField( + fieldName.toString(DataObject.Printer.INTERNAL), + new StructSchema( + null, + fieldName.toString(DataObject.Printer.INTERNAL), + "Converted from XSD", + fields), + null); } } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/object/DataBytes.java b/ksml-data/src/main/java/io/axual/ksml/data/object/DataBytes.java index 0cba562a..af78b06c 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/object/DataBytes.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/object/DataBytes.java @@ -37,15 +37,15 @@ public DataBytes(byte[] value) { } @Override - public String toString() { - StringBuilder builder = new StringBuilder(type().toString()).append(": "); - if (value() == null) return builder.append("null").toString(); - builder.append("["); + public String toString(Printer printer) { + final var sb = new StringBuilder(printer.schemaString(this)); + if (value() == null) return sb.append("null").toString(); + sb.append("["); for (int index = 0; index < value().length; index++) { - if (index > 0) builder.append(", "); - builder.append(String.format("%02X ", value()[index])); + if (index > 0) sb.append(", "); + sb.append(String.format("%02X ", value()[index])); } - builder.append("]"); - return builder.toString(); + sb.append("]"); + return sb.toString(); } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/object/DataEnum.java b/ksml-data/src/main/java/io/axual/ksml/data/object/DataEnum.java index 1753949b..34a6ebf0 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/object/DataEnum.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/object/DataEnum.java @@ -38,9 +38,4 @@ private boolean validateValue(String value) { } return false; } - - @Override - public String toString() { - return type().toString() + ": " + super.toString(); - } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/object/DataList.java b/ksml-data/src/main/java/io/axual/ksml/data/object/DataList.java index feae82e6..28b678bf 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/object/DataList.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/object/DataList.java @@ -20,6 +20,7 @@ * =========================LICENSE_END================================== */ +import io.axual.ksml.data.exception.DataException; import io.axual.ksml.data.exception.ExecutionException; import io.axual.ksml.data.type.DataType; import io.axual.ksml.data.type.ListType; @@ -114,10 +115,16 @@ public int hashCode() { @Override public String toString() { - StringBuilder sb = new StringBuilder(type.toString()).append(": ["); + return toString(Printer.INTERNAL); + } + + @Override + public String toString(Printer printer) { + final var sb = new StringBuilder(printer.schemaString(this)); + sb.append("["); for (int index = 0; index < size(); index++) { if (index > 0) sb.append(", "); - sb.append(get(index).toString()); + sb.append(get(index).toString(printer.childObjectPrinter())); } sb.append("]"); return sb.toString(); diff --git a/ksml-data/src/main/java/io/axual/ksml/data/object/DataObject.java b/ksml-data/src/main/java/io/axual/ksml/data/object/DataObject.java index 5aed19dc..6fa8dbb8 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/object/DataObject.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/object/DataObject.java @@ -24,4 +24,27 @@ public interface DataObject { DataType type(); + + enum Printer { + INTERNAL, + EXTERNAL_NO_SCHEMA, + EXTERNAL_TOP_SCHEMA, + EXTERNAL_ALL_SCHEMA; + + public Printer childObjectPrinter() { + if (this == INTERNAL) return INTERNAL; + return this == EXTERNAL_ALL_SCHEMA ? EXTERNAL_ALL_SCHEMA : EXTERNAL_NO_SCHEMA; + } + + public String forceSchemaString(DataObject value) { + return value.type().schemaName() + ": "; + } + + public String schemaString(DataObject value) { + if (this == INTERNAL || this == EXTERNAL_NO_SCHEMA) return ""; + return forceSchemaString(value); + } + } + + String toString(Printer printer); } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/object/DataPrimitive.java b/ksml-data/src/main/java/io/axual/ksml/data/object/DataPrimitive.java index 59c561bf..4c2faa5c 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/object/DataPrimitive.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/object/DataPrimitive.java @@ -22,6 +22,7 @@ import io.axual.ksml.data.exception.ExecutionException; import io.axual.ksml.data.type.DataType; +import io.axual.ksml.data.util.ValuePrinter; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -46,6 +47,13 @@ private void checkValue() { @Override public String toString() { - return value != null ? value.toString() : type + ": null"; + return toString(Printer.INTERNAL); + } + + @Override + public String toString(Printer printer) { + return value != null + ? ValuePrinter.print(value, printer != Printer.INTERNAL) + : printer.forceSchemaString(this) + ValuePrinter.print(value, printer != Printer.INTERNAL); } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/object/DataStruct.java b/ksml-data/src/main/java/io/axual/ksml/data/object/DataStruct.java index 1d742fb5..936ff008 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/object/DataStruct.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/object/DataStruct.java @@ -23,6 +23,7 @@ import io.axual.ksml.data.exception.ExecutionException; import io.axual.ksml.data.schema.StructSchema; import io.axual.ksml.data.type.StructType; +import io.axual.ksml.data.util.ValuePrinter; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -50,7 +51,6 @@ public class DataStruct implements DataObject { // If both (do not) start with the meta char, then sort as normal return o1.compareTo(o2); }; - private static final String QUOTE = "\""; public interface DataStructApplier { void apply(T value) throws Exception; @@ -93,7 +93,7 @@ public DataObject get(String key) { } public void getIfPresent(String key, Class clazz, DataStructApplier applier) { - var value = get(key); + final var value = get(key); if (value != null && clazz.isAssignableFrom(value.getClass())) { try { applier.apply((T) value); @@ -108,7 +108,7 @@ public T getAs(String key, Class clazz) { } public T getAs(String key, Class clazz, T defaultValue) { - var value = get(key); + final var value = get(key); if (value != null && clazz.isAssignableFrom(value.getClass())) { return (T) value; } @@ -116,7 +116,7 @@ public T getAs(String key, Class clazz, T defaultValue) { } public DataString getAsString(String key) { - var result = get(key); + final var result = get(key); return result instanceof DataString str ? str : result != null ? new DataString(result.toString()) : null; } @@ -136,29 +136,33 @@ public int size() { @Override public String toString() { - var schemaName = type.schemaName() + ": "; + return toString(Printer.INTERNAL); + } + + @Override + public String toString(Printer printer) { + final var schemaName = printer.schemaString(this); // Return NULL as value if (isNull()) return schemaName + "null"; - Iterator> i = entrySet().iterator(); + final var iterator = entrySet().iterator(); // Return empty struct as value - if (!i.hasNext()) return schemaName + "{}"; + if (!iterator.hasNext()) return schemaName + "{}"; - StringBuilder sb = new StringBuilder(); - sb.append(schemaName).append('{'); + // Iterate through all entries + final var sb = new StringBuilder(schemaName).append("{"); while (true) { - Map.Entry e = i.next(); - String key = e.getKey(); - DataObject value = e.getValue(); - var quote = (value instanceof DataString ? QUOTE : ""); - sb.append(QUOTE).append(key).append(QUOTE); - sb.append(':'); - sb.append(quote).append(value == this ? "(this Map)" : value).append(quote); - if (!i.hasNext()) - return sb.append('}').toString(); - sb.append(',').append(' '); + final var entry = iterator.next(); + final var key = entry.getKey(); + final var value = entry.getValue(); + sb.append(ValuePrinter.print(key, true)); + sb.append(": "); + sb.append(value.toString(printer.childObjectPrinter())); + if (!iterator.hasNext()) + return sb.append("}").toString(); + sb.append(", "); } } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/object/DataTuple.java b/ksml-data/src/main/java/io/axual/ksml/data/object/DataTuple.java index 972b649c..068fd90e 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/object/DataTuple.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/object/DataTuple.java @@ -20,6 +20,7 @@ * =========================LICENSE_END================================== */ +import io.axual.ksml.data.exception.DataException; import io.axual.ksml.data.type.DataType; import io.axual.ksml.data.type.TupleType; import io.axual.ksml.data.value.Tuple; @@ -46,6 +47,11 @@ private static DataType[] convertElements(DataObject... elements) { @Override public String toString() { - return type.toString() + ": " + super.toString(); + return toString(Printer.INTERNAL); + } + + @Override + public String toString(Printer printer) { + return printer.schemaString(this) + super.toString(); } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/object/DataUnion.java b/ksml-data/src/main/java/io/axual/ksml/data/object/DataUnion.java index adb58c48..75cc196a 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/object/DataUnion.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/object/DataUnion.java @@ -28,7 +28,7 @@ public DataUnion(UnionType type, DataObject value) { } @Override - public String toString() { - return type().toString() + ": " + super.toString(); + public String toString(Printer printer) { + return printer.schemaString(this) + (value() != null ? value().toString(printer.childObjectPrinter()) : "null"); } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/util/ValuePrinter.java b/ksml-data/src/main/java/io/axual/ksml/data/util/ValuePrinter.java new file mode 100644 index 00000000..3f9629c6 --- /dev/null +++ b/ksml-data/src/main/java/io/axual/ksml/data/util/ValuePrinter.java @@ -0,0 +1,30 @@ +package io.axual.ksml.data.util; + +/*- + * ========================LICENSE_START================================= + * KSML Data Library + * %% + * Copyright (C) 2021 - 2024 Axual B.V. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * =========================LICENSE_END================================== + */ + +public class ValuePrinter { + private static final String QUOTE = "\""; + + public static String print(Object value, boolean quoted) { + var quote = quoted && value instanceof String ? QUOTE : ""; + return value != null ? quote + value + quote : "null"; + } +} diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaStreamsRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaStreamsRunner.java index feeac948..fbd58535 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaStreamsRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaStreamsRunner.java @@ -21,6 +21,7 @@ */ +import io.axual.ksml.runner.streams.KSMLClientSupplier; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.StreamsBuilder; @@ -40,7 +41,6 @@ import io.axual.ksml.generator.TopologyDefinition; import io.axual.ksml.runner.config.ApplicationServerConfig; import io.axual.ksml.runner.exception.RunnerException; -import io.axual.ksml.runner.streams.KSMLClientSupplier; import lombok.Builder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; diff --git a/ksml/src/main/java/io/axual/ksml/user/UserFunction.java b/ksml/src/main/java/io/axual/ksml/user/UserFunction.java index cf232cb9..b321800b 100644 --- a/ksml/src/main/java/io/axual/ksml/user/UserFunction.java +++ b/ksml/src/main/java/io/axual/ksml/user/UserFunction.java @@ -107,15 +107,14 @@ protected void checkType(ParameterDefinition definition, DataObject value) { checkType(definition.type(), value); } - protected void logCall(Object[] parameters, Object result) { + protected void logCall(DataObject[] parameters, Object result) { // Log the called function in debug if (LOG.isDebugEnabled()) { // Check all parameters and copy them into the interpreter as prefixed globals StringBuilder paramsAndValues = new StringBuilder(); for (int index = 0; index < parameters.length; index++) { paramsAndValues.append(!paramsAndValues.isEmpty() ? ", " : ""); - String valueQuote = parameters[index] instanceof String ? "'" : ""; - paramsAndValues.append(this.parameters[index].name()).append("=").append(valueQuote).append(parameters[index] != null ? parameters[index] : "null").append(valueQuote); + paramsAndValues.append(this.parameters[index].name()).append("=").append(parameters[index] != null ? parameters[index].toString(DataObject.Printer.EXTERNAL_ALL_SCHEMA) : "null"); } if (result != null) { LOG.debug("User function {}.{}({}) returned {}", namespace, name, paramsAndValues, result); From 9ba2eda75dba50e62e68b6875869d00c22fba595 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:37:07 +0100 Subject: [PATCH 32/55] Unify lombok configurations --- .../java/io/axual/ksml/runner/KSMLRunner.java | 14 +++++++------- .../io/axual/ksml/runner/config/KSMLConfig.java | 16 ++++++++-------- .../runner/config/KSMLRunnerKSMLConfigTest.java | 6 +++--- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java index e0ee95af..f8621644 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java @@ -85,8 +85,8 @@ public static void main(String[] args) { final var config = readConfiguration(configFile); final var ksmlConfig = config.ksmlConfig(); - log.info("Using directories: config: {}, schema: {}, storage: {}", ksmlConfig.getConfigDirectory(), ksmlConfig.getSchemaDirectory(), ksmlConfig.getStorageDirectory()); - final var definitions = ksmlConfig.getDefinitions(); + log.info("Using directories: config: {}, schema: {}, storage: {}", ksmlConfig.configDirectory(), ksmlConfig.schemaDirectory(), ksmlConfig.storageDirectory()); + final var definitions = ksmlConfig.definitions(); if (definitions == null || definitions.isEmpty()) { throw new ConfigException("definitions", definitions, "No KSML definitions found in configuration"); } @@ -94,7 +94,7 @@ public static void main(String[] args) { KsmlInfo.registerKsmlAppInfo(config.applicationId()); // Start the appserver if needed - final var appServer = ksmlConfig.getApplicationServerConfig(); + final var appServer = ksmlConfig.applicationServerConfig(); RestServer restServer = null; // Start rest server to provide service endpoints if (appServer.enabled()) { @@ -104,7 +104,7 @@ public static void main(String[] args) { } // Set up all default notations and register them in the NotationLibrary - final var notationFactories = new NotationFactories(config.kafkaConfig(), ksmlConfig.getSchemaDirectory()); + final var notationFactories = new NotationFactories(config.kafkaConfig(), ksmlConfig.schemaDirectory()); for (final var notation : notationFactories.notations().entrySet()) { NotationLibrary.register(notation.getKey(), notation.getValue().create(null)); } @@ -134,7 +134,7 @@ public static void main(String[] args) { log.warn("No implementation specified for AVRO notation. If you use AVRO in your KSML definition, add the required configuration to the ksml-runner.yaml"); } - final var errorHandling = ksmlConfig.getErrorHandlingConfig(); + final var errorHandling = ksmlConfig.errorHandlingConfig(); if (errorHandling != null) { ExecutionContext.INSTANCE.setConsumeHandler(getErrorHandler(errorHandling.consumerErrorHandlingConfig())); ExecutionContext.INSTANCE.setProduceHandler(getErrorHandler(errorHandling.producerErrorHandlingConfig())); @@ -169,8 +169,8 @@ public static void main(String[] args) { .kafkaConfig(config.kafkaConfig()) .build()); final var streams = pipelineSpecs.isEmpty() ? null : new KafkaStreamsRunner(KafkaStreamsRunner.Config.builder() - .storageDirectory(ksmlConfig.getStorageDirectory()) - .appServer(ksmlConfig.getApplicationServerConfig()) + .storageDirectory(ksmlConfig.storageDirectory()) + .appServer(ksmlConfig.applicationServerConfig()) .definitions(pipelineSpecs) .kafkaConfig(config.kafkaConfig()) .build()); diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java index cdafbaac..f8792f52 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java @@ -85,23 +85,23 @@ private String getDirectory(String configVariable, String directory) { return configPath.toAbsolutePath().normalize().toString(); } - public String getConfigDirectory() { + public String configDirectory() { return getDirectory("configDirectory", configDirectory != null ? configDirectory : System.getProperty("user.dir")); } - public String getSchemaDirectory() { - return getDirectory("schemaDirectory", schemaDirectory != null ? schemaDirectory : getConfigDirectory()); + public String schemaDirectory() { + return getDirectory("schemaDirectory", schemaDirectory != null ? schemaDirectory : configDirectory()); } - public String getStorageDirectory() { + public String storageDirectory() { return getDirectory("storageDirectory", storageDirectory != null ? storageDirectory : System.getProperty("java.io.tmpdir")); } - public ApplicationServerConfig getApplicationServerConfig() { + public ApplicationServerConfig applicationServerConfig() { return applicationServer; } - public ErrorHandlingConfig getErrorHandlingConfig() { + public ErrorHandlingConfig errorHandlingConfig() { if (errorHandling == null) return new ErrorHandlingConfig(); return errorHandling; } @@ -111,12 +111,12 @@ public Map notations() { return ImmutableMap.of(); } - public Map getDefinitions() { + public Map definitions() { final var result = new HashMap(); if (definitions != null) { for (Map.Entry definition : definitions.entrySet()) { if (definition.getValue() instanceof String definitionFile) { - final var definitionFilePath = Paths.get(getConfigDirectory(), definitionFile); + final var definitionFilePath = Paths.get(configDirectory(), definitionFile); if (Files.notExists(definitionFilePath) || !Files.isRegularFile(definitionFilePath)) { throw new ConfigException("definitionFile", definitionFilePath, "The provided KSML definition file does not exists or is not a regular file"); } diff --git a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerKSMLConfigTest.java b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerKSMLConfigTest.java index 94ece259..267c979e 100644 --- a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerKSMLConfigTest.java +++ b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerKSMLConfigTest.java @@ -44,7 +44,7 @@ void setup() { void shouldValidateConfig() throws Exception { final var yaml = getClass().getClassLoader().getResourceAsStream("ksml-config.yaml"); final var ksmlConfig = objectMapper.readValue(yaml, KSMLConfig.class); - ksmlConfig.getConfigDirectory(); + ksmlConfig.configDirectory(); } @Test @@ -52,7 +52,7 @@ void shouldValidateConfig() throws Exception { void shouldThrowOnWrongConfigdir() throws Exception { final var yaml = getClass().getClassLoader().getResourceAsStream("ksml-config-wrong-configdir.yaml"); final var ksmlConfig = objectMapper.readValue(yaml, KSMLConfig.class); - assertThrows(ConfigException.class, ksmlConfig::getConfigDirectory, "should throw exception for wrong configdir"); + assertThrows(ConfigException.class, ksmlConfig::configDirectory, "should throw exception for wrong configdir"); } @Test @@ -60,6 +60,6 @@ void shouldThrowOnWrongConfigdir() throws Exception { void shouldDefaultConfigToWorkdir() throws Exception { final var yaml = getClass().getClassLoader().getResourceAsStream("ksml-config-no-configdir.yaml"); final var ksmlConfig = objectMapper.readValue(yaml, KSMLConfig.class); - assertEquals(System.getProperty("user.dir"), ksmlConfig.getConfigDirectory(), "config dir should default to working dir"); + assertEquals(System.getProperty("user.dir"), ksmlConfig.configDirectory(), "config dir should default to working dir"); } } From bbb39a45be1f6e2c90ba97334bb0e8d06a4aa6d7 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:40:35 +0100 Subject: [PATCH 33/55] Implement Batch Producer --- docs/ksml-language-spec.json | 5121 ++++++++++++++++- .../ksml/data/notation/NotationLibrary.java | 2 +- .../json/JsonDataObjectConverter.java | 2 +- .../notation/json/JsonDataObjectMapper.java | 10 +- .../data/notation/json/JsonSchemaLoader.java | 2 +- .../data/notation/json/JsonSchemaMapper.java | 10 +- .../data/notation/json/JsonStringMapper.java | 12 +- .../axual/ksml/data/schema/SchemaLibrary.java | 2 +- .../io/axual/ksml/data/serde/JsonSerde.java | 2 +- .../java/io/axual/ksml/runner/KSMLRunner.java | 2 +- .../ksml/runner/backend/AlwaysReschedule.java | 56 - .../runner/backend/CountingReschedule.java | 41 - .../runner/backend/KafkaProducerRunner.java | 22 +- .../runner/backend/RescheduleStrategy.java | 63 - .../runner/backend/SingleShotReschedule.java | 34 - .../ksml/runner/backend/UntilReschedule.java | 43 - .../runner/config/ErrorHandlingConfig.java | 4 + .../axual/ksml/runner/config/KSMLConfig.java | 4 +- .../ExecutableProducer.java | 211 +- .../IntervalSchedule.java | 2 +- .../runner/producer/ProducerStrategy.java | 126 + .../runner/backend/IntervalScheduleTest.java | 2 + .../backend/KafkaProducerRunnerTest.java | 2 +- .../ksml/data/mapper/DataObjectConverter.java | 4 +- .../ksml/definition/ProducerDefinition.java | 10 +- .../parser/ProducerDefinitionParser.java | 9 +- .../main/java/io/axual/ksml/dsl/KSMLDSL.java | 1 + .../axual/ksml/generator/StreamDataType.java | 2 +- .../axual/ksml/parser/DefinitionParser.java | 8 + .../io/axual/ksml/parser/UserTypeParser.java | 8 +- .../ksml/testutil/KSMLTestExtension.java | 4 +- 31 files changed, 5443 insertions(+), 378 deletions(-) delete mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/backend/AlwaysReschedule.java delete mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/backend/CountingReschedule.java delete mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/backend/RescheduleStrategy.java delete mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/backend/SingleShotReschedule.java delete mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/backend/UntilReschedule.java rename ksml-runner/src/main/java/io/axual/ksml/runner/{backend => producer}/ExecutableProducer.java (52%) rename ksml-runner/src/main/java/io/axual/ksml/runner/{backend => producer}/IntervalSchedule.java (98%) create mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/producer/ProducerStrategy.java diff --git a/docs/ksml-language-spec.json b/docs/ksml-language-spec.json index 4764401c..91193b76 100644 --- a/docs/ksml-language-spec.json +++ b/docs/ksml-language-spec.json @@ -1 +1,5120 @@ -{"additionalProperties":false,"definitions":{"AggregateOperation":{"additionalProperties":false,"description":"An aggregate operation","properties":{"adder":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/AggregatorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* (GroupedTable) A function that adds a record to the aggregation result"},"aggregator":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/AggregatorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* (GroupedStream, SessionWindowedStream, TimeWindowedStream) The aggregator function, which combines a value with the previous aggregation result and outputs a new aggregation result"},"initializer":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/InitializerDefinitionWithImplicitType","type":"object"}],"description":"The initializer function, which generates an initial value for every set of aggregated records"},"merger":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/MergerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* (SessionWindowedStream, SessionWindowedCogroupedStream) A function that combines two aggregation results"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/WindowStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the result aggregation"},"subtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/AggregatorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* (GroupedTable) A function that removes a record from the aggregation result"},"type":{"description":"The type of the operation","enum":["aggregate"]}},"required":["initializer","type"],"title":"AggregateOperation","type":"object"},"AggregatorDefinition":{"additionalProperties":false,"description":"Defines a aggregator function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the aggregator"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the aggregator. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the aggregator. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the aggregator. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the aggregator","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the aggregator. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["aggregator"]}},"required":["type"],"title":"AggregatorDefinition","type":"object"},"AggregatorDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a aggregator function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the aggregator"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the aggregator. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the aggregator. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the aggregator. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the aggregator","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the aggregator. Only required for function types, which are not pre-defined.","type":"string"}},"title":"AggregatorDefinitionWithImplicitType","type":"object"},"BranchDefinitionWithPipeline":{"additionalProperties":false,"description":"Defines a branch with sub-pipeline in a BranchOperation","properties":{"as":{"description":"*(optional)* The name to register the pipeline result under, which can be used as source by follow-up pipelines","type":"string"},"branch":{"description":"*(optional)* Defines a single branch, consisting of a condition and a pipeline to execute for messages that fulfil the predicate","items":{"$ref":"#/definitions/StringOrInlinePredicateDefinitionWithImplicitType","type":"object"},"type":"array"},"forEach":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ForEachActionDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that gets called for every message in the stream"},"if":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/PredicateDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Defines the condition under which messages get sent down this branch"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"print":{"$ref":"#/definitions/PrintOperation","description":"*(optional)* The specification of where to print messages to","type":"object"},"to":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ToTopicDefinition","type":"object"}],"description":"*(optional)* Ends the pipeline by sending all messages to a stream, table or globalTable, or to an inline defined output topic and optional partitioner"},"toTopicNameExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ToTopicNameExtractorDefinition","type":"object"}],"description":"*(optional)* Ends the pipeline by sending all messages to a topic provided by a pre-defined topic name extractor function, or to a topic provided by an inline defined topic name extractor and optional partitioner"},"via":{"description":"*(optional)* A series of operations performed on the input stream","items":{"anyOf":[{"$ref":"#/definitions/AggregateOperation","type":"object"},{"$ref":"#/definitions/CogroupOperation","type":"object"},{"$ref":"#/definitions/ConvertKeyOperation","type":"object"},{"$ref":"#/definitions/ConvertKeyValueOperation","type":"object"},{"$ref":"#/definitions/ConvertValueOperation","type":"object"},{"$ref":"#/definitions/CountOperation","type":"object"},{"$ref":"#/definitions/FilterNotOperation","type":"object"},{"$ref":"#/definitions/FilterOperation","type":"object"},{"$ref":"#/definitions/GroupByKeyOperation","type":"object"},{"$ref":"#/definitions/GroupByOperation","type":"object"},{"$ref":"#/definitions/JoinOperationWithGlobalTable","type":"object"},{"$ref":"#/definitions/JoinOperationWithStream","type":"object"},{"$ref":"#/definitions/JoinOperationWithTable","type":"object"},{"$ref":"#/definitions/LeftJoinOperationWithGlobalTable","type":"object"},{"$ref":"#/definitions/LeftJoinOperationWithStream","type":"object"},{"$ref":"#/definitions/LeftJoinOperationWithTable","type":"object"},{"$ref":"#/definitions/MergeOperation","type":"object"},{"$ref":"#/definitions/OuterJoinOperationWithStream","type":"object"},{"$ref":"#/definitions/OuterJoinOperationWithTable","type":"object"},{"$ref":"#/definitions/PeekOperation","type":"object"},{"$ref":"#/definitions/ReduceOperationWithAdderAndSubtractor","type":"object"},{"$ref":"#/definitions/ReduceOperationWithReducer","type":"object"},{"$ref":"#/definitions/RepartitionOperation","type":"object"},{"$ref":"#/definitions/SuppressOperationUntilTimeLimit","type":"object"},{"$ref":"#/definitions/SuppressOperationUntilWindowCloses","type":"object"},{"$ref":"#/definitions/ToStreamOperation","type":"object"},{"$ref":"#/definitions/ToTableOperation","type":"object"},{"$ref":"#/definitions/TransformKeyOperation","type":"object"},{"$ref":"#/definitions/TransformKeyValueOperation","type":"object"},{"$ref":"#/definitions/TransformKeyValueToKeyValueListOperation","type":"object"},{"$ref":"#/definitions/TransformKeyValueToValueListOperation","type":"object"},{"$ref":"#/definitions/TransformMetadataOperation","type":"object"},{"$ref":"#/definitions/TransformValueOperation","type":"object"},{"$ref":"#/definitions/WindowBySessionOperation","type":"object"},{"$ref":"#/definitions/WindowByTimeOperationWithHoppingWindow","type":"object"},{"$ref":"#/definitions/WindowByTimeOperationWithSlidingWindow","type":"object"},{"$ref":"#/definitions/WindowByTimeOperationWithTumblingWindow","type":"object"}]},"type":"array"}},"title":"BranchDefinitionWithPipeline","type":"object"},"CogroupOperation":{"additionalProperties":false,"description":"A cogroup operation","properties":{"aggregator":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/AggregatorDefinitionWithImplicitType","type":"object"}],"description":"(GroupedStream, SessionWindowedStream, TimeWindowedStream) The aggregator function, which combines a value with the previous aggregation result and outputs a new aggregation result"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/WindowStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the co-grouped stream"},"type":{"description":"The type of the operation","enum":["cogroup"]}},"required":["aggregator","type"],"title":"CogroupOperation","type":"object"},"ConvertKeyOperation":{"additionalProperties":false,"description":"An operation to convert the stream key type to another type. Conversion is only syntactic, eg. from Avro to XML.","properties":{"into":{"description":"The type to convert the stream key into","type":"string"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["convertKey"]}},"required":["into","type"],"title":"ConvertKeyOperation","type":"object"},"ConvertKeyValueOperation":{"additionalProperties":false,"description":"An operation to convert the stream key and value types to other types. Conversion is only syntactic, eg. from Avro to XML.","properties":{"into":{"description":"The tuple type to convert the stream key/value into","type":"string"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["convertKeyValue"]}},"required":["into","type"],"title":"ConvertKeyValueOperation","type":"object"},"ConvertValueOperation":{"additionalProperties":false,"description":"An operation to convert the stream value type to another type. Conversion is only syntactic, eg. from Avro to XML.","properties":{"into":{"description":"The type to convert the stream value into","type":"string"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["convertValue"]}},"required":["into","type"],"title":"ConvertValueOperation","type":"object"},"CountOperation":{"additionalProperties":false,"description":"Count the number of times a key is seen in a given window","properties":{"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinition","type":"object"},{"$ref":"#/definitions/SessionStateStoreDefinition","type":"object"},{"$ref":"#/definitions/WindowStateStoreDefinition","type":"object"}],"description":"*(optional)* Materialized view of the count operation's result"},"type":{"description":"The type of the operation","enum":["count"]}},"required":["type"],"title":"CountOperation","type":"object"},"FilterNotOperation":{"additionalProperties":false,"description":"Filter records based on the inverse result of a predicate function","properties":{"if":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/PredicateDefinitionWithImplicitType","type":"object"}],"description":"A function that returns \"false\" when records are accepted, \"true\" otherwise"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the filtered table (only applies to tables, ignored for streams)"},"type":{"description":"The type of the operation","enum":["filterNot"]}},"required":["if","type"],"title":"FilterNotOperation","type":"object"},"FilterOperation":{"additionalProperties":false,"description":"Filter records based on a predicate function","properties":{"if":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/PredicateDefinitionWithImplicitType","type":"object"}],"description":"A function that returns \"true\" when records are accepted, \"false\" otherwise"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the filtered table (only applies to tables, ignored for streams)"},"type":{"description":"The type of the operation","enum":["filter"]}},"required":["if","type"],"title":"FilterOperation","type":"object"},"ForEachActionDefinition":{"additionalProperties":false,"description":"Defines a foreach action function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the foreach action"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the foreach action. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the foreach action. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the foreach action. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the foreach action","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the foreach action. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the foreach action uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"},"type":{"description":"The type of the function","enum":["forEach"]}},"required":["type"],"title":"ForEachActionDefinition","type":"object"},"ForEachActionDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a foreach action function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the foreach action"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the foreach action. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the foreach action. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the foreach action. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the foreach action","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the foreach action. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the foreach action uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"}},"title":"ForEachActionDefinitionWithImplicitType","type":"object"},"ForeignKeyExtractorDefinition":{"additionalProperties":false,"description":"Defines a foreign key extractor function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the foreign key extractor"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the foreign key extractor. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the foreign key extractor. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the foreign key extractor. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the foreign key extractor","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the foreign key extractor. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["foreignKeyExtractor"]}},"required":["type"],"title":"ForeignKeyExtractorDefinition","type":"object"},"ForeignKeyExtractorDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a foreign key extractor function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the foreign key extractor"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the foreign key extractor. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the foreign key extractor. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the foreign key extractor. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the foreign key extractor","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the foreign key extractor. Only required for function types, which are not pre-defined.","type":"string"}},"title":"ForeignKeyExtractorDefinitionWithImplicitType","type":"object"},"GeneratorDefinition":{"additionalProperties":false,"description":"Defines a message generator function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the message generator"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the message generator. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the message generator. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the message generator. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the message generator","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the message generator. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["generator"]}},"required":["type"],"title":"GeneratorDefinition","type":"object"},"GeneratorDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a message generator function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the message generator"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the message generator. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the message generator. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the message generator. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the message generator","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the message generator. Only required for function types, which are not pre-defined.","type":"string"}},"title":"GeneratorDefinitionWithImplicitType","type":"object"},"GenericFunctionDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a generic function function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the generic function"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the generic function. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the generic function. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the generic function. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the generic function","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the generic function. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["generic"]}},"title":"GenericFunctionDefinitionWithImplicitType","type":"object"},"GlobalTableDefinition":{"additionalProperties":false,"description":"Contains a definition of a GlobalTable, which can be referenced by producers and pipelines","properties":{"keyType":{"description":"*(optional)* The key type of the global table","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* KeyValue state store definition"},"topic":{"description":"The name of the Kafka topic for this global table","type":"string"},"valueType":{"description":"*(optional)* The value type of the global table","type":"string"}},"required":["topic"],"title":"GlobalTableDefinition","type":"object"},"GlobalTableDefinitionSource":{"additionalProperties":false,"description":"Contains a definition of a GlobalTable, which can be referenced by producers and pipelines","properties":{"keyType":{"description":"The key type of the global table","type":"string"},"offsetResetPolicy":{"description":"*(optional)* Policy that determines what to do when there is no initial offset in Kafka, or if the current offset does not exist any more on the server (e.g. because that data has been deleted)","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* KeyValue state store definition"},"timestampExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TimestampExtractorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function extracts the event time from a consumed record"},"topic":{"description":"The name of the Kafka topic for this global table","type":"string"},"valueType":{"description":"The value type of the global table","type":"string"}},"required":["topic","keyType","valueType"],"title":"GlobalTableDefinitionSource","type":"object"},"GroupByKeyOperation":{"additionalProperties":false,"description":"Operation to group all messages with the same key together","properties":{"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the grouped stream"},"type":{"description":"The type of the operation","enum":["groupByKey"]}},"required":["type"],"title":"GroupByKeyOperation","type":"object"},"GroupByOperation":{"additionalProperties":false,"description":"Operation to group all messages with together based on a keying function","properties":{"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueMapperDefinitionWithImplicitType","type":"object"}],"description":"Function to map records to a key they can be grouped on"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the grouped stream or table"},"type":{"description":"The type of the operation","enum":["groupBy"]}},"required":["mapper","type"],"title":"GroupByOperation","type":"object"},"InitializerDefinition":{"additionalProperties":false,"description":"Defines a initializer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the initializer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the initializer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the initializer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the initializer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the initializer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the initializer. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["initializer"]}},"required":["type"],"title":"InitializerDefinition","type":"object"},"InitializerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a initializer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the initializer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the initializer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the initializer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the initializer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the initializer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the initializer. Only required for function types, which are not pre-defined.","type":"string"}},"title":"InitializerDefinitionWithImplicitType","type":"object"},"JoinOperationWithGlobalTable":{"additionalProperties":false,"description":"Operation to join with a table","properties":{"globalTable":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/GlobalTableDefinition","type":"object"}],"description":"A reference to the globalTable, or an inline definition of the globalTable to join with"},"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueMapperDefinitionWithImplicitType","type":"object"}],"description":"A function that maps the key value from the stream to the primary key type of the globalTable"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["join"]},"valueJoiner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueJoinerDefinitionWithImplicitType","type":"object"}],"description":"A function that joins two values"}},"required":["globalTable","mapper","valueJoiner","type"],"title":"JoinOperationWithGlobalTable","type":"object"},"JoinOperationWithStream":{"additionalProperties":false,"description":"Operation to join with a stream","properties":{"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinition","type":"object"},{"$ref":"#/definitions/SessionStateStoreDefinition","type":"object"},{"$ref":"#/definitions/WindowStateStoreDefinition","type":"object"}],"description":"*(optional)* Materialized view of the joined streams"},"stream":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamDefinition","type":"object"}],"description":"A reference to the Stream, or an inline definition of the stream to join with"},"timeDifference":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The maximum time difference for a join over two streams on the same key"},"type":{"description":"The type of the operation","enum":["join"]},"valueJoiner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueJoinerDefinitionWithImplicitType","type":"object"}],"description":"A function that joins two values"}},"required":["stream","valueJoiner","timeDifference","type"],"title":"JoinOperationWithStream","type":"object"},"JoinOperationWithTable":{"additionalProperties":false,"description":"Operation to join with a table","properties":{"foreignKeyExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ForeignKeyExtractorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that can translate the join table value to a primary key"},"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"otherPartitioner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamPartitionerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that partitions the records on the join table"},"partitioner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamPartitionerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that partitions the records on the primary table"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinition","type":"object"},{"$ref":"#/definitions/SessionStateStoreDefinition","type":"object"},{"$ref":"#/definitions/WindowStateStoreDefinition","type":"object"}],"description":"*(optional)* Materialized view of the joined streams"},"table":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TableDefinition","type":"object"}],"description":"A reference to the table, or an inline definition of the table to join with"},"type":{"description":"The type of the operation","enum":["join"]},"valueJoiner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueJoinerDefinitionWithImplicitType","type":"object"}],"description":"A function that joins two values"}},"required":["table","valueJoiner","type"],"title":"JoinOperationWithTable","type":"object"},"KeyTransformerDefinition":{"additionalProperties":false,"description":"Defines a key transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the key transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the key transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the key transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the key transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the key transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the key transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the key transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"},"type":{"description":"The type of the function","enum":["keyTransformer"]}},"required":["type"],"title":"KeyTransformerDefinition","type":"object"},"KeyTransformerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a key transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the key transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the key transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the key transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the key transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the key transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the key transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the key transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"}},"title":"KeyTransformerDefinitionWithImplicitType","type":"object"},"KeyValueMapperDefinition":{"additionalProperties":false,"description":"Defines a keyvalue mapper function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue mapper"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue mapper. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue mapper. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue mapper. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue mapper","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue mapper. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["keyValueMapper"]}},"required":["type"],"title":"KeyValueMapperDefinition","type":"object"},"KeyValueMapperDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a keyvalue mapper function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue mapper"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue mapper. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue mapper. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue mapper. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue mapper","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue mapper. Only required for function types, which are not pre-defined.","type":"string"}},"title":"KeyValueMapperDefinitionWithImplicitType","type":"object"},"KeyValuePrinterDefinition":{"additionalProperties":false,"description":"Defines a keyvalue printer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue printer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue printer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue printer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue printer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue printer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue printer. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["keyValuePrinter"]}},"required":["type"],"title":"KeyValuePrinterDefinition","type":"object"},"KeyValuePrinterDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a keyvalue printer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue printer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue printer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue printer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue printer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue printer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue printer. Only required for function types, which are not pre-defined.","type":"string"}},"title":"KeyValuePrinterDefinitionWithImplicitType","type":"object"},"KeyValueStateStoreDefinition":{"additionalProperties":false,"description":"Definition of a keyValue state store","properties":{"caching":{"description":"*(optional)* \"true\" if changed to the keyValue store need to be buffered and periodically released, \"false\" to emit all changes directly","type":"boolean"},"historyRetention":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* (Versioned only) The duration for which old record versions are available for query (cannot be negative)"},"keyType":{"description":"*(optional)* The key type of the keyValue store","type":"string"},"logging":{"description":"*(optional)* \"true\" if a changelog topic should be set up on Kafka for this keyValue store, \"false\" otherwise","type":"boolean"},"name":{"description":"*(optional)* The name of the keyValue store. If this field is not defined, then the name is derived from the context.","type":"string"},"persistent":{"description":"*(optional)* \"true\" if this keyValue store needs to be stored on disk, \"false\" otherwise","type":"boolean"},"segmentInterval":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* Size of segments for storing old record versions (must be positive). Old record versions for the same key in a single segment are stored (updated and accessed) together. The only impact of this parameter is performance. If segments are large and a workload results in many record versions for the same key being collected in a single segment, performance may degrade as a result. On the other hand, historical reads (which access older segments) and out-of-order writes may slow down if there are too many segments."},"timestamped":{"description":"*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise","type":"boolean"},"type":{"description":"The type of the state store","enum":["keyValue"]},"valueType":{"description":"*(optional)* The value type of the keyValue store","type":"string"},"versioned":{"description":"*(optional)* \"true\" if elements in the store are versioned, \"false\" otherwise","type":"boolean"}},"required":["type"],"title":"KeyValueStateStoreDefinition","type":"object"},"KeyValueStateStoreDefinitionWithImplicitType":{"additionalProperties":false,"description":"Definition of a keyValue state store","properties":{"caching":{"description":"*(optional)* \"true\" if changed to the keyValue store need to be buffered and periodically released, \"false\" to emit all changes directly","type":"boolean"},"historyRetention":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* (Versioned only) The duration for which old record versions are available for query (cannot be negative)"},"keyType":{"description":"*(optional)* The key type of the keyValue store","type":"string"},"logging":{"description":"*(optional)* \"true\" if a changelog topic should be set up on Kafka for this keyValue store, \"false\" otherwise","type":"boolean"},"name":{"description":"*(optional)* The name of the keyValue store. If this field is not defined, then the name is derived from the context.","type":"string"},"persistent":{"description":"*(optional)* \"true\" if this keyValue store needs to be stored on disk, \"false\" otherwise","type":"boolean"},"segmentInterval":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* Size of segments for storing old record versions (must be positive). Old record versions for the same key in a single segment are stored (updated and accessed) together. The only impact of this parameter is performance. If segments are large and a workload results in many record versions for the same key being collected in a single segment, performance may degrade as a result. On the other hand, historical reads (which access older segments) and out-of-order writes may slow down if there are too many segments."},"timestamped":{"description":"*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise","type":"boolean"},"type":{"description":"The type of the state store","enum":["keyValue"]},"valueType":{"description":"*(optional)* The value type of the keyValue store","type":"string"},"versioned":{"description":"*(optional)* \"true\" if elements in the store are versioned, \"false\" otherwise","type":"boolean"}},"title":"KeyValueStateStoreDefinitionWithImplicitType","type":"object"},"KeyValueToKeyValueListTransformerDefinition":{"additionalProperties":false,"description":"Defines a keyvalue-to-keyvaluelist transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue-to-keyvaluelist transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue-to-keyvaluelist transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue-to-keyvaluelist transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue-to-keyvaluelist transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue-to-keyvaluelist transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue-to-keyvaluelist transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the keyvalue-to-keyvaluelist transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"},"type":{"description":"The type of the function","enum":["keyValueToKeyValueListTransformer"]}},"required":["type"],"title":"KeyValueToKeyValueListTransformerDefinition","type":"object"},"KeyValueToKeyValueListTransformerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a keyvalue-to-keyvaluelist transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue-to-keyvaluelist transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue-to-keyvaluelist transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue-to-keyvaluelist transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue-to-keyvaluelist transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue-to-keyvaluelist transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue-to-keyvaluelist transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the keyvalue-to-keyvaluelist transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"}},"title":"KeyValueToKeyValueListTransformerDefinitionWithImplicitType","type":"object"},"KeyValueToValueListTransformerDefinition":{"additionalProperties":false,"description":"Defines a keyvalue-to-valuelist transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue-to-valuelist transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue-to-valuelist transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue-to-valuelist transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue-to-valuelist transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue-to-valuelist transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue-to-valuelist transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the keyvalue-to-valuelist transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"},"type":{"description":"The type of the function","enum":["keyValueToValueListTransformer"]}},"required":["type"],"title":"KeyValueToValueListTransformerDefinition","type":"object"},"KeyValueToValueListTransformerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a keyvalue-to-valuelist transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue-to-valuelist transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue-to-valuelist transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue-to-valuelist transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue-to-valuelist transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue-to-valuelist transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue-to-valuelist transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the keyvalue-to-valuelist transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"}},"title":"KeyValueToValueListTransformerDefinitionWithImplicitType","type":"object"},"KeyValueTransformerDefinition":{"additionalProperties":false,"description":"Defines a keyvalue transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the keyvalue transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"},"type":{"description":"The type of the function","enum":["keyValueTransformer"]}},"required":["type"],"title":"KeyValueTransformerDefinition","type":"object"},"KeyValueTransformerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a keyvalue transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the keyvalue transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the keyvalue transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the keyvalue transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the keyvalue transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the keyvalue transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the keyvalue transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"}},"title":"KeyValueTransformerDefinitionWithImplicitType","type":"object"},"LeftJoinOperationWithGlobalTable":{"additionalProperties":false,"description":"Operation to leftJoin with a globalTable","properties":{"globalTable":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/GlobalTableDefinition","type":"object"}],"description":"A reference to the globalTable, or an inline definition of the globalTable to join with"},"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueMapperDefinitionWithImplicitType","type":"object"}],"description":"A function that maps the key value from the stream with the primary key of the globalTable"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["leftJoin"]},"valueJoiner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueJoinerDefinitionWithImplicitType","type":"object"}],"description":"A function that joins two values"}},"required":["globalTable","mapper","valueJoiner","type"],"title":"LeftJoinOperationWithGlobalTable","type":"object"},"LeftJoinOperationWithStream":{"additionalProperties":false,"description":"Operation to leftJoin with a stream","properties":{"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinition","type":"object"},{"$ref":"#/definitions/SessionStateStoreDefinition","type":"object"},{"$ref":"#/definitions/WindowStateStoreDefinition","type":"object"}],"description":"*(optional)* Materialized view of the joined streams"},"stream":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamDefinition","type":"object"}],"description":"A reference to the stream, or an inline definition of the stream to leftJoin with"},"timeDifference":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The maximum time difference for a leftJoin over two streams on the same key"},"type":{"description":"The type of the operation","enum":["leftJoin"]},"valueJoiner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueJoinerDefinitionWithImplicitType","type":"object"}],"description":"A function that joins two values"}},"required":["stream","valueJoiner","timeDifference","type"],"title":"LeftJoinOperationWithStream","type":"object"},"LeftJoinOperationWithTable":{"additionalProperties":false,"description":"Operation to leftJoin with a table","properties":{"foreignKeyExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ForeignKeyExtractorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that can translate the join table value to a primary key"},"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"otherPartitioner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamPartitionerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that partitions the records on the join table"},"partitioner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamPartitionerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that partitions the records on the primary table"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinition","type":"object"},{"$ref":"#/definitions/SessionStateStoreDefinition","type":"object"},{"$ref":"#/definitions/WindowStateStoreDefinition","type":"object"}],"description":"*(optional)* Materialized view of the joined streams"},"table":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TableDefinition","type":"object"}],"description":"A reference to the Table, or an inline definition of the Table to join with"},"type":{"description":"The type of the operation","enum":["leftJoin"]},"valueJoiner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueJoinerDefinitionWithImplicitType","type":"object"}],"description":"A function that joins two values"}},"required":["table","valueJoiner","type"],"title":"LeftJoinOperationWithTable","type":"object"},"MergeOperation":{"additionalProperties":false,"description":"A merge operation to join two Streams","properties":{"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"stream":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamDefinitionSource","type":"object"}],"description":"The stream to merge with"},"type":{"description":"The type of the operation","enum":["merge"]}},"required":["stream","type"],"title":"MergeOperation","type":"object"},"MergerDefinition":{"additionalProperties":false,"description":"Defines a merger function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the merger"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the merger. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the merger. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the merger. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the merger","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the merger. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["merger"]}},"required":["type"],"title":"MergerDefinition","type":"object"},"MergerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a merger function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the merger"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the merger. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the merger. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the merger. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the merger","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the merger. Only required for function types, which are not pre-defined.","type":"string"}},"title":"MergerDefinitionWithImplicitType","type":"object"},"MetadataTransformerDefinition":{"additionalProperties":false,"description":"Defines a metadata transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the metadata transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the metadata transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the metadata transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the metadata transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the metadata transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the metadata transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the metadata transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"},"type":{"description":"The type of the function","enum":["metadataTransformer"]}},"required":["type"],"title":"MetadataTransformerDefinition","type":"object"},"MetadataTransformerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a metadata transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the metadata transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the metadata transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the metadata transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the metadata transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the metadata transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the metadata transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the metadata transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"}},"title":"MetadataTransformerDefinitionWithImplicitType","type":"object"},"OuterJoinOperationWithStream":{"additionalProperties":false,"description":"Operation to outerJoin with a stream","properties":{"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinition","type":"object"},{"$ref":"#/definitions/SessionStateStoreDefinition","type":"object"},{"$ref":"#/definitions/WindowStateStoreDefinition","type":"object"}],"description":"*(optional)* Materialized view of the outerJoined streams"},"stream":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamDefinition","type":"object"}],"description":"A reference to the stream, or an inline definition of the stream to outerJoin with"},"timeDifference":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The maximum time difference for an outerJoin over two streams on the same key"},"type":{"description":"The type of the operation","enum":["outerJoin"]},"valueJoiner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueJoinerDefinitionWithImplicitType","type":"object"}],"description":"A function that joins two values"}},"required":["stream","valueJoiner","timeDifference","type"],"title":"OuterJoinOperationWithStream","type":"object"},"OuterJoinOperationWithTable":{"additionalProperties":false,"description":"Operation to outerJoin with a table","properties":{"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinition","type":"object"},{"$ref":"#/definitions/SessionStateStoreDefinition","type":"object"},{"$ref":"#/definitions/WindowStateStoreDefinition","type":"object"}],"description":"*(optional)* Materialized view of the outerJoined streams"},"table":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TableDefinition","type":"object"}],"description":"A reference to the table, or an inline definition of the table to outerJoin with"},"type":{"description":"The type of the operation","enum":["outerJoin"]},"valueJoiner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueJoinerDefinitionWithImplicitType","type":"object"}],"description":"A function that joins two values"}},"required":["table","valueJoiner","type"],"title":"OuterJoinOperationWithTable","type":"object"},"ParameterDefinition":{"additionalProperties":false,"description":"Defines a parameter for a user function","properties":{"defaultValue":{"description":"*(optional)* The default value for the parameter","type":"string"},"name":{"description":"The name of the parameter","type":"string"},"type":{"description":"The type of the parameter","type":"string"}},"required":["name","type"],"title":"ParameterDefinition","type":"object"},"PeekOperation":{"additionalProperties":false,"description":"Operation to peek into a stream, without modifying the stream contents","properties":{"forEach":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ForEachActionDefinitionWithImplicitType","type":"object"}],"description":"A function that gets called for every message in the stream"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["peek"]}},"required":["forEach","type"],"title":"PeekOperation","type":"object"},"PipelineDefinition":{"additionalProperties":false,"description":"Defines a pipeline through a source, a series of operations to perform on it and a sink operation to close the stream with","properties":{"as":{"description":"*(optional)* The name to register the pipeline result under, which can be used as source by follow-up pipelines","type":"string"},"branch":{"description":"*(optional)* Defines a single branch, consisting of a condition and a pipeline to execute for messages that fulfil the predicate","items":{"$ref":"#/definitions/BranchDefinitionWithPipeline","type":"object"},"type":"array"},"forEach":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ForEachActionDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that gets called for every message in the stream"},"from":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TopicDefinitionSource","type":"object"}],"description":"Pipeline source"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"print":{"$ref":"#/definitions/PrintOperation","description":"*(optional)* The specification of where to print messages to","type":"object"},"to":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ToTopicDefinition","type":"object"}],"description":"*(optional)* Ends the pipeline by sending all messages to a stream, table or globalTable, or to an inline defined output topic and optional partitioner"},"toTopicNameExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ToTopicNameExtractorDefinition","type":"object"}],"description":"*(optional)* Ends the pipeline by sending all messages to a topic provided by a pre-defined topic name extractor function, or to a topic provided by an inline defined topic name extractor and optional partitioner"},"via":{"description":"*(optional)* A series of operations performed on the input stream","items":{"anyOf":[{"$ref":"#/definitions/AggregateOperation","type":"object"},{"$ref":"#/definitions/CogroupOperation","type":"object"},{"$ref":"#/definitions/ConvertKeyOperation","type":"object"},{"$ref":"#/definitions/ConvertKeyValueOperation","type":"object"},{"$ref":"#/definitions/ConvertValueOperation","type":"object"},{"$ref":"#/definitions/CountOperation","type":"object"},{"$ref":"#/definitions/FilterNotOperation","type":"object"},{"$ref":"#/definitions/FilterOperation","type":"object"},{"$ref":"#/definitions/GroupByKeyOperation","type":"object"},{"$ref":"#/definitions/GroupByOperation","type":"object"},{"$ref":"#/definitions/JoinOperationWithGlobalTable","type":"object"},{"$ref":"#/definitions/JoinOperationWithStream","type":"object"},{"$ref":"#/definitions/JoinOperationWithTable","type":"object"},{"$ref":"#/definitions/LeftJoinOperationWithGlobalTable","type":"object"},{"$ref":"#/definitions/LeftJoinOperationWithStream","type":"object"},{"$ref":"#/definitions/LeftJoinOperationWithTable","type":"object"},{"$ref":"#/definitions/MergeOperation","type":"object"},{"$ref":"#/definitions/OuterJoinOperationWithStream","type":"object"},{"$ref":"#/definitions/OuterJoinOperationWithTable","type":"object"},{"$ref":"#/definitions/PeekOperation","type":"object"},{"$ref":"#/definitions/ReduceOperationWithAdderAndSubtractor","type":"object"},{"$ref":"#/definitions/ReduceOperationWithReducer","type":"object"},{"$ref":"#/definitions/RepartitionOperation","type":"object"},{"$ref":"#/definitions/SuppressOperationUntilTimeLimit","type":"object"},{"$ref":"#/definitions/SuppressOperationUntilWindowCloses","type":"object"},{"$ref":"#/definitions/ToStreamOperation","type":"object"},{"$ref":"#/definitions/ToTableOperation","type":"object"},{"$ref":"#/definitions/TransformKeyOperation","type":"object"},{"$ref":"#/definitions/TransformKeyValueOperation","type":"object"},{"$ref":"#/definitions/TransformKeyValueToKeyValueListOperation","type":"object"},{"$ref":"#/definitions/TransformKeyValueToValueListOperation","type":"object"},{"$ref":"#/definitions/TransformMetadataOperation","type":"object"},{"$ref":"#/definitions/TransformValueOperation","type":"object"},{"$ref":"#/definitions/WindowBySessionOperation","type":"object"},{"$ref":"#/definitions/WindowByTimeOperationWithHoppingWindow","type":"object"},{"$ref":"#/definitions/WindowByTimeOperationWithSlidingWindow","type":"object"},{"$ref":"#/definitions/WindowByTimeOperationWithTumblingWindow","type":"object"}]},"type":"array"}},"required":["from"],"title":"PipelineDefinition","type":"object"},"PredicateDefinition":{"additionalProperties":false,"description":"Defines a predicate function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the predicate"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the predicate. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the predicate. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the predicate. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the predicate","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the predicate. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the predicate uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"},"type":{"description":"The type of the function","enum":["predicate"]}},"required":["type"],"title":"PredicateDefinition","type":"object"},"PredicateDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a predicate function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the predicate"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the predicate. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the predicate. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the predicate. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the predicate","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the predicate. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the predicate uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"}},"title":"PredicateDefinitionWithImplicitType","type":"object"},"PrintOperation":{"additionalProperties":false,"description":"Operation to print the contents of a pipeline on the screen or to write them to a file","properties":{"filename":{"description":"*(optional)* The filename to output records to. If nothing is specified, then messages will be printed on stdout.","type":"string"},"label":{"description":"*(optional)* A label to attach to the output records","type":"string"},"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValuePrinterDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function to convert record into a string for output"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"}},"title":"PrintOperation","type":"object"},"ProducerDefinition":{"additionalProperties":false,"description":"Definition of a Producer that regularly generates messages for a topic","properties":{"condition":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/PredicateDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that validates the generator's result message. Returns \"true\" when the message may be produced on the topic, \"false\" otherwise."},"count":{"description":"*(optional)* The number of messages to produce."},"generator":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/GeneratorDefinitionWithImplicitType","type":"object"}],"description":"The function that generates records"},"interval":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The interval with which the generator is called"},"to":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TopicDefinition","type":"object"}],"description":"The topic to produce to"},"until":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/PredicateDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A predicate that returns true to indicate producing should stop."}},"required":["generator","interval","to"],"title":"ProducerDefinition","type":"object"},"ReduceOperationWithAdderAndSubtractor":{"additionalProperties":false,"description":"Operation to reduce a series of records into a single aggregate result","properties":{"adder":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ReducerDefinitionWithImplicitType","type":"object"}],"description":"A function that adds a record to the aggregate result"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/WindowStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the aggregation"},"subtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ReducerDefinitionWithImplicitType","type":"object"}],"description":"A function that removes a record from the aggregate result"},"type":{"description":"The type of the operation","enum":["reduce"]}},"required":["adder","subtractor","type"],"title":"ReduceOperationWithAdderAndSubtractor","type":"object"},"ReduceOperationWithReducer":{"additionalProperties":false,"description":"Operation to reduce a series of records into a single aggregate result","properties":{"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"reducer":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ReducerDefinitionWithImplicitType","type":"object"}],"description":"A function that computes a new aggregate result"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/WindowStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the aggregation"},"type":{"description":"The type of the operation","enum":["reduce"]}},"required":["reducer","type"],"title":"ReduceOperationWithReducer","type":"object"},"ReducerDefinition":{"additionalProperties":false,"description":"Defines a reducer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the reducer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the reducer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the reducer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the reducer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the reducer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the reducer. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["reducer"]}},"required":["type"],"title":"ReducerDefinition","type":"object"},"ReducerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a reducer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the reducer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the reducer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the reducer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the reducer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the reducer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the reducer. Only required for function types, which are not pre-defined.","type":"string"}},"title":"ReducerDefinitionWithImplicitType","type":"object"},"RepartitionOperation":{"additionalProperties":false,"description":"Operation to (re)partition a stream","properties":{"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"numberOfPartitions":{"description":"*(optional)* The target number of partitions"},"partitioner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamPartitionerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that partitions stream records"},"type":{"description":"The type of the operation","enum":["repartition"]}},"required":["type"],"title":"RepartitionOperation","type":"object"},"SessionStateStoreDefinition":{"additionalProperties":false,"description":"Definition of a session state store","properties":{"caching":{"description":"*(optional)* \"true\" if changed to the session store need to be buffered and periodically released, \"false\" to emit all changes directly","type":"boolean"},"keyType":{"description":"*(optional)* The key type of the session store","type":"string"},"logging":{"description":"*(optional)* \"true\" if a changelog topic should be set up on Kafka for this session store, \"false\" otherwise","type":"boolean"},"name":{"description":"*(optional)* The name of the session store. If this field is not defined, then the name is derived from the context.","type":"string"},"persistent":{"description":"*(optional)* \"true\" if this session store needs to be stored on disk, \"false\" otherwise","type":"boolean"},"retention":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The duration for which elements in the session store are retained"},"timestamped":{"description":"*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise","type":"boolean"},"type":{"description":"The type of the state store","enum":["session"]},"valueType":{"description":"*(optional)* The value type of the session store","type":"string"}},"required":["type"],"title":"SessionStateStoreDefinition","type":"object"},"StreamDefinition":{"additionalProperties":false,"description":"Contains a definition of a Stream, which can be referenced by producers and pipelines","properties":{"keyType":{"description":"*(optional)* The key type of the stream","type":"string"},"topic":{"description":"The name of the Kafka topic for this stream","type":"string"},"valueType":{"description":"*(optional)* The value type of the stream","type":"string"}},"required":["topic"],"title":"StreamDefinition","type":"object"},"StreamDefinitionSource":{"additionalProperties":false,"description":"Contains a definition of a Stream, which can be referenced by producers and pipelines","properties":{"keyType":{"description":"The key type of the stream","type":"string"},"offsetResetPolicy":{"description":"*(optional)* Policy that determines what to do when there is no initial offset in Kafka, or if the current offset does not exist any more on the server (e.g. because that data has been deleted)","type":"string"},"timestampExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TimestampExtractorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function extracts the event time from a consumed record"},"topic":{"description":"The name of the Kafka topic for this stream","type":"string"},"valueType":{"description":"The value type of the stream","type":"string"}},"required":["topic","keyType","valueType"],"title":"StreamDefinitionSource","type":"object"},"StreamPartitionerDefinition":{"additionalProperties":false,"description":"Defines a stream partitioner function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the stream partitioner"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the stream partitioner. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the stream partitioner. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the stream partitioner. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the stream partitioner","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the stream partitioner. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["streamPartitioner"]}},"required":["type"],"title":"StreamPartitionerDefinition","type":"object"},"StreamPartitionerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a stream partitioner function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the stream partitioner"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the stream partitioner. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the stream partitioner. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the stream partitioner. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the stream partitioner","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the stream partitioner. Only required for function types, which are not pre-defined.","type":"string"}},"title":"StreamPartitionerDefinitionWithImplicitType","type":"object"},"StringOrInlinePredicateDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines the condition under which messages get sent down this branch","properties":{"if":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/PredicateDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Defines the condition under which messages get sent down this branch"}},"title":"StringOrInlinePredicateDefinitionWithImplicitType","type":"object"},"SuppressOperationUntilTimeLimit":{"additionalProperties":false,"description":"Operation to suppress messages in the source stream until a time limit is reached","properties":{"bufferFullStrategy":{"description":"*(optional)* What to do when the buffer is full","enum":["emitEarlyWhenFull","shutdownWhenFull"]},"duration":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The duration for which messages are suppressed"},"maxBytes":{"description":"*(optional)* The maximum number of bytes in the buffer","type":"string"},"maxRecords":{"description":"*(optional)* The maximum number of records in the buffer","type":"string"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["suppress"]},"until":{"description":"The until of the Operation to suppress messages in the source stream until a certain limit is reached","enum":["timeLimit"]}},"required":["duration","until","type"],"title":"SuppressOperationUntilTimeLimit","type":"object"},"SuppressOperationUntilWindowCloses":{"additionalProperties":false,"description":"Operation to suppress messages in the source stream until a window limit is reached","properties":{"bufferFullStrategy":{"description":"*(optional)* What to do when the buffer is full","enum":["emitEarlyWhenFull","shutdownWhenFull"]},"maxBytes":{"description":"*(optional)* The maximum number of bytes in the buffer","type":"string"},"maxRecords":{"description":"*(optional)* The maximum number of records in the buffer","type":"string"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["suppress"]},"until":{"description":"The until of the Operation to suppress messages in the source stream until a certain limit is reached","enum":["windowCloses"]}},"required":["until","type"],"title":"SuppressOperationUntilWindowCloses","type":"object"},"TableDefinition":{"additionalProperties":false,"description":"Contains a definition of a Table, which can be referenced by producers and pipelines","properties":{"keyType":{"description":"*(optional)* The key type of the table","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* KeyValue state store definition"},"topic":{"description":"The name of the Kafka topic for this table","type":"string"},"valueType":{"description":"*(optional)* The value type of the table","type":"string"}},"required":["topic"],"title":"TableDefinition","type":"object"},"TableDefinitionSource":{"additionalProperties":false,"description":"Contains a definition of a Table, which can be referenced by producers and pipelines","properties":{"keyType":{"description":"The key type of the table","type":"string"},"offsetResetPolicy":{"description":"*(optional)* Policy that determines what to do when there is no initial offset in Kafka, or if the current offset does not exist any more on the server (e.g. because that data has been deleted)","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* KeyValue state store definition"},"timestampExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TimestampExtractorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function extracts the event time from a consumed record"},"topic":{"description":"The name of the Kafka topic for this table","type":"string"},"valueType":{"description":"The value type of the table","type":"string"}},"required":["topic","keyType","valueType"],"title":"TableDefinitionSource","type":"object"},"TimestampExtractorDefinition":{"additionalProperties":false,"description":"Defines a timestamp extractor function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the timestamp extractor"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the timestamp extractor. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the timestamp extractor. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the timestamp extractor. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the timestamp extractor","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the timestamp extractor. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["timestampExtractor"]}},"required":["type"],"title":"TimestampExtractorDefinition","type":"object"},"TimestampExtractorDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a timestamp extractor function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the timestamp extractor"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the timestamp extractor. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the timestamp extractor. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the timestamp extractor. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the timestamp extractor","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the timestamp extractor. Only required for function types, which are not pre-defined.","type":"string"}},"title":"TimestampExtractorDefinitionWithImplicitType","type":"object"},"ToStreamOperation":{"additionalProperties":false,"description":"Convert a Table into a Stream, optionally through a custom key transformer","properties":{"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyTransformerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that computes the output key for every record"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["toStream"]}},"required":["type"],"title":"ToStreamOperation","type":"object"},"ToTableOperation":{"additionalProperties":false,"description":"Convert a Stream into a Table","properties":{"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the result table"},"type":{"description":"The type of the operation","enum":["toTable"]}},"required":["type"],"title":"ToTableOperation","type":"object"},"ToTopicDefinition":{"additionalProperties":false,"description":"Writes out pipeline messages to a topic","properties":{"keyType":{"description":"*(optional)* The key type of the topic","type":"string"},"partitioner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamPartitionerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that partitions the records in the output topic"},"topic":{"description":"The name of the Kafka topic","type":"string"},"valueType":{"description":"*(optional)* The value type of the topic","type":"string"}},"required":["topic"],"title":"ToTopicDefinition","type":"object"},"ToTopicNameExtractorDefinition":{"additionalProperties":false,"description":"Writes out pipeline messages to a topic as given by a topic name extractor","properties":{"partitioner":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/StreamPartitionerDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function that partitions the records in the output topic"},"topicNameExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TopicNameExtractorDefinitionWithImplicitType","type":"object"}],"description":"Reference to a pre-defined topic name extractor, or an inline definition of a topic name extractor"}},"required":["topicNameExtractor"],"title":"ToTopicNameExtractorDefinition","type":"object"},"TopicDefinition":{"additionalProperties":false,"description":"Contains a definition of a Kafka topic, to be used by producers and pipelines","properties":{"keyType":{"description":"*(optional)* The key type of the topic","type":"string"},"topic":{"description":"The name of the Kafka topic","type":"string"},"valueType":{"description":"*(optional)* The value type of the topic","type":"string"}},"required":["topic"],"title":"TopicDefinition","type":"object"},"TopicDefinitionSource":{"additionalProperties":false,"description":"Contains a definition of a Kafka topic, to be used by producers and pipelines","properties":{"keyType":{"description":"The key type of the topic","type":"string"},"offsetResetPolicy":{"description":"*(optional)* Policy that determines what to do when there is no initial offset in Kafka, or if the current offset does not exist any more on the server (e.g. because that data has been deleted)","type":"string"},"timestampExtractor":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/TimestampExtractorDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* A function extracts the event time from a consumed record"},"topic":{"description":"The name of the Kafka topic","type":"string"},"valueType":{"description":"The value type of the topic","type":"string"}},"required":["topic","keyType","valueType"],"title":"TopicDefinitionSource","type":"object"},"TopicNameExtractorDefinition":{"additionalProperties":false,"description":"Defines a topic name extractor function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the topic name extractor"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the topic name extractor. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the topic name extractor. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the topic name extractor. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the topic name extractor","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the topic name extractor. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["topicNameExtractor"]}},"required":["type"],"title":"TopicNameExtractorDefinition","type":"object"},"TopicNameExtractorDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a topic name extractor function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the topic name extractor"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the topic name extractor. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the topic name extractor. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the topic name extractor. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the topic name extractor","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the topic name extractor. Only required for function types, which are not pre-defined.","type":"string"}},"title":"TopicNameExtractorDefinitionWithImplicitType","type":"object"},"TransformKeyOperation":{"additionalProperties":false,"description":"Convert the key of every record in the stream to another key","properties":{"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyTransformerDefinitionWithImplicitType","type":"object"}],"description":"A function that computes a new key for each record"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["transformKey","mapKey","selectKey"]}},"required":["mapper","type"],"title":"TransformKeyOperation","type":"object"},"TransformKeyValueOperation":{"additionalProperties":false,"description":"Convert the key/value of every record in the stream to another key/value","properties":{"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueTransformerDefinitionWithImplicitType","type":"object"}],"description":"A function that computes a new key/value for each record"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["mapKeyValue","map","transformKeyValue"]}},"required":["mapper","type"],"title":"TransformKeyValueOperation","type":"object"},"TransformKeyValueToKeyValueListOperation":{"additionalProperties":false,"description":"Convert a stream by transforming every record into a list of derived records","properties":{"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueToKeyValueListTransformerDefinitionWithImplicitType","type":"object"}],"description":"A function that converts every record of a stream to a list of output records."},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["transformKeyValueToKeyValueList","flatMap"]}},"required":["mapper","type"],"title":"TransformKeyValueToKeyValueListOperation","type":"object"},"TransformKeyValueToValueListOperation":{"additionalProperties":false,"description":"Convert every record in the stream to a list of output records with the same key","properties":{"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueToValueListTransformerDefinitionWithImplicitType","type":"object"}],"description":"A function that converts every key/value into a list of result values, each of which will be combined with the original key to form a new message in the output stream"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["transformKeyValueToValueList","flatMapValues"]}},"required":["mapper","type"],"title":"TransformKeyValueToValueListOperation","type":"object"},"TransformMetadataOperation":{"additionalProperties":false,"description":"Convert the metadata of every record in the stream","properties":{"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/MetadataTransformerDefinitionWithImplicitType","type":"object"}],"description":"A function that converts the metadata (Kafka headers, timestamp) of every record in the stream"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["transformMetadata"]}},"required":["mapper","type"],"title":"TransformMetadataOperation","type":"object"},"TransformValueOperation":{"additionalProperties":false,"description":"Convert the value of every record in the stream to another value","properties":{"mapper":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/ValueTransformerDefinitionWithImplicitType","type":"object"}],"description":"A function that converts the value of every record into another value"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"store":{"anyOf":[{"type":"string"},{"$ref":"#/definitions/KeyValueStateStoreDefinitionWithImplicitType","type":"object"}],"description":"*(optional)* Materialized view of the transformed table (only applies to tables, ignored for streams)"},"type":{"description":"The type of the operation","enum":["mapValue","transformValue","mapValues"]}},"required":["mapper","type"],"title":"TransformValueOperation","type":"object"},"ValueJoinerDefinition":{"additionalProperties":false,"description":"Defines a value joiner function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the value joiner"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the value joiner. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the value joiner. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the value joiner. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the value joiner","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the value joiner. Only required for function types, which are not pre-defined.","type":"string"},"type":{"description":"The type of the function","enum":["valueJoiner"]}},"required":["type"],"title":"ValueJoinerDefinition","type":"object"},"ValueJoinerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a value joiner function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the value joiner"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the value joiner. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the value joiner. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the value joiner. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the value joiner","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the value joiner. Only required for function types, which are not pre-defined.","type":"string"}},"title":"ValueJoinerDefinitionWithImplicitType","type":"object"},"ValueTransformerDefinition":{"additionalProperties":false,"description":"Defines a value transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the value transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the value transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the value transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the value transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the value transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the value transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the value transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"},"type":{"description":"The type of the function","enum":["valueTransformer"]}},"required":["type"],"title":"ValueTransformerDefinition","type":"object"},"ValueTransformerDefinitionWithImplicitType":{"additionalProperties":false,"description":"Defines a value transformer function, that gets injected into the Kafka Streams topology","properties":{"code":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The (multiline) code of the value transformer"},"expression":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* The expression returned by the value transformer. Only required for functions that return values."},"globalCode":{"anyOf":[{"type":"boolean"},{"type":"number"},{"type":"string"}],"description":"*(optional)* Global (multiline) code that gets loaded into the Python context outside of the value transformer. Can be used for defining eg. global variables."},"name":{"description":"*(optional)* The name of the value transformer. If this field is not defined, then the name is derived from the context.","type":"string"},"parameters":{"description":"*(optional)* A list of parameters to be passed into the value transformer","items":{"$ref":"#/definitions/ParameterDefinition","type":"object"},"type":"array"},"resultType":{"description":"*(optional)* The data type returned by the value transformer. Only required for function types, which are not pre-defined.","type":"string"},"stores":{"description":"*(optional)* A list of store names that the value transformer uses. Only required if the function wants to use a state store.","items":{"type":"string"},"type":"array"}},"title":"ValueTransformerDefinitionWithImplicitType","type":"object"},"WindowBySessionOperation":{"additionalProperties":false,"description":"Operation to window messages by session, configured by an inactivity gap","properties":{"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* (Tumbling, Hopping) The grace period, during which out-of-order records can still be processed"},"inactivityGap":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The inactivity gap, below which two messages are considered to be of the same session"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["windowBySession"]}},"required":["inactivityGap","type"],"title":"WindowBySessionOperation","type":"object"},"WindowByTimeOperationWithHoppingWindow":{"additionalProperties":false,"description":"Operation to window records based on time criteria","properties":{"advanceBy":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The amount of time to increase time windows by"},"duration":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The duration of time windows"},"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The grace period, during which out-of-order records can still be processed"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["windowByTime"]},"windowType":{"description":"The windowType of the time window","enum":["hopping"]}},"required":["duration","advanceBy","windowType","type"],"title":"WindowByTimeOperationWithHoppingWindow","type":"object"},"WindowByTimeOperationWithSlidingWindow":{"additionalProperties":false,"description":"Operation to window records based on time criteria","properties":{"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The grace period, during which out-of-order records can still be processed"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"timeDifference":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The maximum amount of time difference between two records"},"type":{"description":"The type of the operation","enum":["windowByTime"]},"windowType":{"description":"The windowType of the time window","enum":["sliding"]}},"required":["timeDifference","windowType","type"],"title":"WindowByTimeOperationWithSlidingWindow","type":"object"},"WindowByTimeOperationWithTumblingWindow":{"additionalProperties":false,"description":"Operation to window records based on time criteria","properties":{"duration":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"The duration of time windows"},"grace":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The grace period, during which out-of-order records can still be processed"},"name":{"description":"*(optional)* The name of the operation processor","type":"string"},"type":{"description":"The type of the operation","enum":["windowByTime"]},"windowType":{"description":"The windowType of the time window","enum":["tumbling"]}},"required":["duration","windowType","type"],"title":"WindowByTimeOperationWithTumblingWindow","type":"object"},"WindowStateStoreDefinition":{"additionalProperties":false,"description":"Definition of a window state store","properties":{"caching":{"description":"*(optional)* \"true\" if changed to the window store need to be buffered and periodically released, \"false\" to emit all changes directly","type":"boolean"},"keyType":{"description":"*(optional)* The key type of the window store","type":"string"},"logging":{"description":"*(optional)* \"true\" if a changelog topic should be set up on Kafka for this window store, \"false\" otherwise","type":"boolean"},"name":{"description":"*(optional)* The name of the window store. If this field is not defined, then the name is derived from the context.","type":"string"},"persistent":{"description":"*(optional)* \"true\" if this window store needs to be stored on disk, \"false\" otherwise","type":"boolean"},"retainDuplicates":{"description":"*(optional)* Whether or not to retain duplicates","type":"boolean"},"retention":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The duration for which elements in the window store are retained"},"timestamped":{"description":"*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise","type":"boolean"},"type":{"description":"The type of the state store","enum":["window"]},"valueType":{"description":"*(optional)* The value type of the window store","type":"string"},"windowSize":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* Size of the windows (cannot be negative)"}},"required":["type"],"title":"WindowStateStoreDefinition","type":"object"},"WindowStateStoreDefinitionWithImplicitType":{"additionalProperties":false,"description":"Definition of a window state store","properties":{"caching":{"description":"*(optional)* \"true\" if changed to the window store need to be buffered and periodically released, \"false\" to emit all changes directly","type":"boolean"},"keyType":{"description":"*(optional)* The key type of the window store","type":"string"},"logging":{"description":"*(optional)* \"true\" if a changelog topic should be set up on Kafka for this window store, \"false\" otherwise","type":"boolean"},"name":{"description":"*(optional)* The name of the window store. If this field is not defined, then the name is derived from the context.","type":"string"},"persistent":{"description":"*(optional)* \"true\" if this window store needs to be stored on disk, \"false\" otherwise","type":"boolean"},"retainDuplicates":{"description":"*(optional)* Whether or not to retain duplicates","type":"boolean"},"retention":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* The duration for which elements in the window store are retained"},"timestamped":{"description":"*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise","type":"boolean"},"type":{"description":"The type of the state store","enum":["window"]},"valueType":{"description":"*(optional)* The value type of the window store","type":"string"},"windowSize":{"anyOf":[{"type":"number"},{"type":"string"}],"description":"*(optional)* Size of the windows (cannot be negative)"}},"title":"WindowStateStoreDefinitionWithImplicitType","type":"object"}},"description":"KSML definition","properties":{"functions":{"description":"*(optional)* Functions that can be referenced in producers and pipelines","patternProperties":{"^[a-zA-Z0-9_]+$":{"anyOf":[{"$ref":"#/definitions/AggregatorDefinition","type":"object"},{"$ref":"#/definitions/ForEachActionDefinition","type":"object"},{"$ref":"#/definitions/ForeignKeyExtractorDefinition","type":"object"},{"$ref":"#/definitions/GeneratorDefinition","type":"object"},{"$ref":"#/definitions/GenericFunctionDefinitionWithImplicitType","type":"object"},{"$ref":"#/definitions/InitializerDefinition","type":"object"},{"$ref":"#/definitions/KeyTransformerDefinition","type":"object"},{"$ref":"#/definitions/KeyValueMapperDefinition","type":"object"},{"$ref":"#/definitions/KeyValuePrinterDefinition","type":"object"},{"$ref":"#/definitions/KeyValueToKeyValueListTransformerDefinition","type":"object"},{"$ref":"#/definitions/KeyValueToValueListTransformerDefinition","type":"object"},{"$ref":"#/definitions/KeyValueTransformerDefinition","type":"object"},{"$ref":"#/definitions/MergerDefinition","type":"object"},{"$ref":"#/definitions/MetadataTransformerDefinition","type":"object"},{"$ref":"#/definitions/PredicateDefinition","type":"object"},{"$ref":"#/definitions/ReducerDefinition","type":"object"},{"$ref":"#/definitions/StreamPartitionerDefinition","type":"object"},{"$ref":"#/definitions/TimestampExtractorDefinition","type":"object"},{"$ref":"#/definitions/TopicNameExtractorDefinition","type":"object"},{"$ref":"#/definitions/ValueJoinerDefinition","type":"object"},{"$ref":"#/definitions/ValueTransformerDefinition","type":"object"}]}},"type":"object"},"globalTables":{"description":"*(optional)* GlobalTables that can be referenced in producers and pipelines","patternProperties":{"^[a-zA-Z0-9_]+$":{"$ref":"#/definitions/GlobalTableDefinitionSource","type":"object"}},"type":"object"},"pipelines":{"description":"*(optional)* Collection of named pipelines","patternProperties":{"^[a-zA-Z0-9_]+$":{"$ref":"#/definitions/PipelineDefinition","type":"object"}},"type":"object"},"producers":{"description":"*(optional)* Collection of named producers","patternProperties":{"^[a-zA-Z0-9_]+$":{"$ref":"#/definitions/ProducerDefinition","type":"object"}},"type":"object"},"stores":{"description":"*(optional)* State stores that can be referenced in producers and pipelines","patternProperties":{"^[a-zA-Z0-9_]+$":{"anyOf":[{"$ref":"#/definitions/KeyValueStateStoreDefinition","type":"object"},{"$ref":"#/definitions/SessionStateStoreDefinition","type":"object"},{"$ref":"#/definitions/WindowStateStoreDefinition","type":"object"}]}},"type":"object"},"streams":{"description":"*(optional)* Streams that can be referenced in producers and pipelines","patternProperties":{"^[a-zA-Z0-9_]+$":{"$ref":"#/definitions/StreamDefinitionSource","type":"object"}},"type":"object"},"tables":{"description":"*(optional)* Tables that can be referenced in producers and pipelines","patternProperties":{"^[a-zA-Z0-9_]+$":{"$ref":"#/definitions/TableDefinitionSource","type":"object"}},"type":"object"}},"title":"TopologyDefinition","type":"object"} +{ + "additionalProperties" : false, + "definitions" : { + "AggregateOperation" : { + "additionalProperties" : false, + "description" : "An aggregate operation", + "properties" : { + "adder" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/AggregatorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* (GroupedTable) A function that adds a record to the aggregation result" + }, + "aggregator" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/AggregatorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* (GroupedStream, SessionWindowedStream, TimeWindowedStream) The aggregator function, which combines a value with the previous aggregation result and outputs a new aggregation result" + }, + "initializer" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/InitializerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "The initializer function, which generates an initial value for every set of aggregated records" + }, + "merger" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/MergerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* (SessionWindowedStream, SessionWindowedCogroupedStream) A function that combines two aggregation results" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the result aggregation" + }, + "subtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/AggregatorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* (GroupedTable) A function that removes a record from the aggregation result" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "aggregate" ] + } + }, + "required" : [ "initializer", "type" ], + "title" : "AggregateOperation", + "type" : "object" + }, + "AggregatorDefinition" : { + "additionalProperties" : false, + "description" : "Defines a aggregator function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the aggregator" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the aggregator. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the aggregator. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the aggregator. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the aggregator", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the aggregator. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "aggregator" ] + } + }, + "required" : [ "type" ], + "title" : "AggregatorDefinition", + "type" : "object" + }, + "AggregatorDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a aggregator function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the aggregator" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the aggregator. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the aggregator. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the aggregator. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the aggregator", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the aggregator. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "AggregatorDefinitionWithImplicitType", + "type" : "object" + }, + "BranchDefinitionWithPipeline" : { + "additionalProperties" : false, + "description" : "Defines a branch with sub-pipeline in a BranchOperation", + "properties" : { + "as" : { + "description" : "*(optional)* The name to register the pipeline result under, which can be used as source by follow-up pipelines", + "type" : "string" + }, + "branch" : { + "description" : "*(optional)* Defines a single branch, consisting of a condition and a pipeline to execute for messages that fulfil the predicate", + "items" : { + "$ref" : "#/definitions/StringOrInlinePredicateDefinitionWithImplicitType", + "type" : "object" + }, + "type" : "array" + }, + "forEach" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ForEachActionDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that gets called for every message in the stream" + }, + "if" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/PredicateDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Defines the condition under which messages get sent down this branch" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "print" : { + "$ref" : "#/definitions/PrintOperation", + "description" : "*(optional)* The specification of where to print messages to", + "type" : "object" + }, + "to" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ToTopicDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Ends the pipeline by sending all messages to a stream, table or globalTable, or to an inline defined output topic and optional partitioner" + }, + "toTopicNameExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ToTopicNameExtractorDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Ends the pipeline by sending all messages to a topic provided by a pre-defined topic name extractor function, or to a topic provided by an inline defined topic name extractor and optional partitioner" + }, + "via" : { + "description" : "*(optional)* A series of operations performed on the input stream", + "items" : { + "anyOf" : [ { + "$ref" : "#/definitions/AggregateOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/CogroupOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ConvertKeyOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ConvertKeyValueOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ConvertValueOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/CountOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/FilterNotOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/FilterOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/GroupByKeyOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/GroupByOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/JoinOperationWithGlobalTable", + "type" : "object" + }, { + "$ref" : "#/definitions/JoinOperationWithStream", + "type" : "object" + }, { + "$ref" : "#/definitions/JoinOperationWithTable", + "type" : "object" + }, { + "$ref" : "#/definitions/LeftJoinOperationWithGlobalTable", + "type" : "object" + }, { + "$ref" : "#/definitions/LeftJoinOperationWithStream", + "type" : "object" + }, { + "$ref" : "#/definitions/LeftJoinOperationWithTable", + "type" : "object" + }, { + "$ref" : "#/definitions/MergeOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/OuterJoinOperationWithStream", + "type" : "object" + }, { + "$ref" : "#/definitions/OuterJoinOperationWithTable", + "type" : "object" + }, { + "$ref" : "#/definitions/PeekOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ReduceOperationWithAdderAndSubtractor", + "type" : "object" + }, { + "$ref" : "#/definitions/ReduceOperationWithReducer", + "type" : "object" + }, { + "$ref" : "#/definitions/RepartitionOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/SuppressOperationUntilTimeLimit", + "type" : "object" + }, { + "$ref" : "#/definitions/SuppressOperationUntilWindowCloses", + "type" : "object" + }, { + "$ref" : "#/definitions/ToStreamOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ToTableOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformKeyOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformKeyValueOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformKeyValueToKeyValueListOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformKeyValueToValueListOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformMetadataOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformValueOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowBySessionOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowByTimeOperationWithHoppingWindow", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowByTimeOperationWithSlidingWindow", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowByTimeOperationWithTumblingWindow", + "type" : "object" + } ] + }, + "type" : "array" + } + }, + "title" : "BranchDefinitionWithPipeline", + "type" : "object" + }, + "CogroupOperation" : { + "additionalProperties" : false, + "description" : "A cogroup operation", + "properties" : { + "aggregator" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/AggregatorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "(GroupedStream, SessionWindowedStream, TimeWindowedStream) The aggregator function, which combines a value with the previous aggregation result and outputs a new aggregation result" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the co-grouped stream" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "cogroup" ] + } + }, + "required" : [ "aggregator", "type" ], + "title" : "CogroupOperation", + "type" : "object" + }, + "ConvertKeyOperation" : { + "additionalProperties" : false, + "description" : "An operation to convert the stream key type to another type. Conversion is only syntactic, eg. from Avro to XML.", + "properties" : { + "into" : { + "description" : "The type to convert the stream key into", + "type" : "string" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "convertKey" ] + } + }, + "required" : [ "into", "type" ], + "title" : "ConvertKeyOperation", + "type" : "object" + }, + "ConvertKeyValueOperation" : { + "additionalProperties" : false, + "description" : "An operation to convert the stream key and value types to other types. Conversion is only syntactic, eg. from Avro to XML.", + "properties" : { + "into" : { + "description" : "The tuple type to convert the stream key/value into", + "type" : "string" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "convertKeyValue" ] + } + }, + "required" : [ "into", "type" ], + "title" : "ConvertKeyValueOperation", + "type" : "object" + }, + "ConvertValueOperation" : { + "additionalProperties" : false, + "description" : "An operation to convert the stream value type to another type. Conversion is only syntactic, eg. from Avro to XML.", + "properties" : { + "into" : { + "description" : "The type to convert the stream value into", + "type" : "string" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "convertValue" ] + } + }, + "required" : [ "into", "type" ], + "title" : "ConvertValueOperation", + "type" : "object" + }, + "CountOperation" : { + "additionalProperties" : false, + "description" : "Count the number of times a key is seen in a given window", + "properties" : { + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/SessionStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the count operation's result" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "count" ] + } + }, + "required" : [ "type" ], + "title" : "CountOperation", + "type" : "object" + }, + "FilterNotOperation" : { + "additionalProperties" : false, + "description" : "Filter records based on the inverse result of a predicate function", + "properties" : { + "if" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/PredicateDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that returns \"false\" when records are accepted, \"true\" otherwise" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the filtered table (only applies to tables, ignored for streams)" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "filterNot" ] + } + }, + "required" : [ "if", "type" ], + "title" : "FilterNotOperation", + "type" : "object" + }, + "FilterOperation" : { + "additionalProperties" : false, + "description" : "Filter records based on a predicate function", + "properties" : { + "if" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/PredicateDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that returns \"true\" when records are accepted, \"false\" otherwise" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the filtered table (only applies to tables, ignored for streams)" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "filter" ] + } + }, + "required" : [ "if", "type" ], + "title" : "FilterOperation", + "type" : "object" + }, + "ForEachActionDefinition" : { + "additionalProperties" : false, + "description" : "Defines a foreach action function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the foreach action" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the foreach action. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the foreach action. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the foreach action. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the foreach action", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the foreach action. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the foreach action uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "forEach" ] + } + }, + "required" : [ "type" ], + "title" : "ForEachActionDefinition", + "type" : "object" + }, + "ForEachActionDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a foreach action function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the foreach action" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the foreach action. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the foreach action. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the foreach action. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the foreach action", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the foreach action. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the foreach action uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + } + }, + "title" : "ForEachActionDefinitionWithImplicitType", + "type" : "object" + }, + "ForeignKeyExtractorDefinition" : { + "additionalProperties" : false, + "description" : "Defines a foreign key extractor function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the foreign key extractor" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the foreign key extractor. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the foreign key extractor. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the foreign key extractor. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the foreign key extractor", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the foreign key extractor. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "foreignKeyExtractor" ] + } + }, + "required" : [ "type" ], + "title" : "ForeignKeyExtractorDefinition", + "type" : "object" + }, + "ForeignKeyExtractorDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a foreign key extractor function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the foreign key extractor" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the foreign key extractor. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the foreign key extractor. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the foreign key extractor. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the foreign key extractor", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the foreign key extractor. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "ForeignKeyExtractorDefinitionWithImplicitType", + "type" : "object" + }, + "GeneratorDefinition" : { + "additionalProperties" : false, + "description" : "Defines a message generator function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the message generator" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the message generator. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the message generator. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the message generator. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the message generator", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the message generator. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "generator" ] + } + }, + "required" : [ "type" ], + "title" : "GeneratorDefinition", + "type" : "object" + }, + "GeneratorDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a message generator function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the message generator" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the message generator. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the message generator. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the message generator. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the message generator", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the message generator. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "GeneratorDefinitionWithImplicitType", + "type" : "object" + }, + "GenericFunctionDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a generic function function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the generic function" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the generic function. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the generic function. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the generic function. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the generic function", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the generic function. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "generic" ] + } + }, + "title" : "GenericFunctionDefinitionWithImplicitType", + "type" : "object" + }, + "GlobalTableDefinition" : { + "additionalProperties" : false, + "description" : "Contains a definition of a GlobalTable, which can be referenced by producers and pipelines", + "properties" : { + "keyType" : { + "description" : "*(optional)* The key type of the global table", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* KeyValue state store definition" + }, + "topic" : { + "description" : "The name of the Kafka topic for this global table", + "type" : "string" + }, + "valueType" : { + "description" : "*(optional)* The value type of the global table", + "type" : "string" + } + }, + "required" : [ "topic" ], + "title" : "GlobalTableDefinition", + "type" : "object" + }, + "GlobalTableDefinitionSource" : { + "additionalProperties" : false, + "description" : "Contains a definition of a GlobalTable, which can be referenced by producers and pipelines", + "properties" : { + "keyType" : { + "description" : "The key type of the global table", + "type" : "string" + }, + "offsetResetPolicy" : { + "description" : "*(optional)* Policy that determines what to do when there is no initial offset in Kafka, or if the current offset does not exist any more on the server (e.g. because that data has been deleted)", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* KeyValue state store definition" + }, + "timestampExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TimestampExtractorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function extracts the event time from a consumed record" + }, + "topic" : { + "description" : "The name of the Kafka topic for this global table", + "type" : "string" + }, + "valueType" : { + "description" : "The value type of the global table", + "type" : "string" + } + }, + "required" : [ "topic", "keyType", "valueType" ], + "title" : "GlobalTableDefinitionSource", + "type" : "object" + }, + "GroupByKeyOperation" : { + "additionalProperties" : false, + "description" : "Operation to group all messages with the same key together", + "properties" : { + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the grouped stream" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "groupByKey" ] + } + }, + "required" : [ "type" ], + "title" : "GroupByKeyOperation", + "type" : "object" + }, + "GroupByOperation" : { + "additionalProperties" : false, + "description" : "Operation to group all messages with together based on a keying function", + "properties" : { + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueMapperDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "Function to map records to a key they can be grouped on" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the grouped stream or table" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "groupBy" ] + } + }, + "required" : [ "mapper", "type" ], + "title" : "GroupByOperation", + "type" : "object" + }, + "InitializerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a initializer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the initializer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the initializer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the initializer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the initializer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the initializer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the initializer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "initializer" ] + } + }, + "required" : [ "type" ], + "title" : "InitializerDefinition", + "type" : "object" + }, + "InitializerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a initializer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the initializer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the initializer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the initializer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the initializer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the initializer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the initializer. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "InitializerDefinitionWithImplicitType", + "type" : "object" + }, + "JoinOperationWithGlobalTable" : { + "additionalProperties" : false, + "description" : "Operation to join with a table", + "properties" : { + "globalTable" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/GlobalTableDefinition", + "type" : "object" + } ], + "description" : "A reference to the globalTable, or an inline definition of the globalTable to join with" + }, + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueMapperDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that maps the key value from the stream to the primary key type of the globalTable" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "join" ] + }, + "valueJoiner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueJoinerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that joins two values" + } + }, + "required" : [ "globalTable", "mapper", "valueJoiner", "type" ], + "title" : "JoinOperationWithGlobalTable", + "type" : "object" + }, + "JoinOperationWithStream" : { + "additionalProperties" : false, + "description" : "Operation to join with a stream", + "properties" : { + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/SessionStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the joined streams" + }, + "stream" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamDefinition", + "type" : "object" + } ], + "description" : "A reference to the Stream, or an inline definition of the stream to join with" + }, + "timeDifference" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The maximum time difference for a join over two streams on the same key" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "join" ] + }, + "valueJoiner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueJoinerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that joins two values" + } + }, + "required" : [ "stream", "valueJoiner", "timeDifference", "type" ], + "title" : "JoinOperationWithStream", + "type" : "object" + }, + "JoinOperationWithTable" : { + "additionalProperties" : false, + "description" : "Operation to join with a table", + "properties" : { + "foreignKeyExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ForeignKeyExtractorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that can translate the join table value to a primary key" + }, + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "otherPartitioner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamPartitionerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that partitions the records on the join table" + }, + "partitioner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamPartitionerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that partitions the records on the primary table" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/SessionStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the joined streams" + }, + "table" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TableDefinition", + "type" : "object" + } ], + "description" : "A reference to the table, or an inline definition of the table to join with" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "join" ] + }, + "valueJoiner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueJoinerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that joins two values" + } + }, + "required" : [ "table", "valueJoiner", "type" ], + "title" : "JoinOperationWithTable", + "type" : "object" + }, + "KeyTransformerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a key transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the key transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the key transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the key transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the key transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the key transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the key transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the key transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "keyTransformer" ] + } + }, + "required" : [ "type" ], + "title" : "KeyTransformerDefinition", + "type" : "object" + }, + "KeyTransformerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a key transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the key transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the key transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the key transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the key transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the key transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the key transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the key transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + } + }, + "title" : "KeyTransformerDefinitionWithImplicitType", + "type" : "object" + }, + "KeyValueMapperDefinition" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue mapper function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue mapper" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue mapper. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue mapper. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue mapper. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue mapper", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue mapper. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "keyValueMapper" ] + } + }, + "required" : [ "type" ], + "title" : "KeyValueMapperDefinition", + "type" : "object" + }, + "KeyValueMapperDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue mapper function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue mapper" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue mapper. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue mapper. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue mapper. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue mapper", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue mapper. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "KeyValueMapperDefinitionWithImplicitType", + "type" : "object" + }, + "KeyValuePrinterDefinition" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue printer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue printer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue printer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue printer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue printer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue printer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue printer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "keyValuePrinter" ] + } + }, + "required" : [ "type" ], + "title" : "KeyValuePrinterDefinition", + "type" : "object" + }, + "KeyValuePrinterDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue printer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue printer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue printer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue printer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue printer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue printer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue printer. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "KeyValuePrinterDefinitionWithImplicitType", + "type" : "object" + }, + "KeyValueStateStoreDefinition" : { + "additionalProperties" : false, + "description" : "Definition of a keyValue state store", + "properties" : { + "caching" : { + "description" : "*(optional)* \"true\" if changed to the keyValue store need to be buffered and periodically released, \"false\" to emit all changes directly", + "type" : "boolean" + }, + "historyRetention" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* (Versioned only) The duration for which old record versions are available for query (cannot be negative)" + }, + "keyType" : { + "description" : "*(optional)* The key type of the keyValue store", + "type" : "string" + }, + "logging" : { + "description" : "*(optional)* \"true\" if a changelog topic should be set up on Kafka for this keyValue store, \"false\" otherwise", + "type" : "boolean" + }, + "name" : { + "description" : "*(optional)* The name of the keyValue store. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "persistent" : { + "description" : "*(optional)* \"true\" if this keyValue store needs to be stored on disk, \"false\" otherwise", + "type" : "boolean" + }, + "segmentInterval" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Size of segments for storing old record versions (must be positive). Old record versions for the same key in a single segment are stored (updated and accessed) together. The only impact of this parameter is performance. If segments are large and a workload results in many record versions for the same key being collected in a single segment, performance may degrade as a result. On the other hand, historical reads (which access older segments) and out-of-order writes may slow down if there are too many segments." + }, + "timestamped" : { + "description" : "*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise", + "type" : "boolean" + }, + "type" : { + "description" : "The type of the state store", + "enum" : [ "keyValue" ] + }, + "valueType" : { + "description" : "*(optional)* The value type of the keyValue store", + "type" : "string" + }, + "versioned" : { + "description" : "*(optional)* \"true\" if elements in the store are versioned, \"false\" otherwise", + "type" : "boolean" + } + }, + "required" : [ "type" ], + "title" : "KeyValueStateStoreDefinition", + "type" : "object" + }, + "KeyValueStateStoreDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Definition of a keyValue state store", + "properties" : { + "caching" : { + "description" : "*(optional)* \"true\" if changed to the keyValue store need to be buffered and periodically released, \"false\" to emit all changes directly", + "type" : "boolean" + }, + "historyRetention" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* (Versioned only) The duration for which old record versions are available for query (cannot be negative)" + }, + "keyType" : { + "description" : "*(optional)* The key type of the keyValue store", + "type" : "string" + }, + "logging" : { + "description" : "*(optional)* \"true\" if a changelog topic should be set up on Kafka for this keyValue store, \"false\" otherwise", + "type" : "boolean" + }, + "name" : { + "description" : "*(optional)* The name of the keyValue store. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "persistent" : { + "description" : "*(optional)* \"true\" if this keyValue store needs to be stored on disk, \"false\" otherwise", + "type" : "boolean" + }, + "segmentInterval" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Size of segments for storing old record versions (must be positive). Old record versions for the same key in a single segment are stored (updated and accessed) together. The only impact of this parameter is performance. If segments are large and a workload results in many record versions for the same key being collected in a single segment, performance may degrade as a result. On the other hand, historical reads (which access older segments) and out-of-order writes may slow down if there are too many segments." + }, + "timestamped" : { + "description" : "*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise", + "type" : "boolean" + }, + "type" : { + "description" : "The type of the state store", + "enum" : [ "keyValue" ] + }, + "valueType" : { + "description" : "*(optional)* The value type of the keyValue store", + "type" : "string" + }, + "versioned" : { + "description" : "*(optional)* \"true\" if elements in the store are versioned, \"false\" otherwise", + "type" : "boolean" + } + }, + "title" : "KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + }, + "KeyValueToKeyValueListTransformerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue-to-keyvaluelist transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue-to-keyvaluelist transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue-to-keyvaluelist transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue-to-keyvaluelist transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue-to-keyvaluelist transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue-to-keyvaluelist transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue-to-keyvaluelist transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the keyvalue-to-keyvaluelist transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "keyValueToKeyValueListTransformer" ] + } + }, + "required" : [ "type" ], + "title" : "KeyValueToKeyValueListTransformerDefinition", + "type" : "object" + }, + "KeyValueToKeyValueListTransformerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue-to-keyvaluelist transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue-to-keyvaluelist transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue-to-keyvaluelist transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue-to-keyvaluelist transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue-to-keyvaluelist transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue-to-keyvaluelist transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue-to-keyvaluelist transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the keyvalue-to-keyvaluelist transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + } + }, + "title" : "KeyValueToKeyValueListTransformerDefinitionWithImplicitType", + "type" : "object" + }, + "KeyValueToValueListTransformerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue-to-valuelist transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue-to-valuelist transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue-to-valuelist transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue-to-valuelist transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue-to-valuelist transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue-to-valuelist transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue-to-valuelist transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the keyvalue-to-valuelist transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "keyValueToValueListTransformer" ] + } + }, + "required" : [ "type" ], + "title" : "KeyValueToValueListTransformerDefinition", + "type" : "object" + }, + "KeyValueToValueListTransformerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue-to-valuelist transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue-to-valuelist transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue-to-valuelist transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue-to-valuelist transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue-to-valuelist transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue-to-valuelist transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue-to-valuelist transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the keyvalue-to-valuelist transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + } + }, + "title" : "KeyValueToValueListTransformerDefinitionWithImplicitType", + "type" : "object" + }, + "KeyValueTransformerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the keyvalue transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "keyValueTransformer" ] + } + }, + "required" : [ "type" ], + "title" : "KeyValueTransformerDefinition", + "type" : "object" + }, + "KeyValueTransformerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a keyvalue transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the keyvalue transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the keyvalue transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the keyvalue transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the keyvalue transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the keyvalue transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the keyvalue transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the keyvalue transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + } + }, + "title" : "KeyValueTransformerDefinitionWithImplicitType", + "type" : "object" + }, + "LeftJoinOperationWithGlobalTable" : { + "additionalProperties" : false, + "description" : "Operation to leftJoin with a globalTable", + "properties" : { + "globalTable" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/GlobalTableDefinition", + "type" : "object" + } ], + "description" : "A reference to the globalTable, or an inline definition of the globalTable to join with" + }, + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueMapperDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that maps the key value from the stream with the primary key of the globalTable" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "leftJoin" ] + }, + "valueJoiner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueJoinerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that joins two values" + } + }, + "required" : [ "globalTable", "mapper", "valueJoiner", "type" ], + "title" : "LeftJoinOperationWithGlobalTable", + "type" : "object" + }, + "LeftJoinOperationWithStream" : { + "additionalProperties" : false, + "description" : "Operation to leftJoin with a stream", + "properties" : { + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/SessionStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the joined streams" + }, + "stream" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamDefinition", + "type" : "object" + } ], + "description" : "A reference to the stream, or an inline definition of the stream to leftJoin with" + }, + "timeDifference" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The maximum time difference for a leftJoin over two streams on the same key" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "leftJoin" ] + }, + "valueJoiner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueJoinerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that joins two values" + } + }, + "required" : [ "stream", "valueJoiner", "timeDifference", "type" ], + "title" : "LeftJoinOperationWithStream", + "type" : "object" + }, + "LeftJoinOperationWithTable" : { + "additionalProperties" : false, + "description" : "Operation to leftJoin with a table", + "properties" : { + "foreignKeyExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ForeignKeyExtractorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that can translate the join table value to a primary key" + }, + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "otherPartitioner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamPartitionerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that partitions the records on the join table" + }, + "partitioner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamPartitionerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that partitions the records on the primary table" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/SessionStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the joined streams" + }, + "table" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TableDefinition", + "type" : "object" + } ], + "description" : "A reference to the Table, or an inline definition of the Table to join with" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "leftJoin" ] + }, + "valueJoiner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueJoinerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that joins two values" + } + }, + "required" : [ "table", "valueJoiner", "type" ], + "title" : "LeftJoinOperationWithTable", + "type" : "object" + }, + "MergeOperation" : { + "additionalProperties" : false, + "description" : "A merge operation to join two Streams", + "properties" : { + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "stream" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamDefinitionSource", + "type" : "object" + } ], + "description" : "The stream to merge with" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "merge" ] + } + }, + "required" : [ "stream", "type" ], + "title" : "MergeOperation", + "type" : "object" + }, + "MergerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a merger function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the merger" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the merger. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the merger. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the merger. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the merger", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the merger. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "merger" ] + } + }, + "required" : [ "type" ], + "title" : "MergerDefinition", + "type" : "object" + }, + "MergerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a merger function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the merger" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the merger. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the merger. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the merger. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the merger", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the merger. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "MergerDefinitionWithImplicitType", + "type" : "object" + }, + "MetadataTransformerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a metadata transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the metadata transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the metadata transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the metadata transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the metadata transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the metadata transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the metadata transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the metadata transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "metadataTransformer" ] + } + }, + "required" : [ "type" ], + "title" : "MetadataTransformerDefinition", + "type" : "object" + }, + "MetadataTransformerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a metadata transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the metadata transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the metadata transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the metadata transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the metadata transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the metadata transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the metadata transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the metadata transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + } + }, + "title" : "MetadataTransformerDefinitionWithImplicitType", + "type" : "object" + }, + "OuterJoinOperationWithStream" : { + "additionalProperties" : false, + "description" : "Operation to outerJoin with a stream", + "properties" : { + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The window grace period (the time to admit out-of-order events after the end of the window)" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/SessionStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the outerJoined streams" + }, + "stream" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamDefinition", + "type" : "object" + } ], + "description" : "A reference to the stream, or an inline definition of the stream to outerJoin with" + }, + "timeDifference" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The maximum time difference for an outerJoin over two streams on the same key" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "outerJoin" ] + }, + "valueJoiner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueJoinerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that joins two values" + } + }, + "required" : [ "stream", "valueJoiner", "timeDifference", "type" ], + "title" : "OuterJoinOperationWithStream", + "type" : "object" + }, + "OuterJoinOperationWithTable" : { + "additionalProperties" : false, + "description" : "Operation to outerJoin with a table", + "properties" : { + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/SessionStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the outerJoined streams" + }, + "table" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TableDefinition", + "type" : "object" + } ], + "description" : "A reference to the table, or an inline definition of the table to outerJoin with" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "outerJoin" ] + }, + "valueJoiner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueJoinerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that joins two values" + } + }, + "required" : [ "table", "valueJoiner", "type" ], + "title" : "OuterJoinOperationWithTable", + "type" : "object" + }, + "ParameterDefinition" : { + "additionalProperties" : false, + "description" : "Defines a parameter for a user function", + "properties" : { + "defaultValue" : { + "description" : "*(optional)* The default value for the parameter", + "type" : "string" + }, + "name" : { + "description" : "The name of the parameter", + "type" : "string" + }, + "type" : { + "description" : "The type of the parameter", + "type" : "string" + } + }, + "required" : [ "name", "type" ], + "title" : "ParameterDefinition", + "type" : "object" + }, + "PeekOperation" : { + "additionalProperties" : false, + "description" : "Operation to peek into a stream, without modifying the stream contents", + "properties" : { + "forEach" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ForEachActionDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that gets called for every message in the stream" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "peek" ] + } + }, + "required" : [ "forEach", "type" ], + "title" : "PeekOperation", + "type" : "object" + }, + "PipelineDefinition" : { + "additionalProperties" : false, + "description" : "Defines a pipeline through a source, a series of operations to perform on it and a sink operation to close the stream with", + "properties" : { + "as" : { + "description" : "*(optional)* The name to register the pipeline result under, which can be used as source by follow-up pipelines", + "type" : "string" + }, + "branch" : { + "description" : "*(optional)* Defines a single branch, consisting of a condition and a pipeline to execute for messages that fulfil the predicate", + "items" : { + "$ref" : "#/definitions/BranchDefinitionWithPipeline", + "type" : "object" + }, + "type" : "array" + }, + "forEach" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ForEachActionDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that gets called for every message in the stream" + }, + "from" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TopicDefinitionSource", + "type" : "object" + } ], + "description" : "Pipeline source" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "print" : { + "$ref" : "#/definitions/PrintOperation", + "description" : "*(optional)* The specification of where to print messages to", + "type" : "object" + }, + "to" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ToTopicDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Ends the pipeline by sending all messages to a stream, table or globalTable, or to an inline defined output topic and optional partitioner" + }, + "toTopicNameExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ToTopicNameExtractorDefinition", + "type" : "object" + } ], + "description" : "*(optional)* Ends the pipeline by sending all messages to a topic provided by a pre-defined topic name extractor function, or to a topic provided by an inline defined topic name extractor and optional partitioner" + }, + "via" : { + "description" : "*(optional)* A series of operations performed on the input stream", + "items" : { + "anyOf" : [ { + "$ref" : "#/definitions/AggregateOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/CogroupOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ConvertKeyOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ConvertKeyValueOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ConvertValueOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/CountOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/FilterNotOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/FilterOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/GroupByKeyOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/GroupByOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/JoinOperationWithGlobalTable", + "type" : "object" + }, { + "$ref" : "#/definitions/JoinOperationWithStream", + "type" : "object" + }, { + "$ref" : "#/definitions/JoinOperationWithTable", + "type" : "object" + }, { + "$ref" : "#/definitions/LeftJoinOperationWithGlobalTable", + "type" : "object" + }, { + "$ref" : "#/definitions/LeftJoinOperationWithStream", + "type" : "object" + }, { + "$ref" : "#/definitions/LeftJoinOperationWithTable", + "type" : "object" + }, { + "$ref" : "#/definitions/MergeOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/OuterJoinOperationWithStream", + "type" : "object" + }, { + "$ref" : "#/definitions/OuterJoinOperationWithTable", + "type" : "object" + }, { + "$ref" : "#/definitions/PeekOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ReduceOperationWithAdderAndSubtractor", + "type" : "object" + }, { + "$ref" : "#/definitions/ReduceOperationWithReducer", + "type" : "object" + }, { + "$ref" : "#/definitions/RepartitionOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/SuppressOperationUntilTimeLimit", + "type" : "object" + }, { + "$ref" : "#/definitions/SuppressOperationUntilWindowCloses", + "type" : "object" + }, { + "$ref" : "#/definitions/ToStreamOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/ToTableOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformKeyOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformKeyValueOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformKeyValueToKeyValueListOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformKeyValueToValueListOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformMetadataOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/TransformValueOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowBySessionOperation", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowByTimeOperationWithHoppingWindow", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowByTimeOperationWithSlidingWindow", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowByTimeOperationWithTumblingWindow", + "type" : "object" + } ] + }, + "type" : "array" + } + }, + "required" : [ "from" ], + "title" : "PipelineDefinition", + "type" : "object" + }, + "PredicateDefinition" : { + "additionalProperties" : false, + "description" : "Defines a predicate function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the predicate" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the predicate. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the predicate. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the predicate. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the predicate", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the predicate. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the predicate uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "predicate" ] + } + }, + "required" : [ "type" ], + "title" : "PredicateDefinition", + "type" : "object" + }, + "PredicateDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a predicate function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the predicate" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the predicate. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the predicate. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the predicate. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the predicate", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the predicate. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the predicate uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + } + }, + "title" : "PredicateDefinitionWithImplicitType", + "type" : "object" + }, + "PrintOperation" : { + "additionalProperties" : false, + "description" : "Operation to print the contents of a pipeline on the screen or to write them to a file", + "properties" : { + "filename" : { + "description" : "*(optional)* The filename to output records to. If nothing is specified, then messages will be printed on stdout.", + "type" : "string" + }, + "label" : { + "description" : "*(optional)* A label to attach to the output records", + "type" : "string" + }, + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValuePrinterDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function to convert record into a string for output" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + } + }, + "title" : "PrintOperation", + "type" : "object" + }, + "ProducerDefinition" : { + "additionalProperties" : false, + "description" : "Definition of a Producer that regularly generates messages for a topic", + "properties" : { + "batchSize" : { + "description" : "*(optional)* The size of batches", + "type" : "number" + }, + "condition" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/PredicateDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that validates the generator's result message. Returns \"true\" when the message may be produced on the topic, \"false\" otherwise." + }, + "count" : { + "description" : "*(optional)* The number of messages to produce.", + "type" : "number" + }, + "generator" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/GeneratorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "The function that generates records" + }, + "interval" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The interval with which the generator is called" + }, + "to" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TopicDefinition", + "type" : "object" + } ], + "description" : "The topic to produce to" + }, + "until" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/PredicateDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A predicate that returns true to indicate producing should stop." + } + }, + "required" : [ "generator", "to" ], + "title" : "ProducerDefinition", + "type" : "object" + }, + "ReduceOperationWithAdderAndSubtractor" : { + "additionalProperties" : false, + "description" : "Operation to reduce a series of records into a single aggregate result", + "properties" : { + "adder" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ReducerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that adds a record to the aggregate result" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the aggregation" + }, + "subtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ReducerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that removes a record from the aggregate result" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "reduce" ] + } + }, + "required" : [ "adder", "subtractor", "type" ], + "title" : "ReduceOperationWithAdderAndSubtractor", + "type" : "object" + }, + "ReduceOperationWithReducer" : { + "additionalProperties" : false, + "description" : "Operation to reduce a series of records into a single aggregate result", + "properties" : { + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "reducer" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ReducerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that computes a new aggregate result" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the aggregation" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "reduce" ] + } + }, + "required" : [ "reducer", "type" ], + "title" : "ReduceOperationWithReducer", + "type" : "object" + }, + "ReducerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a reducer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the reducer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the reducer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the reducer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the reducer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the reducer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the reducer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "reducer" ] + } + }, + "required" : [ "type" ], + "title" : "ReducerDefinition", + "type" : "object" + }, + "ReducerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a reducer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the reducer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the reducer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the reducer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the reducer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the reducer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the reducer. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "ReducerDefinitionWithImplicitType", + "type" : "object" + }, + "RepartitionOperation" : { + "additionalProperties" : false, + "description" : "Operation to (re)partition a stream", + "properties" : { + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "numberOfPartitions" : { + "description" : "*(optional)* The target number of partitions" + }, + "partitioner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamPartitionerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that partitions stream records" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "repartition" ] + } + }, + "required" : [ "type" ], + "title" : "RepartitionOperation", + "type" : "object" + }, + "SessionStateStoreDefinition" : { + "additionalProperties" : false, + "description" : "Definition of a session state store", + "properties" : { + "caching" : { + "description" : "*(optional)* \"true\" if changed to the session store need to be buffered and periodically released, \"false\" to emit all changes directly", + "type" : "boolean" + }, + "keyType" : { + "description" : "*(optional)* The key type of the session store", + "type" : "string" + }, + "logging" : { + "description" : "*(optional)* \"true\" if a changelog topic should be set up on Kafka for this session store, \"false\" otherwise", + "type" : "boolean" + }, + "name" : { + "description" : "*(optional)* The name of the session store. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "persistent" : { + "description" : "*(optional)* \"true\" if this session store needs to be stored on disk, \"false\" otherwise", + "type" : "boolean" + }, + "retention" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The duration for which elements in the session store are retained" + }, + "timestamped" : { + "description" : "*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise", + "type" : "boolean" + }, + "type" : { + "description" : "The type of the state store", + "enum" : [ "session" ] + }, + "valueType" : { + "description" : "*(optional)* The value type of the session store", + "type" : "string" + } + }, + "required" : [ "type" ], + "title" : "SessionStateStoreDefinition", + "type" : "object" + }, + "StreamDefinition" : { + "additionalProperties" : false, + "description" : "Contains a definition of a Stream, which can be referenced by producers and pipelines", + "properties" : { + "keyType" : { + "description" : "*(optional)* The key type of the stream", + "type" : "string" + }, + "topic" : { + "description" : "The name of the Kafka topic for this stream", + "type" : "string" + }, + "valueType" : { + "description" : "*(optional)* The value type of the stream", + "type" : "string" + } + }, + "required" : [ "topic" ], + "title" : "StreamDefinition", + "type" : "object" + }, + "StreamDefinitionSource" : { + "additionalProperties" : false, + "description" : "Contains a definition of a Stream, which can be referenced by producers and pipelines", + "properties" : { + "keyType" : { + "description" : "The key type of the stream", + "type" : "string" + }, + "offsetResetPolicy" : { + "description" : "*(optional)* Policy that determines what to do when there is no initial offset in Kafka, or if the current offset does not exist any more on the server (e.g. because that data has been deleted)", + "type" : "string" + }, + "timestampExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TimestampExtractorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function extracts the event time from a consumed record" + }, + "topic" : { + "description" : "The name of the Kafka topic for this stream", + "type" : "string" + }, + "valueType" : { + "description" : "The value type of the stream", + "type" : "string" + } + }, + "required" : [ "topic", "keyType", "valueType" ], + "title" : "StreamDefinitionSource", + "type" : "object" + }, + "StreamPartitionerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a stream partitioner function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the stream partitioner" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the stream partitioner. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the stream partitioner. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the stream partitioner. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the stream partitioner", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the stream partitioner. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "streamPartitioner" ] + } + }, + "required" : [ "type" ], + "title" : "StreamPartitionerDefinition", + "type" : "object" + }, + "StreamPartitionerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a stream partitioner function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the stream partitioner" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the stream partitioner. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the stream partitioner. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the stream partitioner. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the stream partitioner", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the stream partitioner. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "StreamPartitionerDefinitionWithImplicitType", + "type" : "object" + }, + "StringOrInlinePredicateDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines the condition under which messages get sent down this branch", + "properties" : { + "if" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/PredicateDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Defines the condition under which messages get sent down this branch" + } + }, + "title" : "StringOrInlinePredicateDefinitionWithImplicitType", + "type" : "object" + }, + "SuppressOperationUntilTimeLimit" : { + "additionalProperties" : false, + "description" : "Operation to suppress messages in the source stream until a time limit is reached", + "properties" : { + "bufferFullStrategy" : { + "description" : "*(optional)* What to do when the buffer is full", + "enum" : [ "emitEarlyWhenFull", "shutdownWhenFull" ] + }, + "duration" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The duration for which messages are suppressed" + }, + "maxBytes" : { + "description" : "*(optional)* The maximum number of bytes in the buffer", + "type" : "string" + }, + "maxRecords" : { + "description" : "*(optional)* The maximum number of records in the buffer", + "type" : "string" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "suppress" ] + }, + "until" : { + "description" : "The until of the Operation to suppress messages in the source stream until a certain limit is reached", + "enum" : [ "timeLimit" ] + } + }, + "required" : [ "duration", "until", "type" ], + "title" : "SuppressOperationUntilTimeLimit", + "type" : "object" + }, + "SuppressOperationUntilWindowCloses" : { + "additionalProperties" : false, + "description" : "Operation to suppress messages in the source stream until a window limit is reached", + "properties" : { + "bufferFullStrategy" : { + "description" : "*(optional)* What to do when the buffer is full", + "enum" : [ "emitEarlyWhenFull", "shutdownWhenFull" ] + }, + "maxBytes" : { + "description" : "*(optional)* The maximum number of bytes in the buffer", + "type" : "string" + }, + "maxRecords" : { + "description" : "*(optional)* The maximum number of records in the buffer", + "type" : "string" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "suppress" ] + }, + "until" : { + "description" : "The until of the Operation to suppress messages in the source stream until a certain limit is reached", + "enum" : [ "windowCloses" ] + } + }, + "required" : [ "until", "type" ], + "title" : "SuppressOperationUntilWindowCloses", + "type" : "object" + }, + "TableDefinition" : { + "additionalProperties" : false, + "description" : "Contains a definition of a Table, which can be referenced by producers and pipelines", + "properties" : { + "keyType" : { + "description" : "*(optional)* The key type of the table", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* KeyValue state store definition" + }, + "topic" : { + "description" : "The name of the Kafka topic for this table", + "type" : "string" + }, + "valueType" : { + "description" : "*(optional)* The value type of the table", + "type" : "string" + } + }, + "required" : [ "topic" ], + "title" : "TableDefinition", + "type" : "object" + }, + "TableDefinitionSource" : { + "additionalProperties" : false, + "description" : "Contains a definition of a Table, which can be referenced by producers and pipelines", + "properties" : { + "keyType" : { + "description" : "The key type of the table", + "type" : "string" + }, + "offsetResetPolicy" : { + "description" : "*(optional)* Policy that determines what to do when there is no initial offset in Kafka, or if the current offset does not exist any more on the server (e.g. because that data has been deleted)", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* KeyValue state store definition" + }, + "timestampExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TimestampExtractorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function extracts the event time from a consumed record" + }, + "topic" : { + "description" : "The name of the Kafka topic for this table", + "type" : "string" + }, + "valueType" : { + "description" : "The value type of the table", + "type" : "string" + } + }, + "required" : [ "topic", "keyType", "valueType" ], + "title" : "TableDefinitionSource", + "type" : "object" + }, + "TimestampExtractorDefinition" : { + "additionalProperties" : false, + "description" : "Defines a timestamp extractor function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the timestamp extractor" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the timestamp extractor. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the timestamp extractor. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the timestamp extractor. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the timestamp extractor", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the timestamp extractor. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "timestampExtractor" ] + } + }, + "required" : [ "type" ], + "title" : "TimestampExtractorDefinition", + "type" : "object" + }, + "TimestampExtractorDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a timestamp extractor function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the timestamp extractor" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the timestamp extractor. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the timestamp extractor. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the timestamp extractor. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the timestamp extractor", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the timestamp extractor. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "TimestampExtractorDefinitionWithImplicitType", + "type" : "object" + }, + "ToStreamOperation" : { + "additionalProperties" : false, + "description" : "Convert a Table into a Stream, optionally through a custom key transformer", + "properties" : { + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyTransformerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that computes the output key for every record" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "toStream" ] + } + }, + "required" : [ "type" ], + "title" : "ToStreamOperation", + "type" : "object" + }, + "ToTableOperation" : { + "additionalProperties" : false, + "description" : "Convert a Stream into a Table", + "properties" : { + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the result table" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "toTable" ] + } + }, + "required" : [ "type" ], + "title" : "ToTableOperation", + "type" : "object" + }, + "ToTopicDefinition" : { + "additionalProperties" : false, + "description" : "Writes out pipeline messages to a topic", + "properties" : { + "keyType" : { + "description" : "*(optional)* The key type of the topic", + "type" : "string" + }, + "partitioner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamPartitionerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that partitions the records in the output topic" + }, + "topic" : { + "description" : "The name of the Kafka topic", + "type" : "string" + }, + "valueType" : { + "description" : "*(optional)* The value type of the topic", + "type" : "string" + } + }, + "required" : [ "topic" ], + "title" : "ToTopicDefinition", + "type" : "object" + }, + "ToTopicNameExtractorDefinition" : { + "additionalProperties" : false, + "description" : "Writes out pipeline messages to a topic as given by a topic name extractor", + "properties" : { + "partitioner" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/StreamPartitionerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function that partitions the records in the output topic" + }, + "topicNameExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TopicNameExtractorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "Reference to a pre-defined topic name extractor, or an inline definition of a topic name extractor" + } + }, + "required" : [ "topicNameExtractor" ], + "title" : "ToTopicNameExtractorDefinition", + "type" : "object" + }, + "TopicDefinition" : { + "additionalProperties" : false, + "description" : "Contains a definition of a Kafka topic, to be used by producers and pipelines", + "properties" : { + "keyType" : { + "description" : "*(optional)* The key type of the topic", + "type" : "string" + }, + "topic" : { + "description" : "The name of the Kafka topic", + "type" : "string" + }, + "valueType" : { + "description" : "*(optional)* The value type of the topic", + "type" : "string" + } + }, + "required" : [ "topic" ], + "title" : "TopicDefinition", + "type" : "object" + }, + "TopicDefinitionSource" : { + "additionalProperties" : false, + "description" : "Contains a definition of a Kafka topic, to be used by producers and pipelines", + "properties" : { + "keyType" : { + "description" : "The key type of the topic", + "type" : "string" + }, + "offsetResetPolicy" : { + "description" : "*(optional)* Policy that determines what to do when there is no initial offset in Kafka, or if the current offset does not exist any more on the server (e.g. because that data has been deleted)", + "type" : "string" + }, + "timestampExtractor" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/TimestampExtractorDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* A function extracts the event time from a consumed record" + }, + "topic" : { + "description" : "The name of the Kafka topic", + "type" : "string" + }, + "valueType" : { + "description" : "The value type of the topic", + "type" : "string" + } + }, + "required" : [ "topic", "keyType", "valueType" ], + "title" : "TopicDefinitionSource", + "type" : "object" + }, + "TopicNameExtractorDefinition" : { + "additionalProperties" : false, + "description" : "Defines a topic name extractor function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the topic name extractor" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the topic name extractor. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the topic name extractor. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the topic name extractor. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the topic name extractor", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the topic name extractor. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "topicNameExtractor" ] + } + }, + "required" : [ "type" ], + "title" : "TopicNameExtractorDefinition", + "type" : "object" + }, + "TopicNameExtractorDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a topic name extractor function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the topic name extractor" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the topic name extractor. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the topic name extractor. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the topic name extractor. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the topic name extractor", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the topic name extractor. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "TopicNameExtractorDefinitionWithImplicitType", + "type" : "object" + }, + "TransformKeyOperation" : { + "additionalProperties" : false, + "description" : "Convert the key of every record in the stream to another key", + "properties" : { + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyTransformerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that computes a new key for each record" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "transformKey", "mapKey", "selectKey" ] + } + }, + "required" : [ "mapper", "type" ], + "title" : "TransformKeyOperation", + "type" : "object" + }, + "TransformKeyValueOperation" : { + "additionalProperties" : false, + "description" : "Convert the key/value of every record in the stream to another key/value", + "properties" : { + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueTransformerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that computes a new key/value for each record" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "mapKeyValue", "map", "transformKeyValue" ] + } + }, + "required" : [ "mapper", "type" ], + "title" : "TransformKeyValueOperation", + "type" : "object" + }, + "TransformKeyValueToKeyValueListOperation" : { + "additionalProperties" : false, + "description" : "Convert a stream by transforming every record into a list of derived records", + "properties" : { + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueToKeyValueListTransformerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that converts every record of a stream to a list of output records." + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "transformKeyValueToKeyValueList", "flatMap" ] + } + }, + "required" : [ "mapper", "type" ], + "title" : "TransformKeyValueToKeyValueListOperation", + "type" : "object" + }, + "TransformKeyValueToValueListOperation" : { + "additionalProperties" : false, + "description" : "Convert every record in the stream to a list of output records with the same key", + "properties" : { + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueToValueListTransformerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that converts every key/value into a list of result values, each of which will be combined with the original key to form a new message in the output stream" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "transformKeyValueToValueList", "flatMapValues" ] + } + }, + "required" : [ "mapper", "type" ], + "title" : "TransformKeyValueToValueListOperation", + "type" : "object" + }, + "TransformMetadataOperation" : { + "additionalProperties" : false, + "description" : "Convert the metadata of every record in the stream", + "properties" : { + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/MetadataTransformerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that converts the metadata (Kafka headers, timestamp) of every record in the stream" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "transformMetadata" ] + } + }, + "required" : [ "mapper", "type" ], + "title" : "TransformMetadataOperation", + "type" : "object" + }, + "TransformValueOperation" : { + "additionalProperties" : false, + "description" : "Convert the value of every record in the stream to another value", + "properties" : { + "mapper" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/ValueTransformerDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "A function that converts the value of every record into another value" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "store" : { + "anyOf" : [ { + "type" : "string" + }, { + "$ref" : "#/definitions/KeyValueStateStoreDefinitionWithImplicitType", + "type" : "object" + } ], + "description" : "*(optional)* Materialized view of the transformed table (only applies to tables, ignored for streams)" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "mapValue", "transformValue", "mapValues" ] + } + }, + "required" : [ "mapper", "type" ], + "title" : "TransformValueOperation", + "type" : "object" + }, + "ValueJoinerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a value joiner function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the value joiner" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the value joiner. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the value joiner. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the value joiner. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the value joiner", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the value joiner. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "valueJoiner" ] + } + }, + "required" : [ "type" ], + "title" : "ValueJoinerDefinition", + "type" : "object" + }, + "ValueJoinerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a value joiner function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the value joiner" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the value joiner. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the value joiner. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the value joiner. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the value joiner", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the value joiner. Only required for function types, which are not pre-defined.", + "type" : "string" + } + }, + "title" : "ValueJoinerDefinitionWithImplicitType", + "type" : "object" + }, + "ValueTransformerDefinition" : { + "additionalProperties" : false, + "description" : "Defines a value transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the value transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the value transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the value transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the value transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the value transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the value transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the value transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + }, + "type" : { + "description" : "The type of the function", + "enum" : [ "valueTransformer" ] + } + }, + "required" : [ "type" ], + "title" : "ValueTransformerDefinition", + "type" : "object" + }, + "ValueTransformerDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Defines a value transformer function, that gets injected into the Kafka Streams topology", + "properties" : { + "code" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The (multiline) code of the value transformer" + }, + "expression" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The expression returned by the value transformer. Only required for functions that return values." + }, + "globalCode" : { + "anyOf" : [ { + "type" : "boolean" + }, { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Global (multiline) code that gets loaded into the Python context outside of the value transformer. Can be used for defining eg. global variables." + }, + "name" : { + "description" : "*(optional)* The name of the value transformer. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "parameters" : { + "description" : "*(optional)* A list of parameters to be passed into the value transformer", + "items" : { + "$ref" : "#/definitions/ParameterDefinition", + "type" : "object" + }, + "type" : "array" + }, + "resultType" : { + "description" : "*(optional)* The data type returned by the value transformer. Only required for function types, which are not pre-defined.", + "type" : "string" + }, + "stores" : { + "description" : "*(optional)* A list of store names that the value transformer uses. Only required if the function wants to use a state store.", + "items" : { + "type" : "string" + }, + "type" : "array" + } + }, + "title" : "ValueTransformerDefinitionWithImplicitType", + "type" : "object" + }, + "WindowBySessionOperation" : { + "additionalProperties" : false, + "description" : "Operation to window messages by session, configured by an inactivity gap", + "properties" : { + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* (Tumbling, Hopping) The grace period, during which out-of-order records can still be processed" + }, + "inactivityGap" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The inactivity gap, below which two messages are considered to be of the same session" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "windowBySession" ] + } + }, + "required" : [ "inactivityGap", "type" ], + "title" : "WindowBySessionOperation", + "type" : "object" + }, + "WindowByTimeOperationWithHoppingWindow" : { + "additionalProperties" : false, + "description" : "Operation to window records based on time criteria", + "properties" : { + "advanceBy" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The amount of time to increase time windows by" + }, + "duration" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The duration of time windows" + }, + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The grace period, during which out-of-order records can still be processed" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "windowByTime" ] + }, + "windowType" : { + "description" : "The windowType of the time window", + "enum" : [ "hopping" ] + } + }, + "required" : [ "duration", "advanceBy", "windowType", "type" ], + "title" : "WindowByTimeOperationWithHoppingWindow", + "type" : "object" + }, + "WindowByTimeOperationWithSlidingWindow" : { + "additionalProperties" : false, + "description" : "Operation to window records based on time criteria", + "properties" : { + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The grace period, during which out-of-order records can still be processed" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "timeDifference" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The maximum amount of time difference between two records" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "windowByTime" ] + }, + "windowType" : { + "description" : "The windowType of the time window", + "enum" : [ "sliding" ] + } + }, + "required" : [ "timeDifference", "windowType", "type" ], + "title" : "WindowByTimeOperationWithSlidingWindow", + "type" : "object" + }, + "WindowByTimeOperationWithTumblingWindow" : { + "additionalProperties" : false, + "description" : "Operation to window records based on time criteria", + "properties" : { + "duration" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "The duration of time windows" + }, + "grace" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The grace period, during which out-of-order records can still be processed" + }, + "name" : { + "description" : "*(optional)* The name of the operation processor", + "type" : "string" + }, + "type" : { + "description" : "The type of the operation", + "enum" : [ "windowByTime" ] + }, + "windowType" : { + "description" : "The windowType of the time window", + "enum" : [ "tumbling" ] + } + }, + "required" : [ "duration", "windowType", "type" ], + "title" : "WindowByTimeOperationWithTumblingWindow", + "type" : "object" + }, + "WindowStateStoreDefinition" : { + "additionalProperties" : false, + "description" : "Definition of a window state store", + "properties" : { + "caching" : { + "description" : "*(optional)* \"true\" if changed to the window store need to be buffered and periodically released, \"false\" to emit all changes directly", + "type" : "boolean" + }, + "keyType" : { + "description" : "*(optional)* The key type of the window store", + "type" : "string" + }, + "logging" : { + "description" : "*(optional)* \"true\" if a changelog topic should be set up on Kafka for this window store, \"false\" otherwise", + "type" : "boolean" + }, + "name" : { + "description" : "*(optional)* The name of the window store. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "persistent" : { + "description" : "*(optional)* \"true\" if this window store needs to be stored on disk, \"false\" otherwise", + "type" : "boolean" + }, + "retainDuplicates" : { + "description" : "*(optional)* Whether or not to retain duplicates", + "type" : "boolean" + }, + "retention" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The duration for which elements in the window store are retained" + }, + "timestamped" : { + "description" : "*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise", + "type" : "boolean" + }, + "type" : { + "description" : "The type of the state store", + "enum" : [ "window" ] + }, + "valueType" : { + "description" : "*(optional)* The value type of the window store", + "type" : "string" + }, + "windowSize" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Size of the windows (cannot be negative)" + } + }, + "required" : [ "type" ], + "title" : "WindowStateStoreDefinition", + "type" : "object" + }, + "WindowStateStoreDefinitionWithImplicitType" : { + "additionalProperties" : false, + "description" : "Definition of a window state store", + "properties" : { + "caching" : { + "description" : "*(optional)* \"true\" if changed to the window store need to be buffered and periodically released, \"false\" to emit all changes directly", + "type" : "boolean" + }, + "keyType" : { + "description" : "*(optional)* The key type of the window store", + "type" : "string" + }, + "logging" : { + "description" : "*(optional)* \"true\" if a changelog topic should be set up on Kafka for this window store, \"false\" otherwise", + "type" : "boolean" + }, + "name" : { + "description" : "*(optional)* The name of the window store. If this field is not defined, then the name is derived from the context.", + "type" : "string" + }, + "persistent" : { + "description" : "*(optional)* \"true\" if this window store needs to be stored on disk, \"false\" otherwise", + "type" : "boolean" + }, + "retainDuplicates" : { + "description" : "*(optional)* Whether or not to retain duplicates", + "type" : "boolean" + }, + "retention" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* The duration for which elements in the window store are retained" + }, + "timestamped" : { + "description" : "*(optional)* \"true\" if elements in the store are timestamped, \"false\" otherwise", + "type" : "boolean" + }, + "type" : { + "description" : "The type of the state store", + "enum" : [ "window" ] + }, + "valueType" : { + "description" : "*(optional)* The value type of the window store", + "type" : "string" + }, + "windowSize" : { + "anyOf" : [ { + "type" : "number" + }, { + "type" : "string" + } ], + "description" : "*(optional)* Size of the windows (cannot be negative)" + } + }, + "title" : "WindowStateStoreDefinitionWithImplicitType", + "type" : "object" + } + }, + "description" : "KSML definition", + "properties" : { + "functions" : { + "description" : "*(optional)* Functions that can be referenced in producers and pipelines", + "patternProperties" : { + "^[a-zA-Z0-9_]+$" : { + "anyOf" : [ { + "$ref" : "#/definitions/AggregatorDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/ForEachActionDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/ForeignKeyExtractorDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/GeneratorDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/GenericFunctionDefinitionWithImplicitType", + "type" : "object" + }, { + "$ref" : "#/definitions/InitializerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/KeyTransformerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/KeyValueMapperDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/KeyValuePrinterDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/KeyValueToKeyValueListTransformerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/KeyValueToValueListTransformerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/KeyValueTransformerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/MergerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/MetadataTransformerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/PredicateDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/ReducerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/StreamPartitionerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/TimestampExtractorDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/TopicNameExtractorDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/ValueJoinerDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/ValueTransformerDefinition", + "type" : "object" + } ] + } + }, + "type" : "object" + }, + "globalTables" : { + "description" : "*(optional)* GlobalTables that can be referenced in producers and pipelines", + "patternProperties" : { + "^[a-zA-Z0-9_]+$" : { + "$ref" : "#/definitions/GlobalTableDefinitionSource", + "type" : "object" + } + }, + "type" : "object" + }, + "pipelines" : { + "description" : "*(optional)* Collection of named pipelines", + "patternProperties" : { + "^[a-zA-Z0-9_]+$" : { + "$ref" : "#/definitions/PipelineDefinition", + "type" : "object" + } + }, + "type" : "object" + }, + "producers" : { + "description" : "*(optional)* Collection of named producers", + "patternProperties" : { + "^[a-zA-Z0-9_]+$" : { + "$ref" : "#/definitions/ProducerDefinition", + "type" : "object" + } + }, + "type" : "object" + }, + "stores" : { + "description" : "*(optional)* State stores that can be referenced in producers and pipelines", + "patternProperties" : { + "^[a-zA-Z0-9_]+$" : { + "anyOf" : [ { + "$ref" : "#/definitions/KeyValueStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/SessionStateStoreDefinition", + "type" : "object" + }, { + "$ref" : "#/definitions/WindowStateStoreDefinition", + "type" : "object" + } ] + } + }, + "type" : "object" + }, + "streams" : { + "description" : "*(optional)* Streams that can be referenced in producers and pipelines", + "patternProperties" : { + "^[a-zA-Z0-9_]+$" : { + "$ref" : "#/definitions/StreamDefinitionSource", + "type" : "object" + } + }, + "type" : "object" + }, + "tables" : { + "description" : "*(optional)* Tables that can be referenced in producers and pipelines", + "patternProperties" : { + "^[a-zA-Z0-9_]+$" : { + "$ref" : "#/definitions/TableDefinitionSource", + "type" : "object" + } + }, + "type" : "object" + } + }, + "title" : "TopologyDefinition", + "type" : "object" +} diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/NotationLibrary.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/NotationLibrary.java index 35a4e45b..b0ce0e91 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/NotationLibrary.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/NotationLibrary.java @@ -39,7 +39,7 @@ public static boolean exists(String notation) { return notations.containsKey(notation); } - public static Notation notation(String notation) { + public static Notation get(String notation) { var result = notation != null ? notations.get(notation) : null; if (result != null) return result; throw new DataException("Data notation is not registered in the NotationLibrary: " + (notation != null ? notation : "null")); diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonDataObjectConverter.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonDataObjectConverter.java index 726cdc0c..d3def555 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonDataObjectConverter.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonDataObjectConverter.java @@ -31,7 +31,7 @@ import io.axual.ksml.data.type.UnionType; public class JsonDataObjectConverter implements NotationConverter { - private static final JsonDataObjectMapper DATA_OBJECT_MAPPER = new JsonDataObjectMapper(); + private static final JsonDataObjectMapper DATA_OBJECT_MAPPER = new JsonDataObjectMapper(false); @Override public DataObject convert(DataObject value, UserType targetType) { diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonDataObjectMapper.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonDataObjectMapper.java index bf880467..81a9ea96 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonDataObjectMapper.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonDataObjectMapper.java @@ -28,17 +28,21 @@ public class JsonDataObjectMapper implements DataObjectMapper { private static final NativeDataObjectMapper NATIVE_MAPPER = new NativeDataObjectMapper(); - private static final StringMapper STRING_MAPPER = new JsonStringMapper(); + private final StringMapper stringMapper; + + public JsonDataObjectMapper(boolean prettyPrint) { + stringMapper = new JsonStringMapper(prettyPrint); + } @Override public DataObject toDataObject(DataType expected, String value) { - var object = STRING_MAPPER.fromString(value); + var object = stringMapper.fromString(value); return NATIVE_MAPPER.toDataObject(expected, object); } @Override public String fromDataObject(DataObject value) { var object = NATIVE_MAPPER.fromDataObject(value); - return STRING_MAPPER.toString(object); + return stringMapper.toString(object); } } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonSchemaLoader.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonSchemaLoader.java index 8e68bad3..e068d51e 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonSchemaLoader.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonSchemaLoader.java @@ -26,7 +26,7 @@ @Slf4j public class JsonSchemaLoader extends SchemaLoader { - private static final JsonSchemaMapper MAPPER = new JsonSchemaMapper(); + private static final JsonSchemaMapper MAPPER = new JsonSchemaMapper(false); public JsonSchemaLoader(String schemaDirectory) { super("JSON", schemaDirectory, ".json"); diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonSchemaMapper.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonSchemaMapper.java index 70830f7d..fbd39a43 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonSchemaMapper.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonSchemaMapper.java @@ -35,7 +35,6 @@ import static io.axual.ksml.data.schema.DataField.NO_INDEX; public class JsonSchemaMapper implements DataSchemaMapper { - private static final JsonDataObjectMapper MAPPER = new JsonDataObjectMapper(); private static final String TITLE_NAME = "title"; private static final String DESCRIPTION_NAME = "description"; private static final String TYPE_NAME = "type"; @@ -55,11 +54,16 @@ public class JsonSchemaMapper implements DataSchemaMapper { private static final String NUMBER_TYPE = "number"; private static final String OBJECT_TYPE = "object"; private static final String STRING_TYPE = "string"; + private final JsonDataObjectMapper mapper; + + public JsonSchemaMapper(boolean prettyPrint) { + mapper = new JsonDataObjectMapper(prettyPrint); + } @Override public DataSchema toDataSchema(String namespace, String name, String value) { // Convert JSON to internal DataObject format - var schema = MAPPER.toDataObject(value); + var schema = mapper.toDataObject(value); if (schema instanceof DataStruct schemaStruct) { return toDataSchema(namespace, name, schemaStruct); } @@ -135,7 +139,7 @@ public String fromDataSchema(DataSchema schema) { final var result = fromDataSchema(structSchema); // First translate the schema into DataObjects // The use the mapper to convert it into JSON - return MAPPER.fromDataObject(result); + return mapper.fromDataObject(result); } return null; } diff --git a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonStringMapper.java b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonStringMapper.java index 920db9e9..aa52406b 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonStringMapper.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/notation/json/JsonStringMapper.java @@ -29,8 +29,13 @@ import java.io.StringWriter; public class JsonStringMapper implements StringMapper { - protected final ObjectMapper mapper = new ObjectMapper(); private static final JsonNodeNativeMapper NATIVE_MAPPER = new JsonNodeNativeMapper(); + protected final ObjectMapper mapper = new ObjectMapper(); + private final boolean prettyPrint; + + public JsonStringMapper(boolean prettyPrint) { + this.prettyPrint = prettyPrint; + } @Override public Object fromString(String value) { @@ -48,7 +53,10 @@ public String toString(Object value) { if (value == null) return null; // Allow null as native input, return null string as output try { final var writer = new StringWriter(); - mapper.writeTree(mapper.createGenerator(writer), NATIVE_MAPPER.fromNative(value)); + final var generator = prettyPrint + ? mapper.writerWithDefaultPrettyPrinter().createGenerator(writer) + : mapper.createGenerator(writer); + mapper.writeTree(generator, NATIVE_MAPPER.fromNative(value)); return writer.toString(); } catch (IOException e) { throw new DataException("Can not convert object to JSON string: " + value, e); diff --git a/ksml-data/src/main/java/io/axual/ksml/data/schema/SchemaLibrary.java b/ksml-data/src/main/java/io/axual/ksml/data/schema/SchemaLibrary.java index 13053948..8fa438b6 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/schema/SchemaLibrary.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/schema/SchemaLibrary.java @@ -60,7 +60,7 @@ public static DataSchema getSchema(String notationName, String schemaName, boole if (schema != null) return schema; } - final var notation = NotationLibrary.notation(notationName); + final var notation = NotationLibrary.get(notationName); if (notation.loader() == null) return null; var schema = notation.loader().load(schemaName); diff --git a/ksml-data/src/main/java/io/axual/ksml/data/serde/JsonSerde.java b/ksml-data/src/main/java/io/axual/ksml/data/serde/JsonSerde.java index 9613ef4b..28d79c92 100644 --- a/ksml-data/src/main/java/io/axual/ksml/data/serde/JsonSerde.java +++ b/ksml-data/src/main/java/io/axual/ksml/data/serde/JsonSerde.java @@ -26,7 +26,7 @@ import io.axual.ksml.data.type.DataType; public class JsonSerde extends StringSerde { - private static final DataObjectMapper STRING_MAPPER = new JsonDataObjectMapper(); + private static final DataObjectMapper STRING_MAPPER = new JsonDataObjectMapper(false); public JsonSerde(NativeDataObjectMapper nativeMapper, DataType expectedType) { super(nativeMapper, STRING_MAPPER, expectedType); diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java index f8621644..27538e11 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java @@ -329,7 +329,7 @@ private static void checkForSchemaOutput(String[] args) { // KSML definitions with on stdout and exit if (args.length >= 1 && WRITE_KSML_SCHEMA_ARGUMENT.equals(args[0])) { final var parser = new TopologyDefinitionParser("dummy"); - final var schema = new JsonSchemaMapper().fromDataSchema(parser.schema()); + final var schema = new JsonSchemaMapper(true).fromDataSchema(parser.schema()); final var filename = args.length >= 2 ? args[1] : null; if (filename != null) { diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/AlwaysReschedule.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/AlwaysReschedule.java deleted file mode 100644 index c12f1192..00000000 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/AlwaysReschedule.java +++ /dev/null @@ -1,56 +0,0 @@ -package io.axual.ksml.runner.backend; - -/*- - * ========================LICENSE_START================================= - * KSML Runner - * %% - * Copyright (C) 2021 - 2024 Axual B.V. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * =========================LICENSE_END================================== - */ - -import io.axual.ksml.data.object.DataObject; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; - -public class AlwaysReschedule implements RescheduleStrategy { - - private final List others = new ArrayList<>(); - - private final Duration interval; - - public AlwaysReschedule(Duration interval) { - this.interval = interval; - } - - public void combine(RescheduleStrategy other) { - others.add(other); - } - - @Override - public boolean shouldReschedule(DataObject key, DataObject value) { - var reschedule = true; - for (var other: others) { - reschedule = reschedule && other.shouldReschedule(key, value); - } - return reschedule; - } - - @Override - public Duration interval() { - return interval; - } -} diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/CountingReschedule.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/CountingReschedule.java deleted file mode 100644 index e69addcf..00000000 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/CountingReschedule.java +++ /dev/null @@ -1,41 +0,0 @@ -package io.axual.ksml.runner.backend; - -/*- - * ========================LICENSE_START================================= - * KSML Runner - * %% - * Copyright (C) 2021 - 2024 Axual B.V. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * =========================LICENSE_END================================== - */ - -import io.axual.ksml.data.object.DataObject; - -public class CountingReschedule implements RescheduleStrategy { - - private int count; - - public CountingReschedule(int count) { - this.count = count; - } - - @Override - public boolean shouldReschedule(DataObject key, DataObject value) { - if (count > 0) { - count--; - return true; - } - return false; - } -} diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaProducerRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaProducerRunner.java index 74f512a8..df783141 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaProducerRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/KafkaProducerRunner.java @@ -20,6 +20,14 @@ * =========================LICENSE_END================================== */ +import io.axual.ksml.client.producer.ResolvingProducer; +import io.axual.ksml.generator.TopologyDefinition; +import io.axual.ksml.python.PythonContext; +import io.axual.ksml.python.PythonFunction; +import io.axual.ksml.runner.exception.RunnerException; +import io.axual.ksml.runner.producer.ExecutableProducer; +import io.axual.ksml.runner.producer.IntervalSchedule; +import lombok.Builder; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.common.serialization.ByteArraySerializer; import org.slf4j.Logger; @@ -29,13 +37,6 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; -import io.axual.ksml.client.producer.ResolvingProducer; -import io.axual.ksml.generator.TopologyDefinition; -import io.axual.ksml.python.PythonContext; -import io.axual.ksml.python.PythonFunction; -import io.axual.ksml.runner.exception.RunnerException; -import lombok.Builder; - import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; @@ -95,9 +96,12 @@ public void run() { while (!stopRunning.get() && !hasFailed.get() && scheduler.hasScheduledItems()) { var scheduledGenerator = scheduler.getScheduledItem(); if (scheduledGenerator != null) { - scheduledGenerator.producer().produceMessage(producer); + scheduledGenerator.producer().produceMessages(producer); if (scheduledGenerator.producer().shouldReschedule()) { - final long nextTime = scheduledGenerator.startTime() + scheduledGenerator.producer().interval().toMillis(); + final var interval = scheduledGenerator.producer().interval() != null + ? scheduledGenerator.producer().interval().toMillis() + : 0L; + final long nextTime = scheduledGenerator.startTime() + interval; scheduler.schedule(scheduledGenerator.producer(), nextTime); } } diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/RescheduleStrategy.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/RescheduleStrategy.java deleted file mode 100644 index 4f5c972c..00000000 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/RescheduleStrategy.java +++ /dev/null @@ -1,63 +0,0 @@ -package io.axual.ksml.runner.backend; - -/*- - * ========================LICENSE_START================================= - * KSML Runner - * %% - * Copyright (C) 2021 - 2024 Axual B.V. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * =========================LICENSE_END================================== - */ - -import io.axual.ksml.data.object.DataObject; -import io.axual.ksml.user.UserFunction; - -import java.time.Duration; - -/** - * Interface to indicate if an {@link ExecutableProducer} should be rescheduled. - */ -public interface RescheduleStrategy { - - /** - * Implementations indicate if the producer should be scheduled again. - * @return - */ - boolean shouldReschedule(DataObject key, DataObject value); - - /** - * The only implementation keeping track of interval is {@link AlwaysReschedule}; all others just return {@link Duration#ZERO}. - * @return the interval until the next reschedule. - */ - default Duration interval() { - return Duration.ZERO; - } - - static AlwaysReschedule always(Duration interval) { - return new AlwaysReschedule(interval); - } - - static RescheduleStrategy once() { - return new SingleShotReschedule(); - } - - static RescheduleStrategy counting(int count) { - return new CountingReschedule(count); - } - - static RescheduleStrategy until(UserFunction userFunction) { - return new UntilReschedule(userFunction); - } - -} diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/SingleShotReschedule.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/SingleShotReschedule.java deleted file mode 100644 index bf4c998f..00000000 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/SingleShotReschedule.java +++ /dev/null @@ -1,34 +0,0 @@ -package io.axual.ksml.runner.backend; - -/*- - * ========================LICENSE_START================================= - * KSML Runner - * %% - * Copyright (C) 2021 - 2024 Axual B.V. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * =========================LICENSE_END================================== - */ - -import io.axual.ksml.data.object.DataObject; - -/** - * Reschedule strategy which runs the function only once. - */ -public class SingleShotReschedule implements RescheduleStrategy { - - @Override - public boolean shouldReschedule(DataObject key, DataObject value) { - return false; - } -} diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/UntilReschedule.java b/ksml-runner/src/main/java/io/axual/ksml/runner/backend/UntilReschedule.java deleted file mode 100644 index 1d410e00..00000000 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/UntilReschedule.java +++ /dev/null @@ -1,43 +0,0 @@ -package io.axual.ksml.runner.backend; - -/*- - * ========================LICENSE_START================================= - * KSML Runner - * %% - * Copyright (C) 2021 - 2024 Axual B.V. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * =========================LICENSE_END================================== - */ - -import io.axual.ksml.data.object.DataBoolean; -import io.axual.ksml.data.object.DataObject; -import io.axual.ksml.user.UserFunction; - -public class UntilReschedule implements RescheduleStrategy { - - private final UserFunction condition; - - public UntilReschedule(UserFunction condition) { - this.condition = condition; - } - - @Override - public boolean shouldReschedule(DataObject key, DataObject value) { - DataObject result = condition.call(key, value); - - // Note: "until" should NOT reschedule if the predicate became true, negate the result! - if (result instanceof DataBoolean resultBoolean) return !(resultBoolean.value()); - throw new io.axual.ksml.data.exception.ExecutionException("Producer condition did not return a boolean value: " + condition.name); - } -} diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/ErrorHandlingConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/ErrorHandlingConfig.java index dc3590bb..02dbb70b 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/ErrorHandlingConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/ErrorHandlingConfig.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; @JsonIgnoreProperties(ignoreUnknown = true) @@ -71,8 +72,11 @@ private ErrorTypeHandlingConfig defaultErrorTypeHandlingConfig(String logger) { @Data public static class ErrorTypeHandlingConfig { private boolean log = true; + @JsonProperty("logPayload") private boolean logPayload = false; + @JsonProperty("loggerName") private String loggerName; + @JsonProperty("handler") private Handler handler = Handler.STOP; public enum Handler { diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java index f8792f52..97f599c4 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/config/KSMLConfig.java @@ -53,7 +53,7 @@ public class KSMLConfig { @JsonProperty("applicationServer") @Builder.Default - private ApplicationServerConfig applicationServer = DEFAULT_APPSERVER_CONFIG; + private ApplicationServerConfig applicationServerConfig = DEFAULT_APPSERVER_CONFIG; @JsonProperty("prometheus") @Builder.Default @Getter @@ -98,7 +98,7 @@ public String storageDirectory() { } public ApplicationServerConfig applicationServerConfig() { - return applicationServer; + return applicationServerConfig; } public ErrorHandlingConfig errorHandlingConfig() { diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/ExecutableProducer.java b/ksml-runner/src/main/java/io/axual/ksml/runner/producer/ExecutableProducer.java similarity index 52% rename from ksml-runner/src/main/java/io/axual/ksml/runner/backend/ExecutableProducer.java rename to ksml-runner/src/main/java/io/axual/ksml/runner/producer/ExecutableProducer.java index 64803d41..9177842b 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/ExecutableProducer.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/producer/ExecutableProducer.java @@ -1,4 +1,4 @@ -package io.axual.ksml.runner.backend; +package io.axual.ksml.runner.producer; /*- * ========================LICENSE_START================================= @@ -20,33 +20,34 @@ * =========================LICENSE_END================================== */ -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.header.internals.RecordHeaders; -import org.apache.kafka.common.serialization.Serializer; - -import java.time.Duration; -import java.util.Map; -import java.util.concurrent.ExecutionException; - import io.axual.ksml.client.serde.ResolvingSerializer; import io.axual.ksml.data.mapper.DataObjectConverter; import io.axual.ksml.data.mapper.NativeDataObjectMapper; import io.axual.ksml.data.notation.NotationLibrary; import io.axual.ksml.data.notation.UserType; -import io.axual.ksml.data.object.DataNull; import io.axual.ksml.data.object.DataObject; import io.axual.ksml.data.object.DataTuple; import io.axual.ksml.data.tag.ContextTags; -import io.axual.ksml.definition.FunctionDefinition; +import io.axual.ksml.data.value.Pair; import io.axual.ksml.definition.ProducerDefinition; +import io.axual.ksml.exception.TopologyException; import io.axual.ksml.python.PythonContext; import io.axual.ksml.python.PythonFunction; import io.axual.ksml.user.UserFunction; import io.axual.ksml.user.UserGenerator; -import io.axual.ksml.user.UserPredicate; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.serialization.Serializer; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; import static io.axual.ksml.data.notation.UserType.DEFAULT_NOTATION; @@ -58,35 +59,31 @@ public class ExecutableProducer { @Getter private final String name; private final UserGenerator generator; - private final UserPredicate condition; + private final ProducerStrategy producerStrategy; private final String topic; private final UserType keyType; private final UserType valueType; private final Serializer keySerializer; private final Serializer valueSerializer; - private final RescheduleStrategy rescheduleStrategy; - - private DataObject lastKey = DataNull.INSTANCE; - private DataObject lastValue = DataNull.INSTANCE; + private long batchCount = 0; + private boolean stopProducing = false; private ExecutableProducer(UserFunction generator, - UserFunction condition, + ProducerStrategy producerStrategy, ContextTags tags, String topic, UserType keyType, UserType valueType, Serializer keySerializer, - Serializer valueSerializer, - RescheduleStrategy rescheduleStrategy) { + Serializer valueSerializer) { this.name = generator.name; this.generator = new UserGenerator(generator, tags); - this.condition = condition != null ? new UserPredicate(condition, tags) : null; + this.producerStrategy = producerStrategy; this.topic = topic; this.keyType = keyType; this.valueType = valueType; this.keySerializer = keySerializer; this.valueSerializer = valueSerializer; - this.rescheduleStrategy = rescheduleStrategy; } /** @@ -101,94 +98,137 @@ private ExecutableProducer(UserFunction generator, */ public static ExecutableProducer forProducer(PythonContext context, String namespace, String name, ProducerDefinition producerDefinition, Map kafkaConfig) { final var target = producerDefinition.target(); - final var gen = producerDefinition.generator(); final var tags = new ContextTags().append("namespace", namespace); + + // Initialize the message generator + final var gen = producerDefinition.generator(); + if (gen == null) { + throw new TopologyException("Missing generator function for producer \"" + name + "\""); + } final var generator = gen.name() != null ? PythonFunction.forGenerator(context, namespace, gen.name(), gen) : PythonFunction.forGenerator(context, namespace, name, gen); - final var cond = producerDefinition.condition(); - final var condition = cond != null - ? cond.name() != null - ? PythonFunction.forPredicate(context, namespace, cond.name(), cond) - : PythonFunction.forPredicate(context, namespace, name, cond) - : null; - final var keySerde = NotationLibrary.notation(target.keyType().notation()).serde(target.keyType().dataType(), true); + + // Initialize the producer strategy + final var producerStrategy = new ProducerStrategy(context, namespace, name, tags, producerDefinition); + + // Initialize serializers + final var keySerde = NotationLibrary.get(target.keyType().notation()).serde(target.keyType().dataType(), true); final var keySerializer = new ResolvingSerializer<>(keySerde.serializer(), kafkaConfig); - final var valueSerde = NotationLibrary.notation(target.valueType().notation()).serde(target.valueType().dataType(), false); + final var valueSerde = NotationLibrary.get(target.valueType().notation()).serde(target.valueType().dataType(), false); final var valueSerializer = new ResolvingSerializer<>(valueSerde.serializer(), kafkaConfig); - final var reschedulingStrategy = setupRescheduling(producerDefinition, context, namespace, name); - return new ExecutableProducer(generator, condition, tags, target.topic(), target.keyType(), target.valueType(), keySerializer, valueSerializer, reschedulingStrategy); + // Set up the producer + return new ExecutableProducer(generator, producerStrategy, tags, target.topic(), target.keyType(), target.valueType(), keySerializer, valueSerializer); + } + + public void produceMessages(Producer producer) { + final var messages = generateBatch(); + final var futures = new ArrayList>(); + try { + for (var message : messages) { + ProducerRecord record = new ProducerRecord<>( + topic, + message.left(), + message.right() + ); + futures.add(producer.send(record)); + } + + batchCount++; + + for (var future : futures) { + var metadata = future.get(); + if (metadata != null && metadata.hasOffset()) { + producerStrategy.successfullyProducedOneMessage(); + log.info("Produced message: producer={}, batch #{}, message #{}, topic={}, partition={}, offset={}", name, batchCount, producerStrategy.messagesProduced(), metadata.topic(), metadata.partition(), metadata.offset()); + } else { + log.error("Error producing message to topic {}", topic); + } + } + } catch (InterruptedException | ExecutionException e) { + throw new io.axual.ksml.data.exception.ExecutionException("Could not produce to topic " + topic, e); + } } - public void produceMessage(Producer producer) { + private List> generateBatch() { + final var result = new ArrayList>(); + for (int index = 0; index < producerStrategy.batchSize(); index++) { + Pair message = null; + for (int t = 0; t < 10; t++) { + message = generateMessage(); + if (message != null) break; + } + + if (message != null) { + final var key = message.left(); + final var value = message.right(); + + // Log the generated messages + final var keyStr = key != null ? key.toString(DataObject.Printer.EXTERNAL_TOP_SCHEMA).replace("\n", "\\\\n") : "null"; + final var valueStr = value != null ? value.toString(DataObject.Printer.EXTERNAL_TOP_SCHEMA).replace("\n", "\\\\n") : "null"; + log.info("Message: key={}, value={}", keyStr, valueStr); + + // Serialize the message + var serializedKey = keySerializer.serialize(topic, NATIVE_MAPPER.fromDataObject(key)); + var serializedValue = valueSerializer.serialize(topic, NATIVE_MAPPER.fromDataObject(value)); + + // Add the serialized message to the batch + result.add(new Pair<>(serializedKey, serializedValue)); + + // Check if this should be the last message produced + if (!producerStrategy.continueAfterMessage(key, value)) { + stopProducing = true; + break; + } + } else { + log.warn("Could not generate a valid message after 10 tries, skipping..."); + } + } + + // Return the batch of messages + return result; + } + + private Pair generateMessage() { DataObject result = generator.apply(); if (result instanceof DataTuple tuple && tuple.size() == 2) { var key = tuple.get(0); var value = tuple.get(1); - if (condition == null || condition.test(key, value)) { + if (producerStrategy.validateMessage(key, value)) { // keep produced key and value to determine rescheduling later - lastKey = key; - lastValue = value; - key = DATA_OBJECT_CONVERTER.convert(DEFAULT_NOTATION, key, keyType); value = DATA_OBJECT_CONVERTER.convert(DEFAULT_NOTATION, value, valueType); + var okay = true; if (key != null && !keyType.dataType().isAssignableFrom(key.type())) { - log.error("Can not convert {} to topic key type {}", key.type(), keyType); + log.error("Wrong topic key type: expected={} key={}", keyType, key.type()); okay = false; } if (value != null && !valueType.dataType().isAssignableFrom(value.type())) { - log.error("Can not convert {} to topic value type {}", value.type(), valueType); + log.error("Wrong topic value type: expected={} value={}", valueType, value.type()); okay = false; } if (okay) { - var keyStr = key != null ? key.toString() : "null"; - var valueStr = value != null ? value.toString() : "null"; - - keyStr = keyStr.replace("\n", "\\\\n"); - valueStr = valueStr.replace("\n", "\\\\n"); - log.info("Message: key={}, value={}", keyStr, valueStr); - - // Headers are sometimes needed for Apicurio serialisation - var headers = new RecordHeaders(); - var serializedKey = keySerializer.serialize(topic, headers, NATIVE_MAPPER.fromDataObject(key)); - var serializedValue = valueSerializer.serialize(topic, headers, NATIVE_MAPPER.fromDataObject(value)); - ProducerRecord message = new ProducerRecord<>( - topic, - null, - serializedKey, - serializedValue, - headers - ); - var future = producer.send(message); - try { - var metadata = future.get(); - if (metadata != null && metadata.hasOffset()) { - log.info("Produced message to {}, partition {}, offset {}", metadata.topic(), metadata.partition(), metadata.offset()); - } else { - log.error("Error producing message to topic {}", topic); - } - } catch (InterruptedException | ExecutionException e) { - throw new io.axual.ksml.data.exception.ExecutionException("Could not produce to topic " + topic, e); - } + return new Pair<>(key, value); } } else { - log.info("Condition FALSE, skipping message"); + log.warn("Skipping invalid message: key={} value={}", key, value); } } + return null; } /** - * Indicate if this instance wants to be rescheduled after its most recent run. + * Indicate if this producer wants to be rescheduled after its most recent run. * * @return true if should reschedule. */ public boolean shouldReschedule() { - return rescheduleStrategy.shouldReschedule(lastKey, lastValue); + return !stopProducing && producerStrategy.shouldReschedule(); } /** @@ -197,29 +237,6 @@ public boolean shouldReschedule() { * @return the desired wait until next run. */ public Duration interval() { - return rescheduleStrategy.interval(); - } - - private static RescheduleStrategy setupRescheduling(ProducerDefinition definition, PythonContext context, String namespace, String name) { - if (definition.interval() == null) { - return RescheduleStrategy.once(); - } - - AlwaysReschedule strategy = RescheduleStrategy.always(definition.interval()); - - if (definition.count() != null) { - // since the producer is always called first before calling the strategy, decrement count by 1 - strategy.combine(RescheduleStrategy.counting(definition.count() - 1)); - } - - if (definition.until() != null) { - FunctionDefinition untilDefinition = definition.until(); - final var untilFunction = untilDefinition.name() != null - ? PythonFunction.forPredicate(context, namespace, untilDefinition.name(), untilDefinition) - : PythonFunction.forPredicate(context, namespace, name, untilDefinition); - strategy.combine(RescheduleStrategy.until(untilFunction)); - } - - return strategy; + return producerStrategy.interval(); } } diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/IntervalSchedule.java b/ksml-runner/src/main/java/io/axual/ksml/runner/producer/IntervalSchedule.java similarity index 98% rename from ksml-runner/src/main/java/io/axual/ksml/runner/backend/IntervalSchedule.java rename to ksml-runner/src/main/java/io/axual/ksml/runner/producer/IntervalSchedule.java index 746345c4..87df0396 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/backend/IntervalSchedule.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/producer/IntervalSchedule.java @@ -1,4 +1,4 @@ -package io.axual.ksml.runner.backend; +package io.axual.ksml.runner.producer; /*- * ========================LICENSE_START================================= diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/producer/ProducerStrategy.java b/ksml-runner/src/main/java/io/axual/ksml/runner/producer/ProducerStrategy.java new file mode 100644 index 00000000..2ff552aa --- /dev/null +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/producer/ProducerStrategy.java @@ -0,0 +1,126 @@ +package io.axual.ksml.runner.producer; + +/*- + * ========================LICENSE_START================================= + * KSML Runner + * %% + * Copyright (C) 2021 - 2024 Axual B.V. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * =========================LICENSE_END================================== + */ + +import io.axual.ksml.data.object.DataObject; +import io.axual.ksml.data.tag.ContextTags; +import io.axual.ksml.definition.FunctionDefinition; +import io.axual.ksml.definition.ProducerDefinition; +import io.axual.ksml.python.PythonContext; +import io.axual.ksml.python.PythonFunction; +import io.axual.ksml.user.UserPredicate; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.time.Duration; + +@Slf4j +public class ProducerStrategy { + private static final int INFINITE = -1; + + private final boolean once; + private final long messageCount; + private final long batchSize; + @Getter + private final Duration interval; + private final UserPredicate condition; + private final UserPredicate until; + @Getter + private long messagesProduced = 0; + private boolean untilTriggered = false; + + public ProducerStrategy(PythonContext context, String namespace, String name, ContextTags tags, ProducerDefinition definition) { + // If interval, messageCount and until are not defined, then only produce one message + once = (definition.interval() == null && definition.messageCount() == null && definition.until() == null); + if (once) { + messageCount = 1; + batchSize = 1; + interval = Duration.ofMillis(0); + condition = null; + until = null; + return; + } + + // Set the message count + messageCount = definition.messageCount() != null + ? definition.messageCount() >= 1 + ? definition.messageCount() + : 1 + : INFINITE; + + // Set the batch size + if (definition.batchSize() != null) { + if (definition.batchSize() >= 1 && definition.batchSize() <= 1000) { + batchSize = definition.batchSize(); + } else { + log.warn("Batch size must be between 1 and 1000. Using 1 as default."); + batchSize = 1; + } + } else { + batchSize = 1; + } + + // Set the interval + interval = definition.interval() != null ? definition.interval() : Duration.ofMillis(0); + + // Set up the message validator + condition = userPredicateFrom(definition.condition(), context, namespace, name, tags); + + // Set up the user function that - when returns true - halts the producer + until = userPredicateFrom(definition.until(), context, namespace, name, tags); + } + + private UserPredicate userPredicateFrom(FunctionDefinition function, PythonContext context, String namespace, String name, ContextTags tags) { + return function != null + ? function.name() != null + ? new UserPredicate(PythonFunction.forPredicate(context, namespace, function.name(), function), tags) + : new UserPredicate(PythonFunction.forPredicate(context, namespace, name, function), tags) + : null; + } + + public boolean shouldReschedule() { + // Reschedule if not once, not all messages were produced yet and until never returned true + return !once && (messageCount == INFINITE || messagesProduced < messageCount) && !untilTriggered; + } + + public boolean validateMessage(DataObject key, DataObject value) { + return (condition == null || condition.test(key, value)); + } + + public boolean continueAfterMessage(DataObject key, DataObject value) { + if (until != null) { + if (until.test(key, value)) { + untilTriggered = true; + return false; + } + } + return true; + } + + public long batchSize() { + if (messageCount == INFINITE) return batchSize; + return Math.min(batchSize, messageCount - messagesProduced); + } + + public void successfullyProducedOneMessage() { + messagesProduced++; + } +} diff --git a/ksml-runner/src/test/java/io/axual/ksml/runner/backend/IntervalScheduleTest.java b/ksml-runner/src/test/java/io/axual/ksml/runner/backend/IntervalScheduleTest.java index 7c2451ca..81990b29 100644 --- a/ksml-runner/src/test/java/io/axual/ksml/runner/backend/IntervalScheduleTest.java +++ b/ksml-runner/src/test/java/io/axual/ksml/runner/backend/IntervalScheduleTest.java @@ -20,6 +20,8 @@ * =========================LICENSE_END================================== */ +import io.axual.ksml.runner.producer.ExecutableProducer; +import io.axual.ksml.runner.producer.IntervalSchedule; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/ksml-runner/src/test/java/io/axual/ksml/runner/backend/KafkaProducerRunnerTest.java b/ksml-runner/src/test/java/io/axual/ksml/runner/backend/KafkaProducerRunnerTest.java index 3f921fbb..244cc109 100644 --- a/ksml-runner/src/test/java/io/axual/ksml/runner/backend/KafkaProducerRunnerTest.java +++ b/ksml-runner/src/test/java/io/axual/ksml/runner/backend/KafkaProducerRunnerTest.java @@ -141,7 +141,7 @@ private Map loadDefinitions(String filename) throws /** * Create a KafkaProducerRunner from the given config, but with a mock Kafka producer. * - * @param config a {@link io.axual.ksml.runner.backend.KafkaProducerRunner.Config}. + * @param config a {@link KafkaProducerRunner.Config}. * @return a {@link KafkaProducerRunner} with a mocked producer. */ private KafkaProducerRunner runnerUnderTest(KafkaProducerRunner.Config config) { diff --git a/ksml/src/main/java/io/axual/ksml/data/mapper/DataObjectConverter.java b/ksml/src/main/java/io/axual/ksml/data/mapper/DataObjectConverter.java index 69c0d391..f4f56bea 100644 --- a/ksml/src/main/java/io/axual/ksml/data/mapper/DataObjectConverter.java +++ b/ksml/src/main/java/io/axual/ksml/data/mapper/DataObjectConverter.java @@ -105,14 +105,14 @@ private DataObject applyNotationConverters(String sourceNotation, DataObject val } // First we see if the target notation is able to interpret the source value - var targetConverter = NotationLibrary.notation(targetType.notation()).converter(); + var targetConverter = NotationLibrary.get(targetType.notation()).converter(); if (targetConverter != null) { final var target = targetConverter.convert(value, targetType); if (target != null && targetType.dataType().isAssignableFrom(target.type())) return target; } // If the target notation was not able to convert, then try the source notation - var sourceConverter = NotationLibrary.notation(sourceNotation).converter(); + var sourceConverter = NotationLibrary.get(sourceNotation).converter(); if (sourceConverter != null) { final var target = sourceConverter.convert(value, targetType); if (target != null && targetType.dataType().isAssignableFrom(target.type())) return target; diff --git a/ksml/src/main/java/io/axual/ksml/definition/ProducerDefinition.java b/ksml/src/main/java/io/axual/ksml/definition/ProducerDefinition.java index b1ef21b2..d06deb4e 100644 --- a/ksml/src/main/java/io/axual/ksml/definition/ProducerDefinition.java +++ b/ksml/src/main/java/io/axual/ksml/definition/ProducerDefinition.java @@ -22,9 +22,13 @@ import java.time.Duration; -public record ProducerDefinition(FunctionDefinition generator, Duration interval, FunctionDefinition condition, - TopicDefinition target, Integer count, - FunctionDefinition until) implements Definition { +public record ProducerDefinition(FunctionDefinition generator, + FunctionDefinition condition, + FunctionDefinition until, + TopicDefinition target, + Long messageCount, + Long batchSize, + Duration interval) implements Definition { @Override public String toString() { return definitionType(); diff --git a/ksml/src/main/java/io/axual/ksml/definition/parser/ProducerDefinitionParser.java b/ksml/src/main/java/io/axual/ksml/definition/parser/ProducerDefinitionParser.java index af66320d..ca311c58 100644 --- a/ksml/src/main/java/io/axual/ksml/definition/parser/ProducerDefinitionParser.java +++ b/ksml/src/main/java/io/axual/ksml/definition/parser/ProducerDefinitionParser.java @@ -40,11 +40,12 @@ public StructsParser parser() { "", "Definition of a Producer that regularly generates messages for a topic", functionField(Producers.GENERATOR, "The function that generates records", new GeneratorDefinitionParser(false)), - durationField(Producers.INTERVAL, "The interval with which the generator is called"), optional(functionField(Producers.CONDITION, "A function that validates the generator's result message. Returns \"true\" when the message may be produced on the topic, \"false\" otherwise.", new PredicateDefinitionParser(false))), - topicField(Producers.TARGET, "The topic to produce to", new TopicDefinitionParser(resources(), false)), - optional(integerField(Producers.COUNT, "The number of messages to produce.")), optional(functionField(Producers.UNTIL, "A predicate that returns true to indicate producing should stop.", new PredicateDefinitionParser(false))), - (generator, interval, condition, target, count, until, tags) -> new ProducerDefinition(generator, interval, condition, target, count, until)); + topicField(Producers.TARGET, "The topic to produce to", new TopicDefinitionParser(resources(), false)), + optional(longField(Producers.COUNT, "The number of messages to produce.")), + optional(longField(Producers.BATCH_SIZE, 1L, "The size of batches")), + optional(durationField(Producers.INTERVAL, "The interval with which the generator is called")), + (generator, condition, until, target, count, batchSize, interval, tags) -> new ProducerDefinition(generator, condition, until, target, count, batchSize, interval)); } } diff --git a/ksml/src/main/java/io/axual/ksml/dsl/KSMLDSL.java b/ksml/src/main/java/io/axual/ksml/dsl/KSMLDSL.java index 596e3ecb..d0b5721c 100644 --- a/ksml/src/main/java/io/axual/ksml/dsl/KSMLDSL.java +++ b/ksml/src/main/java/io/axual/ksml/dsl/KSMLDSL.java @@ -74,6 +74,7 @@ public static class Parameters { @NoArgsConstructor(access = AccessLevel.PRIVATE) public static class Producers { public static final String GENERATOR = "generator"; + public static final String BATCH_SIZE = "batchSize"; public static final String INTERVAL = "interval"; public static final String CONDITION = "condition"; public static final String TARGET = "to"; diff --git a/ksml/src/main/java/io/axual/ksml/generator/StreamDataType.java b/ksml/src/main/java/io/axual/ksml/generator/StreamDataType.java index 7c40f2a0..48bf81da 100644 --- a/ksml/src/main/java/io/axual/ksml/generator/StreamDataType.java +++ b/ksml/src/main/java/io/axual/ksml/generator/StreamDataType.java @@ -47,7 +47,7 @@ public String toString() { } public Serde serde() { - final var notation = NotationLibrary.notation(userType.notation()); + final var notation = NotationLibrary.get(userType.notation()); if (userType.dataType() instanceof UnionType unionType) return new UnionSerde(FLATTENER.flatten(unionType), isKey, notation::serde); var serde = notation.serde(FLATTENER.flatten(userType.dataType()), isKey); diff --git a/ksml/src/main/java/io/axual/ksml/parser/DefinitionParser.java b/ksml/src/main/java/io/axual/ksml/parser/DefinitionParser.java index 2f7b47b9..cac26947 100644 --- a/ksml/src/main/java/io/axual/ksml/parser/DefinitionParser.java +++ b/ksml/src/main/java/io/axual/ksml/parser/DefinitionParser.java @@ -172,6 +172,14 @@ protected StructsParser integerField(String childName, Integer valueIfN return freeField(childName, valueIfNull, doc, ParserWithSchemas.of(ParseNode::asInt, DataSchema.integerSchema())); } + protected StructsParser longField(String childName, String doc) { + return longField(childName, null, doc); + } + + protected StructsParser longField(String childName, Long valueIfNull, String doc) { + return freeField(childName, valueIfNull, doc, ParserWithSchemas.of(ParseNode::asLong, DataSchema.longSchema())); + } + protected StructsParser stringField(String childName, String doc) { return stringField(childName, null, doc); } diff --git a/ksml/src/main/java/io/axual/ksml/parser/UserTypeParser.java b/ksml/src/main/java/io/axual/ksml/parser/UserTypeParser.java index 45008b23..2392f7bf 100644 --- a/ksml/src/main/java/io/axual/ksml/parser/UserTypeParser.java +++ b/ksml/src/main/java/io/axual/ksml/parser/UserTypeParser.java @@ -98,7 +98,7 @@ private static UserType parseTypeAndNotation(String composedType, String default // Internal types final var internalType = parseType(datatype); if (internalType != null) { - final var n = NotationLibrary.notation(notation); + final var n = NotationLibrary.get(notation); if (n.defaultType() != null && !n.defaultType().isAssignableFrom(internalType)) { throw new DataException("Notation " + notation + " does not allow for data type " + datatype); } @@ -146,12 +146,12 @@ private static UserType parseTypeAndNotation(String composedType, String default // Notation type with specific schema if (NotationLibrary.exists(notation)) { - final var loader = NotationLibrary.notation(notation).loader(); + final var loader = NotationLibrary.get(notation).loader(); if (loader != null) { // If we reach this point, then we assume that the datatype refers to a schema to load final var schema = SchemaLibrary.getSchema(notation, datatype, false); final var schemaType = new DataTypeSchemaMapper().fromDataSchema(schema); - if (!(NotationLibrary.notation(notation).defaultType().isAssignableFrom(schemaType))) { + if (!(NotationLibrary.get(notation).defaultType().isAssignableFrom(schemaType))) { throw new DataException("Notation " + notation + " does not allow for data type " + datatype); } return new UserType(notation, schemaType); @@ -160,7 +160,7 @@ private static UserType parseTypeAndNotation(String composedType, String default // Notation without specific schema if (NotationLibrary.exists(datatype)) { - return new UserType(datatype, NotationLibrary.notation(datatype).defaultType()); + return new UserType(datatype, NotationLibrary.get(datatype).defaultType()); } throw new TopologyException("Unknown data type: " + datatype); diff --git a/ksml/src/test/java/io/axual/ksml/testutil/KSMLTestExtension.java b/ksml/src/test/java/io/axual/ksml/testutil/KSMLTestExtension.java index 33ff64f2..3e060e07 100644 --- a/ksml/src/test/java/io/axual/ksml/testutil/KSMLTestExtension.java +++ b/ksml/src/test/java/io/axual/ksml/testutil/KSMLTestExtension.java @@ -208,7 +208,7 @@ private Serializer getValueSerializer(KSMLTopic ksmlTopic) { private Serializer getSerializer(KSMLTopic ksmlTopic, boolean isKey) { return switch (isKey ? ksmlTopic.keySerde() : ksmlTopic.valueSerde()) { case AVRO -> { - final var avroNotation = (MockAvroNotation) NotationLibrary.notation(MockAvroNotation.NAME); + final var avroNotation = (MockAvroNotation) NotationLibrary.get(MockAvroNotation.NAME); var result = new KafkaAvroSerializer(avroNotation.mockSchemaRegistryClient()); result.configure(avroNotation.getSchemaRegistryConfigs(), isKey); yield result; @@ -228,7 +228,7 @@ private Deserializer getValueDeserializer(KSMLTopic kamlTopic) { private Deserializer getDeserializer(KSMLTopic kamlTopic, boolean isKey) { return switch (isKey ? kamlTopic.keySerde() : kamlTopic.valueSerde()) { case AVRO -> { - final var avroNotation = (MockAvroNotation) NotationLibrary.notation(MockAvroNotation.NAME); + final var avroNotation = (MockAvroNotation) NotationLibrary.get(MockAvroNotation.NAME); final var result = new KafkaAvroDeserializer(avroNotation.mockSchemaRegistryClient()); result.configure(avroNotation.getSchemaRegistryConfigs(), isKey); yield result; From 681ca31d48f493fd8d17b1e3f4d77eedd09508fa Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:41:29 +0100 Subject: [PATCH 34/55] Finalize merge From 36a96cfc35cc15e51583a0e2461334ab4d119564 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:42:17 +0100 Subject: [PATCH 35/55] Finalize merge From c97ba51ebbcb66ca133c07406968f6fc315e49f7 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:58:04 +0100 Subject: [PATCH 36/55] Update NOTICE --- ksml-reporting/NOTICE.txt | 132 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 ksml-reporting/NOTICE.txt diff --git a/ksml-reporting/NOTICE.txt b/ksml-reporting/NOTICE.txt new file mode 100644 index 00000000..8aceb42f --- /dev/null +++ b/ksml-reporting/NOTICE.txt @@ -0,0 +1,132 @@ + +Lists of 130 third-party dependencies. + (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Classic Module (ch.qos.logback:logback-classic:1.5.6 - http://logback.qos.ch/logback-classic) + (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Core Module (ch.qos.logback:logback-core:1.5.6 - http://logback.qos.ch/logback-core) + (MIT License) minimal-json (com.eclipsesource.minimal-json:minimal-json:0.9.5 - https://github.com/ralfstx/minimal-json) + (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.17.1 - https://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.17.1 - https://github.com/FasterXML/jackson-core) + (The Apache Software License, Version 2.0) jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.17.1 - https://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) Jackson-dataformat-CSV (com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.17.1 - https://github.com/FasterXML/jackson-dataformats-text) + (The Apache Software License, Version 2.0) Jackson-dataformat-XML (com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.1 - https://github.com/FasterXML/jackson-dataformat-xml) + (The Apache Software License, Version 2.0) Jackson-dataformat-YAML (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.17.1 - https://github.com/FasterXML/jackson-dataformats-text) + (The Apache Software License, Version 2.0) Jackson Jakarta-RS: base (com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-base:2.17.1 - https://github.com/FasterXML/jackson-jakarta-rs-providers/jackson-jakarta-rs-base) + (The Apache Software License, Version 2.0) Jackson Jakarta-RS: JSON (com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider:2.17.1 - https://github.com/FasterXML/jackson-jakarta-rs-providers/jackson-jakarta-rs-json-provider) + (The Apache Software License, Version 2.0) Jackson module: Jakarta XML Bind Annotations (jakarta.xml.bind) (com.fasterxml.jackson.module:jackson-module-jakarta-xmlbind-annotations:2.17.1 - https://github.com/FasterXML/jackson-modules-base) + (The Apache License, Version 2.0) Woodstox (com.fasterxml.woodstox:woodstox-core:6.6.2 - https://github.com/FasterXML/woodstox) + (BSD 2-Clause License) zstd-jni (com.github.luben:zstd-jni:1.5.6-3 - https://github.com/luben/zstd-jni) + (The Apache Software License, Version 2.0) FindBugs-jsr305 (com.google.code.findbugs:jsr305:3.0.2 - http://findbugs.sourceforge.net/) + (Apache 2.0) error-prone annotations (com.google.errorprone:error_prone_annotations:2.26.1 - https://errorprone.info/error_prone_annotations) + (The Apache Software License, Version 2.0) Guava InternalFutureFailureAccess and InternalFutures (com.google.guava:failureaccess:1.0.2 - https://github.com/google/guava/failureaccess) + (Apache License, Version 2.0) Guava: Google Core Libraries for Java (com.google.guava:guava:33.2.0-jre - https://github.com/google/guava) + (The Apache Software License, Version 2.0) Guava ListenableFuture only (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava - https://github.com/google/guava/listenablefuture) + (Apache License, Version 2.0) J2ObjC Annotations (com.google.j2objc:j2objc-annotations:3.0.0 - https://github.com/google/j2objc/) + (Go License) RE2/J (com.google.re2j:re2j:1.6 - http://github.com/google/re2j) + (Eclipse Distribution License - v 1.0) Jakarta SOAP Implementation (com.sun.xml.messaging.saaj:saaj-impl:3.0.4 - https://projects.eclipse.org/projects/ee4j/metro-saaj/saaj-impl) + (Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.17.0 - https://commons.apache.org/proper/commons-codec/) + (Apache-2.0) Apache Commons IO (commons-io:commons-io:2.16.1 - https://commons.apache.org/proper/commons-io/) + (Apache License Version 2.0) apicurio-common-rest-client-common (io.apicurio:apicurio-common-rest-client-common:0.1.18.Final - https://www.apicur.io/apicurio-common-rest-client-common/) + (Apache License Version 2.0) apicurio-common-rest-client-jdk (io.apicurio:apicurio-common-rest-client-jdk:0.1.18.Final - https://www.apicur.io/apicurio-common-rest-client-jdk/) + (Apache License Version 2.0) apicurio-registry-client (io.apicurio:apicurio-registry-client:2.6.5.Final - https://www.apicur.io/apicurio-registry-client/) + (Apache License Version 2.0) apicurio-registry-common (io.apicurio:apicurio-registry-common:2.6.5.Final - https://www.apicur.io/apicurio-registry-common/) + (Apache License Version 2.0) apicurio-registry-schema-resolver (io.apicurio:apicurio-registry-schema-resolver:2.6.5.Final - https://www.apicur.io/apicurio-registry-schema-resolver/) + (Apache License Version 2.0) apicurio-registry-serde-common (io.apicurio:apicurio-registry-serde-common:2.6.5.Final - https://www.apicur.io/apicurio-registry-serde-common/) + (Apache License Version 2.0) apicurio-registry-serdes-avro-serde (io.apicurio:apicurio-registry-serdes-avro-serde:2.6.5.Final - https://www.apicur.io/apicurio-registry-serdes-avro-serde/) + (Apache 2.0) KSML (io.axual.ksml:ksml:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml) + (Apache 2.0) KSML Data Library (io.axual.ksml:ksml-data:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-data) + (Apache 2.0) KSML Data Library - AVRO (io.axual.ksml:ksml-data-avro:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-data-avro) + (Apache 2.0) KSML Data Library - CSV (io.axual.ksml:ksml-data-csv:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-data-csv) + (Apache 2.0) KSML Data Library - SOAP (io.axual.ksml:ksml-data-soap:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-data-soap) + (Apache 2.0) KSML Data Library - XML (io.axual.ksml:ksml-data-xml:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-data-xml) + (Apache 2.0) KSML Kafka clients (io.axual.ksml:ksml-kafka-clients:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-kafka-clients) + (Apache 2.0) KSML Queryable State Store (io.axual.ksml:ksml-query:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-query) + (Apache 2.0) KSML Runner (io.axual.ksml:ksml-runner:1.1.0-SNAPSHOT - https://github.com/Axual/ksml/ksml-runner) + (Apache License 2.0) utils (io.confluent:common-utils:7.6.1 - https://confluent.io/common-utils) + (Apache License 2.0) kafka-avro-serializer (io.confluent:kafka-avro-serializer:7.6.1 - http://confluent.io/kafka-avro-serializer) + (Apache License 2.0) kafka-schema-registry-client (io.confluent:kafka-schema-registry-client:7.6.1 - http://confluent.io/kafka-schema-registry-client) + (Apache License 2.0) kafka-schema-serializer (io.confluent:kafka-schema-serializer:7.6.1 - http://confluent.io/kafka-schema-serializer) + (The Apache Software License, Version 2.0) Log Redactor (io.confluent:logredactor:1.0.12 - https://github.com/confluentinc/logredactor) + (The Apache Software License, Version 2.0) Log Redactor Metrics (io.confluent:logredactor-metrics:1.0.12 - https://github.com/confluentinc/logredactor) + (Apache License 2.0) Metrics Core (io.dropwizard.metrics:metrics-core:4.2.25 - https://metrics.dropwizard.io/metrics-core) + (Apache License 2.0) Metrics Integration with JMX (io.dropwizard.metrics:metrics-jmx:4.2.25 - https://metrics.dropwizard.io/metrics-jmx) + (The Apache Software License, Version 2.0) Prometheus Metrics Config (io.prometheus:prometheus-metrics-config:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-config) + (The Apache Software License, Version 2.0) Prometheus Metrics Core (io.prometheus:prometheus-metrics-core:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-core) + (The Apache Software License, Version 2.0) Prometheus Metrics Exporter - Common (io.prometheus:prometheus-metrics-exporter-common:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-exporter-common) + (The Apache Software License, Version 2.0) Prometheus Metrics Exporter - HTTP Server (io.prometheus:prometheus-metrics-exporter-httpserver:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-exporter-httpserver) + (The Apache Software License, Version 2.0) Prometheus Metrics Exposition Formats (io.prometheus:prometheus-metrics-exposition-formats:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-exposition-formats) + (The Apache Software License, Version 2.0) Prometheus Metrics Instrumentation - JVM (io.prometheus:prometheus-metrics-instrumentation-jvm:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-instrumentation-jvm) + (The Apache Software License, Version 2.0) Prometheus Metrics Model (io.prometheus:prometheus-metrics-model:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-model) + (The Apache Software License, Version 2.0) Shaded Protobuf (io.prometheus:prometheus-metrics-shaded-protobuf:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-shaded-dependencies/prometheus-metrics-shaded-protobuf) + (The Apache Software License, Version 2.0) Prometheus Metrics Tracer Common (io.prometheus:prometheus-metrics-tracer-common:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-tracer/prometheus-metrics-tracer-common) + (The Apache Software License, Version 2.0) Prometheus Metrics Tracer Initializer (io.prometheus:prometheus-metrics-tracer-initializer:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-tracer/prometheus-metrics-tracer-initializer) + (The Apache Software License, Version 2.0) Prometheus Metrics Tracer OpenTelemetry (io.prometheus:prometheus-metrics-tracer-otel:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-tracer/prometheus-metrics-tracer-otel) + (The Apache Software License, Version 2.0) Prometheus Metrics Tracer OpenTelemetry Agent (io.prometheus:prometheus-metrics-tracer-otel-agent:1.3.1 - http://github.com/prometheus/client_java/prometheus-metrics-tracer/prometheus-metrics-tracer-otel-agent) + (The Apache Software License, Version 2.0) Prometheus JMX Exporter - Collector (io.prometheus.jmx:collector:1.0.1 - http://github.com/prometheus/jmx_exporter) + (The Apache Software License, Version 2.0) Prometheus JMX Exporter - Common (io.prometheus.jmx:jmx_prometheus_common:1.0.1 - http://github.com/prometheus/jmx_exporter) + (Apache License 2.0) swagger-annotations (io.swagger.core.v3:swagger-annotations:2.1.10 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations) + (EDL 1.0) Jakarta Activation API (jakarta.activation:jakarta.activation-api:2.1.3 - https://github.com/jakartaee/jaf-api) + (EPL 2.0) (GPL2 w/ CPE) Jakarta Annotations API (jakarta.annotation:jakarta.annotation-api:2.1.1 - https://projects.eclipse.org/projects/ee4j.ca) + (The Apache Software License, Version 2.0) Jakarta Dependency Injection (jakarta.inject:jakarta.inject-api:2.0.1 - https://github.com/eclipse-ee4j/injection-api) + (EPL 2.0) (GPL2 w/ CPE) Jakarta Servlet (jakarta.servlet:jakarta.servlet-api:6.0.0 - https://projects.eclipse.org/projects/ee4j.servlet) + (Apache License 2.0) Jakarta Bean Validation API (jakarta.validation:jakarta.validation-api:3.0.2 - https://beanvalidation.org) + (EPL-2.0) (GPL-2.0-with-classpath-exception) Jakarta RESTful WS API (jakarta.ws.rs:jakarta.ws.rs-api:3.1.0 - https://github.com/eclipse-ee4j/jaxrs-api) + (Eclipse Distribution License - v 1.0) Jakarta XML Binding API (jakarta.xml.bind:jakarta.xml.bind-api:4.0.2 - https://github.com/jakartaee/jaxb-api/jakarta.xml.bind-api) + (Eclipse Distribution License - v 1.0) Jakarta SOAP with Attachments API (jakarta.xml.soap:jakarta.xml.soap-api:3.0.2 - https://github.com/jakartaee/saaj-api) + (Apache License, Version 2.0) (MIT License) Logstash Logback Encoder (net.logstash.logback:logstash-logback-encoder:7.4 - https://github.com/logfellow/logstash-logback-encoder) + (Apache-2.0) Apache Avro (org.apache.avro:avro:1.12.0 - https://avro.apache.org) + (Apache-2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.26.2 - https://commons.apache.org/proper/commons-compress/) + (Apache-2.0) Apache Commons Lang (org.apache.commons:commons-lang3:3.14.0 - https://commons.apache.org/proper/commons-lang/) + (Apache-2.0) Apache Commons Text (org.apache.commons:commons-text:1.12.0 - https://commons.apache.org/proper/commons-text) + (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-clients:3.8.0 - https://kafka.apache.org) + (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-streams:3.8.0 - https://kafka.apache.org) + (Bouncy Castle Licence) Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs (org.bouncycastle:bcpkix-jdk18on:1.76 - https://www.bouncycastle.org/java.html) + (Bouncy Castle Licence) Bouncy Castle Provider (org.bouncycastle:bcprov-jdk18on:1.76 - https://www.bouncycastle.org/java.html) + (Bouncy Castle Licence) Bouncy Castle ASN.1 Extension and Utility APIs (org.bouncycastle:bcutil-jdk18on:1.76 - https://www.bouncycastle.org/java.html) + (The MIT License) Checker Qual (org.checkerframework:checker-qual:3.42.0 - https://checkerframework.org/) + (BSD-3-Clause) commons-compiler (org.codehaus.janino:commons-compiler:3.1.12 - http://janino-compiler.github.io/commons-compiler/) + (BSD-3-Clause) janino (org.codehaus.janino:janino:3.1.12 - http://janino-compiler.github.io/janino/) + (The BSD 2-Clause License) Stax2 API (org.codehaus.woodstox:stax2-api:4.2.2 - http://github.com/FasterXML/stax2-api) + (EDL 1.0) Angus Activation Registries (org.eclipse.angus:angus-activation:2.0.2 - https://github.com/eclipse-ee4j/angus-activation/angus-activation) + (EPL-2.0) grizzly-framework (org.glassfish.grizzly:grizzly-framework:4.0.2 - https://projects.eclipse.org/projects/ee4j.grizzly/grizzly-framework) + (EPL-2.0) grizzly-http (org.glassfish.grizzly:grizzly-http:4.0.2 - https://projects.eclipse.org/projects/ee4j.grizzly/grizzly-http) + (EPL-2.0) grizzly-http-server (org.glassfish.grizzly:grizzly-http-server:4.0.2 - https://projects.eclipse.org/projects/ee4j.grizzly/grizzly-http-server) + (EPL-2.0) grizzly-http-servlet (org.glassfish.grizzly:grizzly-http-servlet:4.0.2 - https://projects.eclipse.org/projects/ee4j.grizzly/grizzly-http-servlet) + (EPL 2.0) (GPL2 w/ CPE) HK2 API module (org.glassfish.hk2:hk2-api:3.0.6 - https://github.com/eclipse-ee4j/glassfish-hk2/hk2-api) + (EPL 2.0) (GPL2 w/ CPE) ServiceLocator Default Implementation (org.glassfish.hk2:hk2-locator:3.0.6 - https://github.com/eclipse-ee4j/glassfish-hk2/hk2-locator) + (EPL 2.0) (GPL2 w/ CPE) HK2 Implementation Utilities (org.glassfish.hk2:hk2-utils:3.0.6 - https://github.com/eclipse-ee4j/glassfish-hk2/hk2-utils) + (EPL 2.0) (GPL2 w/ CPE) OSGi resource locator (org.glassfish.hk2:osgi-resource-locator:1.0.3 - https://projects.eclipse.org/projects/ee4j/osgi-resource-locator) + (EPL 2.0) (GPL2 w/ CPE) aopalliance version 1.0 repackaged as a module (org.glassfish.hk2.external:aopalliance-repackaged:3.0.6 - https://github.com/eclipse-ee4j/glassfish-hk2/external/aopalliance-repackaged) + (Apache License, 2.0) (BSD 2-Clause) (EDL 1.0) (EPL 2.0) (GPL2 w/ CPE) (MIT license) (Modified BSD) (Public Domain) (W3C license) (jQuery license) jersey-container-grizzly2-http (org.glassfish.jersey.containers:jersey-container-grizzly2-http:3.1.7 - https://projects.eclipse.org/projects/ee4j.jersey/project/jersey-container-grizzly2-http) + (Apache License, 2.0) (BSD 2-Clause) (EDL 1.0) (EPL 2.0) (GPL2 w/ CPE) (MIT license) (Modified BSD) (Public Domain) (W3C license) (jQuery license) jersey-container-grizzly2-servlet (org.glassfish.jersey.containers:jersey-container-grizzly2-servlet:3.1.7 - https://projects.eclipse.org/projects/ee4j.jersey/project/jersey-container-grizzly2-servlet) + (Apache License, 2.0) (BSD 2-Clause) (EDL 1.0) (EPL 2.0) (GPL2 w/ CPE) (MIT license) (Modified BSD) (Public Domain) (W3C license) (jQuery license) jersey-container-servlet (org.glassfish.jersey.containers:jersey-container-servlet:3.1.7 - https://projects.eclipse.org/projects/ee4j.jersey/project/jersey-container-servlet) + (Apache License, 2.0) (BSD 2-Clause) (EDL 1.0) (EPL 2.0) (GPL2 w/ CPE) (MIT license) (Modified BSD) (Public Domain) (W3C license) (jQuery license) jersey-container-servlet-core (org.glassfish.jersey.containers:jersey-container-servlet-core:3.1.7 - https://projects.eclipse.org/projects/ee4j.jersey/project/jersey-container-servlet-core) + (Apache License, 2.0) (BSD 2-Clause) (EDL 1.0) (EPL 2.0) (GPL2 w/ CPE) (MIT license) (Modified BSD) (Public Domain) (W3C license) (jQuery license) jersey-core-client (org.glassfish.jersey.core:jersey-client:3.1.7 - https://projects.eclipse.org/projects/ee4j.jersey/jersey-client) + (Apache License, 2.0) (EPL 2.0) (Public Domain) (The GNU General Public License (GPL), Version 2, With Classpath Exception) jersey-core-common (org.glassfish.jersey.core:jersey-common:3.1.7 - https://projects.eclipse.org/projects/ee4j.jersey/jersey-common) + (Apache License, 2.0) (EPL 2.0) (Modified BSD) (The GNU General Public License (GPL), Version 2, With Classpath Exception) jersey-core-server (org.glassfish.jersey.core:jersey-server:3.1.7 - https://projects.eclipse.org/projects/ee4j.jersey/jersey-server) + (Apache License, 2.0) (BSD 2-Clause) (EDL 1.0) (EPL 2.0) (GPL2 w/ CPE) (MIT license) (Modified BSD) (Public Domain) (W3C license) (jQuery license) jersey-ext-entity-filtering (org.glassfish.jersey.ext:jersey-entity-filtering:3.1.7 - https://projects.eclipse.org/projects/ee4j.jersey/project/jersey-entity-filtering) + (Apache License, 2.0) (BSD 2-Clause) (EDL 1.0) (EPL 2.0) (GPL2 w/ CPE) (MIT license) (Modified BSD) (Public Domain) (W3C license) (jQuery license) jersey-inject-hk2 (org.glassfish.jersey.inject:jersey-hk2:3.1.7 - https://projects.eclipse.org/projects/ee4j.jersey/project/jersey-hk2) + (Apache License, 2.0) (EPL 2.0) (The GNU General Public License (GPL), Version 2, With Classpath Exception) jersey-media-json-jackson (org.glassfish.jersey.media:jersey-media-json-jackson:3.1.7 - https://projects.eclipse.org/projects/ee4j.jersey/project/jersey-media-json-jackson) + (New BSD License (3-clause BSD license)) Sulong API (org.graalvm.llvm:llvm-api:23.1.2 - http://www.graalvm.org/) + (Universal Permissive License, Version 1.0) Polyglot (org.graalvm.polyglot:polyglot:23.1.2 - https://github.com/oracle/graal) + (MIT License) (Python Software Foundation License) (Universal Permissive License, Version 1.0) Graalpython (org.graalvm.python:python-language:23.1.2 - http://www.graalvm.org/) + (MIT License) (Python Software Foundation License) (Universal Permissive License, Version 1.0) Graalpython Resources (org.graalvm.python:python-resources:23.1.2 - http://www.graalvm.org/) + (Universal Permissive License, Version 1.0) Tregex (org.graalvm.regex:regex:23.1.2 - http://www.graalvm.org/) + (Universal Permissive License, Version 1.0) Collections (org.graalvm.sdk:collections:23.1.2 - https://github.com/oracle/graal) + (Universal Permissive License, Version 1.0) Jniutils (org.graalvm.sdk:jniutils:23.1.2 - https://github.com/oracle/graal) + (Universal Permissive License, Version 1.0) Nativeimage (org.graalvm.sdk:nativeimage:23.1.2 - https://github.com/oracle/graal) + (Universal Permissive License, Version 1.0) Word (org.graalvm.sdk:word:23.1.2 - https://github.com/oracle/graal) + (Unicode/ICU License) Truffle Icu4j (org.graalvm.shadowed:icu4j:23.1.2 - http://openjdk.java.net/projects/graal) + (Universal Permissive License, Version 1.0) Truffle Json (org.graalvm.shadowed:json:23.1.2 - http://openjdk.java.net/projects/graal) + (GNU General Public License, version 2, with the Classpath Exception) Truffle Profiler (org.graalvm.tools:profiler-tool:23.1.2 - http://openjdk.java.net/projects/graal) + (Universal Permissive License, Version 1.0) Truffle API (org.graalvm.truffle:truffle-api:23.1.2 - http://openjdk.java.net/projects/graal) + (Universal Permissive License, Version 1.0) Truffle Compiler (org.graalvm.truffle:truffle-compiler:23.1.2 - http://openjdk.java.net/projects/graal) + (Universal Permissive License, Version 1.0) Truffle Nfi (org.graalvm.truffle:truffle-nfi:23.1.2 - http://openjdk.java.net/projects/graal) + (Universal Permissive License, Version 1.0) Truffle Nfi Libffi (org.graalvm.truffle:truffle-nfi-libffi:23.1.2 - http://openjdk.java.net/projects/graal) + (Universal Permissive License, Version 1.0) Truffle Runtime (org.graalvm.truffle:truffle-runtime:23.1.2 - http://openjdk.java.net/projects/graal) + (Apache License 2.0) (LGPL 2.1) (MPL 1.1) Javassist (org.javassist:javassist:3.30.2-GA - https://www.javassist.org/) + (Eclipse Distribution License - v 1.0) Extended StAX API (org.jvnet.staxex:stax-ex:2.1.0 - https://projects.eclipse.org/projects/ee4j/stax-ex) + (The Apache Software License, Version 2.0) LZ4 and xxHash (org.lz4:lz4-java:1.8.0 - https://github.com/lz4/lz4-java) + (Apache License 2.0) (GNU General Public License, version 2) RocksDB JNI (org.rocksdb:rocksdbjni:7.9.2 - https://rocksdb.org) + (MIT License) SLF4J API Module (org.slf4j:slf4j-api:2.0.13 - http://www.slf4j.org) + (Public Domain) XZ for Java (org.tukaani:xz:1.9 - https://tukaani.org/xz/java.html) + (Apache-2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.10.5 - https://github.com/xerial/snappy-java) + (Apache License, Version 2.0) SnakeYAML (org.yaml:snakeyaml:2.2 - https://bitbucket.org/snakeyaml/snakeyaml) From b35397e2749abc9633f75ac74008376d587af6ca Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:59:45 +0100 Subject: [PATCH 37/55] Upgrade to Kafka 3.8.0 # Conflicts: # pom.xml From 707a586ffa9a6c0af17c4008c12bbbea17044d71 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 21:00:29 +0100 Subject: [PATCH 38/55] Update byte manipulation example - Generate example data for byte manipulation example - Have byte manipulation example read from alternative topic # Conflicts: # examples/00-example-generate-sensordata.yaml From f399da629f59e04590d8cbaf1f2017b28fbede48 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 21:43:55 +0100 Subject: [PATCH 39/55] Introduce notation factories to flexibly choose between notation serdes --- .../ksml/data/notation/avro/AvroNotation.java | 16 ++++++------- .../java/io/axual/ksml/runner/KSMLRunner.java | 23 ++++++++----------- .../java/io/axual/ksml/runner/lombok.config | 3 +++ 3 files changed, 20 insertions(+), 22 deletions(-) create mode 100644 ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config diff --git a/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/AvroNotation.java b/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/AvroNotation.java index 9f42f349..aa9290bb 100644 --- a/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/AvroNotation.java +++ b/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/AvroNotation.java @@ -21,15 +21,6 @@ */ import com.google.common.collect.ImmutableMap; - -import org.apache.kafka.common.header.Headers; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.common.serialization.Serializer; - -import java.nio.ByteBuffer; -import java.util.Map; - import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; import io.apicurio.registry.serde.avro.AvroKafkaSerializer; import io.axual.ksml.data.exception.DataException; @@ -44,6 +35,13 @@ import io.confluent.kafka.serializers.KafkaAvroDeserializer; import io.confluent.kafka.serializers.KafkaAvroSerializer; import lombok.Getter; +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serializer; + +import java.nio.ByteBuffer; +import java.util.Map; public class AvroNotation implements Notation { public static final DataType DEFAULT_TYPE = new StructType(); diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java index fcd9a66e..8aad5233 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java @@ -112,27 +112,24 @@ public static void main(String[] args) { // Set up all notation overrides from the KSML config for (final var notationEntry : ksmlConfig.notations().entrySet()) { - final var notationStr = notationEntry.getKey() != null ? notationEntry.getKey() : "undefined"; + final var notation = notationEntry.getKey(); final var notationConfig = notationEntry.getValue(); - final var factoryName = notationConfig != null ? notationConfig.type() : "unknown"; - if (notationConfig != null && factoryName != null) { - final var factory = notationFactories.notations().get(factoryName); - if (factory == null) { - throw FatalError.reportAndExit(new ConfigException("Unknown notation type: " + factoryName)); + if (notation != null && notationConfig != null) { + final var n = notationFactories.notations().get(notationConfig.type()); + if (n == null) { + throw FatalError.reportAndExit(new ConfigException("Notation type '" + notationConfig.type() + "' not found")); } - NotationLibrary.register(notationStr, factory.create(notationConfig.config())); - } else { - log.warn("Notation configuration incomplete: notation=" + notationStr + ", type=" + factoryName); + NotationLibrary.register(notation, n.create(notationConfig.config())); } } - // Ensure typical defaults are used for AVRO - // WARNING: Defaults for notations will be deprecated in the future. Make sure you explicitly configure - // notations with multiple implementations (like AVRO) in your ksml-runner.yaml. + // Ensure typical defaults are used + // WARNING: Defaults for notations will be deprecated in the future. Make sure you explicitly define + // notations that have multiple implementations in your ksml-runner.yaml. if (!NotationLibrary.exists(NotationFactories.AVRO)) { final var defaultAvro = notationFactories.confluentAvro(); NotationLibrary.register(NotationFactories.AVRO, defaultAvro.create(null)); - log.warn("No implementation specified for AVRO notation. If you use AVRO in your KSML definition, add the required configuration to the ksml-runner.yaml"); + log.warn("No implementation specified for AVRO notation. If you plan to use AVRO, add the required implementation to the ksml-runner.yaml"); } final var errorHandling = ksmlConfig.errorHandlingConfig(); diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config b/ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config new file mode 100644 index 00000000..881a8aa5 --- /dev/null +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/lombok.config @@ -0,0 +1,3 @@ +lombok.accessors.fluent=true +lombok.accessors.chain=false +lombok.equalsAndHashCode.callSuper=call \ No newline at end of file From 36104b9dac96f0bda9238cccb0659bd758794115 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 21:44:40 +0100 Subject: [PATCH 40/55] Updates to the code --- .../io/axual/ksml/runner/config/KSMLRunnerConfigTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java index 90da3a51..b536e067 100644 --- a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java +++ b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java @@ -45,7 +45,7 @@ void shouldLoadWithoutExceptions() throws IOException { final var yaml = getClass().getClassLoader().getResourceAsStream("ksml-runner-config.yaml"); final var ksmlRunnerConfig = objectMapper.readValue(yaml, KSMLRunnerConfig.class); - assertNotNull(ksmlRunnerConfig.ksmlConfig()); - assertNotNull(ksmlRunnerConfig.kafkaConfig()); + assertNotNull(ksmlRunnerConfig.ksml()); + assertNotNull(ksmlRunnerConfig.kafka()); } } From a88fdcf0b17bfe9f5a06a9ba03710f8ceb5c8430 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 21:21:05 +0100 Subject: [PATCH 41/55] Update documentation and example runner definitions # Conflicts: # examples/ksml-data-generator.yaml # examples/ksml-runner.yaml --- examples/ksml-data-generator.yaml | 2 +- examples/ksml-runner.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/ksml-data-generator.yaml b/examples/ksml-data-generator.yaml index b5ab388c..b5c30c93 100644 --- a/examples/ksml-data-generator.yaml +++ b/examples/ksml-data-generator.yaml @@ -42,7 +42,7 @@ ksml: ## Below this line, specify properties to be passed into Confluent's KafkaAvroSerializer and KafkaAvroDeserializer config: # The example uses Apicurio, using the Apicurio Confluent Compatibility URL - schema.registry.url: http://schema_registry:8081/apis/ccompat/v7 + schema.registry.url: http://schema-registry:8081/apis/ccompat/v7 auto.register.schemas: true normalize.schemas: true # Below is an example SSL configuration for Confluent Serialization library diff --git a/examples/ksml-runner.yaml b/examples/ksml-runner.yaml index 9d4ff609..e85a2b57 100644 --- a/examples/ksml-runner.yaml +++ b/examples/ksml-runner.yaml @@ -42,7 +42,7 @@ ksml: ## Below this line, specify properties to be passed into Confluent's KafkaAvroSerializer and KafkaAvroDeserializer config: # Link to Apicurio Confluent Compatibility URL - schema.registry.url: http://schema_registry:8081/apis/ccompat/v7 + schema.registry.url: http://schema-registry:8081/apis/ccompat/v7 auto.register.schemas: true normalize.schemas: true # Below is an example SSL configuration for Confluent Serialization library From 0d2133f321189621bbdb90547adb4dcf136dfe40 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 21:21:24 +0100 Subject: [PATCH 42/55] Update apicurio serde version # Conflicts: # pom.xml From 93e4cf7ffcd28b8f0cfb31b7ad53e6c5dadd7848 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 21:24:06 +0100 Subject: [PATCH 43/55] Processed Richard's review comments for MR From 6a51dc05afbded46da3e434d6a1635b52cd78030 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 21:26:22 +0100 Subject: [PATCH 44/55] Add header support to serialisation/deserialisation to allow Apicurio notation to use headers --- .../ksml/data/notation/avro/AvroNotation.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/AvroNotation.java b/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/AvroNotation.java index aa9290bb..9f42f349 100644 --- a/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/AvroNotation.java +++ b/ksml-data-avro/src/main/java/io/axual/ksml/data/notation/avro/AvroNotation.java @@ -21,6 +21,15 @@ */ import com.google.common.collect.ImmutableMap; + +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serializer; + +import java.nio.ByteBuffer; +import java.util.Map; + import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; import io.apicurio.registry.serde.avro.AvroKafkaSerializer; import io.axual.ksml.data.exception.DataException; @@ -35,13 +44,6 @@ import io.confluent.kafka.serializers.KafkaAvroDeserializer; import io.confluent.kafka.serializers.KafkaAvroSerializer; import lombok.Getter; -import org.apache.kafka.common.header.Headers; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.common.serialization.Serializer; - -import java.nio.ByteBuffer; -import java.util.Map; public class AvroNotation implements Notation { public static final DataType DEFAULT_TYPE = new StructType(); From 1c885ea959cef82f69b2defdda736d96660b2fd3 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 21:45:27 +0100 Subject: [PATCH 45/55] Unify lombok configurations From 37f610a1616aa4dd7da7f88211cd393b8ac9890c Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 21:46:35 +0100 Subject: [PATCH 46/55] Implement Batch Producer From 1e711d0917fd4c4fec6dbf5cb6710182b007918d Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 19:22:27 +0100 Subject: [PATCH 47/55] Introduce performance measurement example * Split up example message generator into a normal one (interval), a binary one (intended to run example 12) and a batch one (intended for performance measurements) * Introduce example 19 to perform generic performance measurements on any KSML pipeline * Remove reading back from binary topic in example 1 to allow running all examples in parallel (ie no conflict with example 12) * Remove dead code from TopologyGenerator * Improve error message in UserTypeParser --- examples/01-example-inspect.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/examples/01-example-inspect.yaml b/examples/01-example-inspect.yaml index ea0e3622..235a9f94 100644 --- a/examples/01-example-inspect.yaml +++ b/examples/01-example-inspect.yaml @@ -8,10 +8,6 @@ streams: keyType: string valueType: avro:SensorData offsetResetPolicy: latest - sensor_source_avro_binary: - topic: ksml_sensordata_avro_binary - keyType: string - valueType: otherAvro:SensorData sensor_source_csv: topic: ksml_sensordata_csv keyType: string @@ -40,10 +36,6 @@ pipelines: from: sensor_source_avro forEach: code: log_message(key, value, format="AVRO") - consume_avro_binary: - from: sensor_source_avro_binary - forEach: - code: log_message(key, value, format="OTHER_AVRO") consume_csv: from: sensor_source_csv forEach: From c73f638db396d5bf24b25431d842e72eacd3fbb6 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:17:53 +0100 Subject: [PATCH 48/55] Improved configuration syntax + introduced named notation factories # Conflicts: # ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java --- .../java/io/axual/ksml/runner/KSMLRunner.java | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java index 8aad5233..fcd9a66e 100644 --- a/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java +++ b/ksml-runner/src/main/java/io/axual/ksml/runner/KSMLRunner.java @@ -112,24 +112,27 @@ public static void main(String[] args) { // Set up all notation overrides from the KSML config for (final var notationEntry : ksmlConfig.notations().entrySet()) { - final var notation = notationEntry.getKey(); + final var notationStr = notationEntry.getKey() != null ? notationEntry.getKey() : "undefined"; final var notationConfig = notationEntry.getValue(); - if (notation != null && notationConfig != null) { - final var n = notationFactories.notations().get(notationConfig.type()); - if (n == null) { - throw FatalError.reportAndExit(new ConfigException("Notation type '" + notationConfig.type() + "' not found")); + final var factoryName = notationConfig != null ? notationConfig.type() : "unknown"; + if (notationConfig != null && factoryName != null) { + final var factory = notationFactories.notations().get(factoryName); + if (factory == null) { + throw FatalError.reportAndExit(new ConfigException("Unknown notation type: " + factoryName)); } - NotationLibrary.register(notation, n.create(notationConfig.config())); + NotationLibrary.register(notationStr, factory.create(notationConfig.config())); + } else { + log.warn("Notation configuration incomplete: notation=" + notationStr + ", type=" + factoryName); } } - // Ensure typical defaults are used - // WARNING: Defaults for notations will be deprecated in the future. Make sure you explicitly define - // notations that have multiple implementations in your ksml-runner.yaml. + // Ensure typical defaults are used for AVRO + // WARNING: Defaults for notations will be deprecated in the future. Make sure you explicitly configure + // notations with multiple implementations (like AVRO) in your ksml-runner.yaml. if (!NotationLibrary.exists(NotationFactories.AVRO)) { final var defaultAvro = notationFactories.confluentAvro(); NotationLibrary.register(NotationFactories.AVRO, defaultAvro.create(null)); - log.warn("No implementation specified for AVRO notation. If you plan to use AVRO, add the required implementation to the ksml-runner.yaml"); + log.warn("No implementation specified for AVRO notation. If you use AVRO in your KSML definition, add the required configuration to the ksml-runner.yaml"); } final var errorHandling = ksmlConfig.errorHandlingConfig(); From 4e77a5afeb5a8df856016cfb94fa5a1b509a52e7 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:21:55 +0100 Subject: [PATCH 49/55] Update apicurio serde version # Conflicts: # pom.xml From 0043445a8327407dc1aa4d53a25f8eead7ab24a9 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:26:50 +0100 Subject: [PATCH 50/55] Processed Richard's review comments for MR From 005f7d2dc5fbbb0597184144de6e7ca6bcb60519 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:29:36 +0100 Subject: [PATCH 51/55] Add header support to serialisation/deserialisation to allow Apicurio notation to use headers From 4aafe4b7647a1fe5f461eb7b88558d8cce2d7549 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 21:39:48 +0100 Subject: [PATCH 52/55] Unify lombok configurations From b3fa82c56e1bbe99caadb6cbf88a9a62b20f69fd Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:41:29 +0100 Subject: [PATCH 53/55] Finalize merge From 3ef9c6bb093c2739c1d540400de1b5f8eb604a4c Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 20:42:17 +0100 Subject: [PATCH 54/55] Finalize merge From 4486438c28db633013928b8f83cbdeef915dce26 Mon Sep 17 00:00:00 2001 From: jeroenvandisseldorp Date: Wed, 13 Nov 2024 22:00:07 +0100 Subject: [PATCH 55/55] Update variable names --- .../io/axual/ksml/runner/config/KSMLRunnerConfigTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java index b536e067..90da3a51 100644 --- a/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java +++ b/ksml-runner/src/test/java/io/axual/ksml/runner/config/KSMLRunnerConfigTest.java @@ -45,7 +45,7 @@ void shouldLoadWithoutExceptions() throws IOException { final var yaml = getClass().getClassLoader().getResourceAsStream("ksml-runner-config.yaml"); final var ksmlRunnerConfig = objectMapper.readValue(yaml, KSMLRunnerConfig.class); - assertNotNull(ksmlRunnerConfig.ksml()); - assertNotNull(ksmlRunnerConfig.kafka()); + assertNotNull(ksmlRunnerConfig.ksmlConfig()); + assertNotNull(ksmlRunnerConfig.kafkaConfig()); } }