Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

(draft) feat: added historical plans w.r.t. new processor apis #10663

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

(draft) feat: added historical plans w.r.t. new processor apis #10663

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
The diff you're trying to view is too large. We only load the first 3000 changed files.
6 changes: 5 additions & 1 deletion ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/model/TestHeader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,11 @@ public void serialize(
) throws IOException {
jsonGenerator.writeStartObject();
jsonGenerator.writeObjectField("KEY", record.key);
jsonGenerator.writeBinaryField("VALUE", record.value);
if (record.value == null) {
jsonGenerator.writeNullField("VALUE");
} else {
jsonGenerator.writeBinaryField("VALUE", record.value);
}
jsonGenerator.writeEndObject();
}
}
Expand Down
211 changes: 211 additions & 0 deletions ...t/resources/historical_plans/array-min-max-sort_-_array_max/8.0.0_1737614475683/plan.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
{
"plan" : [ {
"@type" : "ksqlPlanV1",
"statementText" : "CREATE STREAM INPUT (ID STRING KEY, BOOL_ARRAY ARRAY<BOOLEAN>, INT_ARRAY ARRAY<INTEGER>, BIGINT_ARRAY ARRAY<BIGINT>, DOUBLE_ARRAY ARRAY<DOUBLE>, STRING_ARRAY ARRAY<STRING>, DECIMAL_ARRAY ARRAY<DECIMAL(2, 1)>) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');",
"ddlCommand" : {
"@type" : "createStreamV1",
"sourceName" : "INPUT",
"schema" : "`ID` STRING KEY, `BOOL_ARRAY` ARRAY<BOOLEAN>, `INT_ARRAY` ARRAY<INTEGER>, `BIGINT_ARRAY` ARRAY<BIGINT>, `DOUBLE_ARRAY` ARRAY<DOUBLE>, `STRING_ARRAY` ARRAY<STRING>, `DECIMAL_ARRAY` ARRAY<DECIMAL(2, 1)>",
"timestampColumn" : null,
"topicName" : "test_topic",
"formats" : {
"keyFormat" : {
"format" : "KAFKA",
"properties" : { }
},
"valueFormat" : {
"format" : "JSON",
"properties" : { }
}
},
"windowInfo" : null,
"orReplace" : false,
"isSource" : false
},
"queryPlan" : null
}, {
"@type" : "ksqlPlanV1",
"statementText" : "CREATE STREAM OUTPUT AS SELECT\n INPUT.ID ID,\n ARRAY_MAX(INPUT.BOOL_ARRAY) BOOL_MAX,\n ARRAY_MAX(INPUT.INT_ARRAY) INT_MAX,\n ARRAY_MAX(INPUT.BIGINT_ARRAY) BIGINT_MAX,\n ARRAY_MAX(INPUT.DOUBLE_ARRAY) DOUBLE_MAX,\n ARRAY_MAX(INPUT.STRING_ARRAY) STRING_MAX,\n ARRAY_MAX(INPUT.DECIMAL_ARRAY) DECIMAL_MAX\nFROM INPUT INPUT\nEMIT CHANGES",
"ddlCommand" : {
"@type" : "createStreamV1",
"sourceName" : "OUTPUT",
"schema" : "`ID` STRING KEY, `BOOL_MAX` BOOLEAN, `INT_MAX` INTEGER, `BIGINT_MAX` BIGINT, `DOUBLE_MAX` DOUBLE, `STRING_MAX` STRING, `DECIMAL_MAX` DECIMAL(2, 1)",
"timestampColumn" : null,
"topicName" : "OUTPUT",
"formats" : {
"keyFormat" : {
"format" : "KAFKA",
"properties" : { }
},
"valueFormat" : {
"format" : "JSON",
"properties" : { }
}
},
"windowInfo" : null,
"orReplace" : false,
"isSource" : false
},
"queryPlan" : {
"sources" : [ "INPUT" ],
"sink" : "OUTPUT",
"physicalPlan" : {
"@type" : "streamSinkV1",
"properties" : {
"queryContext" : "OUTPUT"
},
"source" : {
"@type" : "streamSelectV1",
"properties" : {
"queryContext" : "Project"
},
"source" : {
"@type" : "streamSourceV1",
"properties" : {
"queryContext" : "KsqlTopic/Source"
},
"topicName" : "test_topic",
"formats" : {
"keyFormat" : {
"format" : "KAFKA",
"properties" : { }
},
"valueFormat" : {
"format" : "JSON",
"properties" : { }
}
},
"timestampColumn" : null,
"sourceSchema" : "`ID` STRING KEY, `BOOL_ARRAY` ARRAY<BOOLEAN>, `INT_ARRAY` ARRAY<INTEGER>, `BIGINT_ARRAY` ARRAY<BIGINT>, `DOUBLE_ARRAY` ARRAY<DOUBLE>, `STRING_ARRAY` ARRAY<STRING>, `DECIMAL_ARRAY` ARRAY<DECIMAL(2, 1)>",
"pseudoColumnVersion" : 1
},
"keyColumnNames" : [ "ID" ],
"selectedKeys" : null,
"selectExpressions" : [ "ARRAY_MAX(BOOL_ARRAY) AS BOOL_MAX", "ARRAY_MAX(INT_ARRAY) AS INT_MAX", "ARRAY_MAX(BIGINT_ARRAY) AS BIGINT_MAX", "ARRAY_MAX(DOUBLE_ARRAY) AS DOUBLE_MAX", "ARRAY_MAX(STRING_ARRAY) AS STRING_MAX", "ARRAY_MAX(DECIMAL_ARRAY) AS DECIMAL_MAX" ]
},
"formats" : {
"keyFormat" : {
"format" : "KAFKA",
"properties" : { }
},
"valueFormat" : {
"format" : "JSON",
"properties" : { }
}
},
"topicName" : "OUTPUT",
"timestampColumn" : null
},
"queryId" : "CSAS_OUTPUT_0",
"runtimeId" : null
}
} ],
"configs" : {
"ksql.extension.dir" : "ext",
"ksql.streams.cache.max.bytes.buffering" : "0",
"metric.reporters" : "",
"ksql.query.status.running.threshold.seconds" : "300",
"ksql.connect.basic.auth.credentials.reload" : "false",
"ksql.output.topic.name.prefix" : "",
"ksql.query.pull.stream.enabled" : "true",
"ksql.query.push.v2.interpreter.enabled" : "true",
"ksql.queryanonymizer.logs_enabled" : "true",
"ksql.variable.substitution.enable" : "true",
"ksql.streams.shutdown.timeout.ms" : "300000",
"ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807",
"ksql.query.pull.max.qps" : "2147483647",
"ksql.access.validator.enable" : "auto",
"ksql.streams.bootstrap.servers" : "localhost:0",
"ksql.query.pull.metrics.enabled" : "true",
"ksql.metrics.extension" : null,
"ksql.query.push.v2.alos.enabled" : "true",
"ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647",
"ksql.query.pull.range.scan.enabled" : "true",
"ksql.transient.query.cleanup.service.initial.delay.seconds" : "600",
"ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses",
"ksql.lambdas.enabled" : "true",
"ksql.source.table.materialization.enabled" : "true",
"ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647",
"ksql.client.ip_port.configuration.enabled" : "false",
"ksql.sink.window.change.log.additional.retention" : "1000000",
"ksql.deployment.type" : "selfManaged",
"ksql.query.persistent.active.limit" : "2147483647",
"ksql.websocket.connection.max.timeout.ms" : "3600000",
"ksql.persistence.wrap.single.values" : null,
"ksql.query.transient.max.bytes.buffering.total" : "-1",
"ksql.connect.basic.auth.credentials.source" : "NONE",
"ksql.schema.registry.url" : "schema_registry.url:0",
"ksql.properties.overrides.denylist" : "",
"ksql.service.id" : "some.ksql.service.id",
"ksql.query.push.v2.max.catchup.consumers" : "5",
"ksql.assert.topic.default.timeout.ms" : "1000",
"ksql.query.pull.forwarding.timeout.ms" : "20000",
"ksql.query.push.v2.enabled" : "false",
"ksql.transient.query.cleanup.service.enable" : "true",
"ksql.query.push.v2.metrics.enabled" : "true",
"ksql.streams.commit.interval.ms" : "2000",
"ksql.query.pull.table.scan.enabled" : "true",
"ksql.streams.auto.commit.interval.ms" : "0",
"ksql.streams.topology.optimization" : "all",
"ksql.endpoint.migrate.query" : "true",
"ksql.query.push.v2.registry.installed" : "false",
"ksql.streams.num.stream.threads" : "4",
"ksql.metrics.tags.custom" : "",
"ksql.query.push.v2.catchup.consumer.msg.window" : "50",
"ksql.runtime.feature.shared.enabled" : "false",
"ksql.udf.collect.metrics" : "false",
"ksql.new.query.planner.enabled" : "false",
"ksql.connect.request.headers.plugin" : null,
"ksql.security.extension.class" : null,
"ksql.transient.prefix" : "transient_",
"ksql.headers.columns.enabled" : "true",
"enable.fips" : "false",
"ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler",
"ksql.connect.request.timeout.ms" : "5000",
"ksql.query.pull.enable.standby.reads" : "false",
"ksql.persistence.default.format.key" : "KAFKA",
"ksql.query.persistent.max.bytes.buffering.total" : "-1",
"ksql.query.error.max.queue.size" : "10",
"ksql.query.cleanup.shutdown.timeout.ms" : "30000",
"ksql.internal.topic.min.insync.replicas" : "1",
"ksql.internal.topic.replicas" : "1",
"ksql.insert.into.values.enabled" : "true",
"ksql.queryanonymizer.cluster_namespace" : null,
"ksql.create.or.replace.enabled" : "true",
"ksql.shared.runtimes.count" : "2",
"ksql.cast.strings.preserve.nulls" : "true",
"ksql.authorization.cache.max.entries" : "10000",
"ksql.pull.queries.enable" : "true",
"ksql.transient.query.cleanup.service.period.seconds" : "600",
"ksql.suppress.enabled" : "true",
"ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses",
"ksql.connect.basic.auth.credentials.file" : "",
"ksql.authorization.cache.expiry.time.secs" : "30",
"ksql.query.retry.backoff.initial.ms" : "15000",
"ksql.fetch.remote.hosts.max.timeout.seconds" : "10",
"ksql.query.pull.max.concurrent.requests" : "2147483647",
"ksql.streams.auto.offset.reset" : "earliest",
"ksql.connect.url" : "http://localhost:8083",
"ksql.query.push.v2.new.latest.delay.ms" : "5000",
"ksql.query.push.v2.latest.reset.age.ms" : "30000",
"ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler",
"ksql.query.pull.interpreter.enabled" : "true",
"ksql.json_sr.converter.deserializer.enabled" : "true",
"ksql.assert.schema.default.timeout.ms" : "1000",
"ksql.query.pull.limit.clause.enabled" : "true",
"ksql.query.pull.router.thread.pool.size" : "50",
"ksql.query.push.v2.continuation.tokens.enabled" : "false",
"ksql.query.retry.backoff.max.ms" : "900000",
"ksql.timestamp.throw.on.invalid" : "false",
"ksql.persistence.default.format.value" : null,
"ksql.udfs.enabled" : "true",
"ksql.udf.enable.security.manager" : "true",
"ksql.connect.worker.config" : "",
"ksql.nested.error.set.null" : "true",
"ksql.query.pull.thread.pool.size" : "50",
"ksql.persistent.prefix" : "query_",
"ksql.metastore.backup.location" : "",
"ksql.error.classifier.regex" : "",
"ksql.suppress.buffer.size.bytes" : "-1"
}
}
152 changes: 152 additions & 0 deletions ...t/resources/historical_plans/array-min-max-sort_-_array_max/8.0.0_1737614475683/spec.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
{
"version" : "8.0.0",
"timestamp" : 1737614475683,
"path" : "query-validation-tests/array-min-max-sort.json",
"schemas" : {
"CSAS_OUTPUT_0.KsqlTopic.Source" : {
"schema" : "`ID` STRING KEY, `BOOL_ARRAY` ARRAY<BOOLEAN>, `INT_ARRAY` ARRAY<INTEGER>, `BIGINT_ARRAY` ARRAY<BIGINT>, `DOUBLE_ARRAY` ARRAY<DOUBLE>, `STRING_ARRAY` ARRAY<STRING>, `DECIMAL_ARRAY` ARRAY<DECIMAL(2, 1)>",
"keyFormat" : {
"format" : "KAFKA"
},
"valueFormat" : {
"format" : "JSON"
}
},
"CSAS_OUTPUT_0.OUTPUT" : {
"schema" : "`ID` STRING KEY, `BOOL_MAX` BOOLEAN, `INT_MAX` INTEGER, `BIGINT_MAX` BIGINT, `DOUBLE_MAX` DOUBLE, `STRING_MAX` STRING, `DECIMAL_MAX` DECIMAL(2, 1)",
"keyFormat" : {
"format" : "KAFKA"
},
"valueFormat" : {
"format" : "JSON"
}
}
},
"testCase" : {
"name" : "array_max",
"inputs" : [ {
"topic" : "test_topic",
"key" : "r1",
"value" : {
"bool_array" : [ false, true, false ],
"int_array" : [ 0, 0, 1, 0, -1 ],
"bigint_array" : [ 234, -123, 345 ],
"double_array" : [ 0.0, 0.1, -12345.678, 0.2, 0.3 ],
"string_array" : [ "foo", "bar" ],
"decimal_array" : [ 1.0, 1.1, 1.2, -0.2, 1.9, 9.0, -9.9, 1.5 ]
}
}, {
"topic" : "test_topic",
"key" : "r2",
"value" : {
"bool_array" : [ null, false, true ],
"int_array" : [ 0, null, 1, 0, -1 ],
"bigint_array" : [ null, -123, 345 ],
"double_array" : [ 0.0, 0.1, -12345.678, null, 0.3 ],
"string_array" : [ "foo", "fo", "Food", null, "F", "food" ],
"decimal_array" : [ 1.0, 1.1, 1.2, -0.2, null, 9.0 ]
}
}, {
"topic" : "test_topic",
"key" : "r3",
"value" : {
"bool_array" : [ ],
"int_array" : [ ],
"bigint_array" : [ ],
"double_array" : [ ],
"string_array" : [ ],
"decimal_array" : [ ]
}
} ],
"outputs" : [ {
"topic" : "OUTPUT",
"key" : "r1",
"value" : {
"BOOL_MAX" : true,
"INT_MAX" : 1,
"BIGINT_MAX" : 345,
"DOUBLE_MAX" : 0.3,
"STRING_MAX" : "foo",
"DECIMAL_MAX" : 9.0
}
}, {
"topic" : "OUTPUT",
"key" : "r2",
"value" : {
"BOOL_MAX" : true,
"INT_MAX" : 1,
"BIGINT_MAX" : 345,
"DOUBLE_MAX" : 0.3,
"STRING_MAX" : "food",
"DECIMAL_MAX" : 9.0
}
}, {
"topic" : "OUTPUT",
"key" : "r3",
"value" : {
"BOOL_MAX" : null,
"INT_MAX" : null,
"BIGINT_MAX" : null,
"DOUBLE_MAX" : null,
"STRING_MAX" : null,
"DECIMAL_MAX" : null
}
} ],
"topics" : [ {
"name" : "OUTPUT",
"replicas" : 1,
"numPartitions" : 4
}, {
"name" : "test_topic",
"replicas" : 1,
"numPartitions" : 4
} ],
"statements" : [ "CREATE STREAM INPUT (ID STRING KEY, bool_array ARRAY<BOOLEAN>, int_array ARRAY<INT>, bigint_array ARRAY<BIGINT>, double_array ARRAY<DOUBLE>, string_array ARRAY<STRING>, decimal_array ARRAY<DECIMAL(2,1)>) WITH (kafka_topic='test_topic', value_format='JSON');", "CREATE STREAM OUTPUT AS SELECT ID, array_max(bool_array) as bool_max, array_max(int_array) as int_max, array_max(bigint_array) as bigint_max, array_max(double_array) as double_max, array_max(string_array) as string_max, array_max(decimal_array) as decimal_max FROM INPUT;" ],
"post" : {
"sources" : [ {
"name" : "INPUT",
"type" : "STREAM",
"schema" : "`ID` STRING KEY, `BOOL_ARRAY` ARRAY<BOOLEAN>, `INT_ARRAY` ARRAY<INTEGER>, `BIGINT_ARRAY` ARRAY<BIGINT>, `DOUBLE_ARRAY` ARRAY<DOUBLE>, `STRING_ARRAY` ARRAY<STRING>, `DECIMAL_ARRAY` ARRAY<DECIMAL(2, 1)>",
"keyFormat" : {
"format" : "KAFKA"
},
"valueFormat" : "JSON",
"keyFeatures" : [ ],
"valueFeatures" : [ ],
"isSource" : false
}, {
"name" : "OUTPUT",
"type" : "STREAM",
"schema" : "`ID` STRING KEY, `BOOL_MAX` BOOLEAN, `INT_MAX` INTEGER, `BIGINT_MAX` BIGINT, `DOUBLE_MAX` DOUBLE, `STRING_MAX` STRING, `DECIMAL_MAX` DECIMAL(2, 1)",
"keyFormat" : {
"format" : "KAFKA"
},
"valueFormat" : "JSON",
"keyFeatures" : [ ],
"valueFeatures" : [ ],
"isSource" : false
} ],
"topics" : {
"topics" : [ {
"name" : "test_topic",
"keyFormat" : {
"format" : "KAFKA"
},
"valueFormat" : {
"format" : "JSON"
},
"partitions" : 4
}, {
"name" : "OUTPUT",
"keyFormat" : {
"format" : "KAFKA"
},
"valueFormat" : {
"format" : "JSON"
},
"partitions" : 4
} ]
}
}
}
}
16 changes: 16 additions & 0 deletions ...st/resources/historical_plans/array-min-max-sort_-_array_max/8.0.0_1737614475683/topology
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
Topologies:
Sub-topology: 0
Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic])
--> KSTREAM-PROCESSVALUES-0000000001
Processor: KSTREAM-PROCESSVALUES-0000000001 (stores: [])
--> Project, KSTREAM-PEEK-0000000002
<-- KSTREAM-SOURCE-0000000000
Processor: Project (stores: [])
--> KSTREAM-SINK-0000000003
<-- KSTREAM-PROCESSVALUES-0000000001
Processor: KSTREAM-PEEK-0000000002 (stores: [])
--> none
<-- KSTREAM-PROCESSVALUES-0000000001
Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT)
<-- Project

Loading