Merge branch 'release/0.9.112'

.gitignore

@@ -31,3 +31,6 @@ node_modules/
 *.pdf
 *.zshrc
 hs_err_pid*.log
+*.interp
+*.tokens
+gen/

CHANGELOG.md

@@ -2,7 +2,22 @@
 
 ## [Unreleased](https://github.com/aklivity/zilla/tree/HEAD)
 
-[Full Changelog](https://github.com/aklivity/zilla/compare/0.9.110...HEAD)
+[Full Changelog](https://github.com/aklivity/zilla/compare/0.9.111...HEAD)
+
+**Fixed bugs:**
+
+- Docs search bar doesn't work [\#1351](https://github.com/aklivity/zilla/issues/1351)
+- InputMismatchException is thrown for `grpc` binding with googleapis/devtools/build proto [\#1230](https://github.com/aklivity/zilla/issues/1230)
+
+**Merged pull requests:**
+
+- Ztable support and convert transformation logic to state machine  [\#1352](https://github.com/aklivity/zilla/pull/1352) ([akrambek](https://github.com/akrambek))
+- Improve gRPC parser to allow complex types in optionValue [\#1350](https://github.com/aklivity/zilla/pull/1350) ([bmaidics](https://github.com/bmaidics))
+- Bump org.apache.avro:avro from 1.11.3 to 1.12.0 [\#1290](https://github.com/aklivity/zilla/pull/1290) ([dependabot[bot]](https://github.com/apps/dependabot))
+
+## [0.9.111](https://github.com/aklivity/zilla/tree/0.9.111) (2024-12-13)
+
+[Full Changelog](https://github.com/aklivity/zilla/compare/0.9.110...0.9.111)
 
 **Implemented enhancements:**
 

build/flyweight-maven-plugin/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>build</artifactId>
-    <version>0.9.111</version>
+    <version>0.9.112</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

build/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>zilla</artifactId>
-    <version>0.9.111</version>
+    <version>0.9.112</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

cloud/docker-image/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>cloud</artifactId>
-    <version>0.9.111</version>
+    <version>0.9.112</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

cloud/helm-chart/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>cloud</artifactId>
-    <version>0.9.111</version>
+    <version>0.9.112</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

cloud/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>zilla</artifactId>
-    <version>0.9.111</version>
+    <version>0.9.112</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

conf/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>zilla</artifactId>
-    <version>0.9.111</version>
+    <version>0.9.112</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

incubator/binding-amqp.spec/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>incubator</artifactId>
-    <version>0.9.111</version>
+    <version>0.9.112</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

incubator/binding-amqp/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>incubator</artifactId>
-    <version>0.9.111</version>
+    <version>0.9.112</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

incubator/binding-pgsql-kafka.spec/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>incubator</artifactId>
-    <version>0.9.111</version>
+    <version>0.9.112</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

incubator/binding-pgsql-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/pgsql/kafka/config/proxy.generated.as.yaml

@@ -0,0 +1,36 @@
+#
+# Copyright 2021-2024 Aklivity Inc
+#
+# Licensed under the Aklivity Community License (the "License"); you may not use
+# this file except in compliance with the License.  You may obtain a copy of the
+# License at
+#
+#   https://www.aklivity.io/aklivity-community-license/
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OF ANY KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations under the License.
+#
+
+---
+name: test
+catalogs:
+  catalog0:
+    type: test
+    options:
+      url: http://localhost:8081
+      id: 1
+      schema: |-
+        {
+            "schema": "{\"fields\":[{\"name\":\"id\",\"type\":[\"null\",{\"type\":\"string\"}]},{\"name\":\"city\",\"type\":[\"null\",{\"type\":\"string\"}]},{\"name\":\"owner_id\",\"type\":[\"null\",{\"type\":\"string\"}],\"zillaType\":\"ZILLA_IDENTITY\"},{\"name\":\"created_at\",\"type\":[\"null\",{\"type\":\"long\",\"logicalType\":\"timestamp-millis\"}],\"zillaType\":\"ZILLA_TIMESTAMP\"}],\"name\":\"cities\",\"namespace\":\"public\",\"type\":\"record\"}",
+            "schemaType": "AVRO"
+        }
+bindings:
+  app0:
+    type: pgsql-kafka
+    kind: proxy
+    catalog:
+      catalog0:
+        - strategy: topic
+    exit: app1

incubator/binding-risingwave.spec/src/main/scripts/io/aklivity/zilla/specs/binding/risingwave/streams/pgsql/create.table.with.primary.key.and.includes/client.rpt → incubator/binding-pgsql-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/pgsql/kafka/streams/pgsql/create.topic.with.generated.as/client.rpt

@@ -17,6 +17,7 @@ connect "zilla://streams/app0"
     option zilla:window 8192
     option zilla:transmission "duplex"
 
+
 write zilla:begin.ext ${pgsql:beginEx()
                               .typeId(zilla:id("pgsql"))
                               .parameter("user", "app")
@@ -32,18 +33,19 @@ write zilla:data.ext ${pgsql:dataEx()
                               .query()
                                 .build()
                               .build()}
-write "CREATE TABLE IF NOT EXISTS cities "
-      "(id VARCHAR, name VARCHAR, description VARCHAR,"
-      " zilla_correlation_id VARCHAR, zilla_identity VARCHAR, zilla_timestamp TIMESTAMP,"
-      " PRIMARY KEY (id));"
+write "CREATE TOPIC IF NOT EXISTS cities ("
+      "id VARCHAR, "
+      "city VARCHAR, "
+      "owner_id VARCHAR GENERATED ALWAYS AS IDENTITY, "
+      "created_at TIMESTAMP GENERATED ALWAYS AS NOW, "
+      "PRIMARY KEY (id, city));"
       [0x00]
-
 write flush
 
 read advised zilla:flush ${pgsql:flushEx()
                               .typeId(zilla:id("pgsql"))
                               .completion()
-                                .tag("CREATE_TABLE")
+                                .tag("CREATE_TOPIC")
                                 .build()
                               .build()}
 
@@ -54,3 +56,5 @@ read advised zilla:flush ${pgsql:flushEx()
                                 .build()
                               .build()}
 
+write close
+read closed

incubator/binding-pgsql-kafka.spec/src/main/scripts/io/aklivity/zilla/specs/binding/pgsql/kafka/streams/pgsql/create.topic.with.generated.as/server.rpt

@@ -0,0 +1,61 @@
+#
+# Copyright 2021-2024 Aklivity Inc
+#
+# Licensed under the Aklivity Community License (the "License"); you may not use
+# this file except in compliance with the License.  You may obtain a copy of the
+# License at
+#
+#   https://www.aklivity.io/aklivity-community-license/
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OF ANY KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations under the License.
+#
+
+accept "zilla://streams/app0"
+    option zilla:window 8192
+    option zilla:transmission "duplex"
+
+accepted
+
+read zilla:begin.ext ${pgsql:beginEx()
+                              .typeId(zilla:id("pgsql"))
+                              .parameter("user", "root")
+                              .parameter("database", "dev")
+                              .parameter("application_name", "psql")
+                              .parameter("client_encoding", "UTF8")
+                              .build()}
+
+connected
+
+read zilla:data.ext ${pgsql:dataEx()
+                              .typeId(zilla:id("pgsql"))
+                              .query()
+                                .build()
+                              .build()}
+read "CREATE TOPIC IF NOT EXISTS cities ("
+     "id VARCHAR, "
+     "city VARCHAR, "
+     "owner_id VARCHAR GENERATED ALWAYS AS IDENTITY, "
+     "created_at TIMESTAMP GENERATED ALWAYS AS NOW, "
+     "PRIMARY KEY (id, city));"
+     [0x00]
+
+write advise zilla:flush ${pgsql:flushEx()
+                              .typeId(zilla:id("pgsql"))
+                              .completion()
+                                .tag("CREATE_TOPIC")
+                                .build()
+                              .build()}
+
+write notify CREATE_TOPIC_COMPLETED
+
+write advise zilla:flush ${pgsql:flushEx()
+                              .typeId(zilla:id("pgsql"))
+                              .ready()
+                                .status("IDLE")
+                                .build()
+                              .build()}
+read closed
+write close

incubator/binding-pgsql-kafka/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>incubator</artifactId>
-    <version>0.9.111</version>
+    <version>0.9.112</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

incubator/binding-pgsql-kafka/src/main/java/io/aklivity/zilla/runtime/binding/pgsql/kafka/internal/schema/AvroField.java

@@ -16,6 +16,7 @@
 
 public record AvroField(
     String name,
-    Object type)
+    Object type,
+    Object zillaType)
 {
 }

incubator/binding-pgsql-kafka/src/main/java/io/aklivity/zilla/runtime/binding/pgsql/kafka/internal/schema/PgsqlKafkaAvroSchemaTemplate.java

@@ -15,6 +15,7 @@
 package io.aklivity.zilla.runtime.binding.pgsql.kafka.internal.schema;
 
 import java.util.List;
+import java.util.Map;
 
 import jakarta.json.Json;
 import jakarta.json.JsonArray;
@@ -25,13 +26,25 @@
 import jakarta.json.bind.JsonbBuilder;
 import jakarta.json.bind.JsonbConfig;
 
+import org.agrona.collections.Object2ObjectHashMap;
+
 import io.aklivity.zilla.runtime.binding.pgsql.parser.model.AlterExpression;
 import io.aklivity.zilla.runtime.binding.pgsql.parser.model.Operation;
 
 public abstract class PgsqlKafkaAvroSchemaTemplate
 {
     protected static final String DATABASE_PLACEHOLDER = "{database}";
 
+    protected static final String ZILLA_IDENTITY = "GENERATED ALWAYS AS IDENTITY";
+    protected static final String ZILLA_TIMESTAMP = "GENERATED ALWAYS AS NOW";
+
+    protected static final Map<String, String> ZILLA_MAPPINGS = new Object2ObjectHashMap<>();
+    static
+    {
+        ZILLA_MAPPINGS.put(ZILLA_IDENTITY, "ZILLA_IDENTITY");
+        ZILLA_MAPPINGS.put(ZILLA_TIMESTAMP, "ZILLA_TIMESTAMP");
+    }
+
     protected Jsonb jsonbFormatted = JsonbBuilder.create(new JsonbConfig().withFormatting(true));
     protected Jsonb jsonb = JsonbBuilder.create();
 

incubator/binding-pgsql-kafka/src/main/java/io/aklivity/zilla/runtime/binding/pgsql/kafka/internal/schema/PgsqlKafkaKeyAvroSchemaTemplate.java

@@ -36,7 +36,7 @@ public String generate(
         final String newNamespace = namespace.replace(DATABASE_PLACEHOLDER, database);
 
         List<AvroField> fields = command.columns().stream()
-            .map(column -> new AvroField(column.name(), mapSqlTypeToAvroType(column.type())))
+            .map(column -> new AvroField(column.name(), mapSqlTypeToAvroType(column.type()), null))
             .collect(Collectors.toList());
 
         AvroSchema schema = new AvroSchema("record", command.name(), newNamespace, fields);

incubator/binding-pgsql-kafka/src/main/java/io/aklivity/zilla/runtime/binding/pgsql/kafka/internal/schema/PgsqlKafkaValueAvroSchemaTemplate.java

@@ -49,12 +49,19 @@ public String generate(
                 String columnName = column.name();
                 String sqlType = column.type();
                 Object avroType = mapSqlTypeToAvroType(sqlType);
+                List<String> constraints = column.constraints();
 
-                boolean isNullable = !column.constraints().contains("NOT NULL");
+                boolean isNullable = !constraints.contains("NOT NULL");
+
+                String zillaType = ZILLA_MAPPINGS.entrySet().stream()
+                        .filter(e -> constraints.contains(e.getKey()))
+                        .map(Map.Entry::getValue)
+                        .findFirst()
+                        .orElse(null);
 
                 return isNullable
-                    ? new AvroField(columnName, new Object[]{"null", avroType})
-                    : new AvroField(columnName, avroType);
+                    ? new AvroField(columnName, new Object[]{"null", avroType}, zillaType)
+                    : new AvroField(columnName, avroType, zillaType);
             })
             .collect(Collectors.toList());
 

incubator/binding-pgsql-kafka/src/test/java/io/aklivity/zilla/runtime/binding/pgsql/kafka/internal/stream/ProxyIT.java

@@ -59,6 +59,17 @@ public void shouldCreateTopic() throws Exception
         k3po.finish();
     }
 
+    @Test
+    @Configuration("proxy.generated.as.yaml")
+    @Specification({
+        "${pgsql}/create.topic.with.generated.as/client",
+        "${kafka}/create.topic/server"
+    })
+    public void shouldCreateTopicWithGeneratedAs() throws Exception
+    {
+        k3po.finish();
+    }
+
     @Test
     @Configuration("proxy.alter.yaml")
     @Specification({

incubator/binding-pgsql.spec/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>io.aklivity.zilla</groupId>
     <artifactId>incubator</artifactId>
-    <version>0.9.111</version>
+    <version>0.9.112</version>
     <relativePath>../pom.xml</relativePath>
   </parent>