[Issue #13] Integrate Retina and add Debezium Engine as a data source.

.gitignore

@@ -22,7 +22,6 @@
 # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
 hs_err_pid*
 
-.idea
 build
 target
 .classpath*
@@ -48,3 +47,11 @@ resources/*.xml
 *.so
 *.o
 .vscode
+
+data/
+tmp/
+
+!requirements.txt
+freshness*.png
+rate*.png
+resulti7i/

conf/jvm.conf

@@ -0,0 +1,26 @@
+-server
+-XX:+AlwaysPreTouch
+-Dfile.encoding=UTF-8
+-Duser.timezone=UTC
+
+-Xms8g
+-Xmx90g
+
+-XX:+UseG1GC
+-XX:MaxGCPauseMillis=200
+-XX:InitiatingHeapOccupancyPercent=35
+-XX:+ParallelRefProcEnabled
+-XX:+UnlockExperimentalVMOptions
+-XX:+TrustFinalNonStaticFields
+-XX:+DisableExplicitGC
+
+-Xss512k
+
+
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:HeapDumpPath=/var/log/java/java_heapdump.hprof
+-XX:+ExitOnOutOfMemoryError
+
+-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:10086
+--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+--add-opens=java.base/java.nio=ALL-UNNAMED

conf/pixels-sink.aws.properties

@@ -0,0 +1,108 @@
+# engine | kafka | storage
+sink.datasource=engine
+# -1 means no limit, Only implement in retina sink mode yet
+sink.datasource.rate.limit=1000000
+# Sink Config: retina | csv | proto | flink | none
+sink.mode=proto
+sink.retina.client=1
+sink.retina.log.queue=false
+## batch or single or record, batch is recommend. record is faster, but doesn't have ACID feature
+sink.trans.mode=batch
+sink.monitor.report.enable=true
+sink.monitor.report.file=/home/ubuntu/pixels-sink/resulti7i_100/100k_rate.csv
+sink.monitor.freshness.file=/home/ubuntu/pixels-sink/resulti7i_100/100k_fresh.csv
+# trino for freshness query
+trino.url=jdbc:trino://realtime-kafka-2:8080/pixels/pixels_bench_sf100x
+# trino.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench_sf10x
+trino.user=pixels
+trino.password=password
+trino.parallel=4
+# row or txn or embed
+sink.monitor.freshness.level=row
+sink.monitor.freshness.embed.warmup=10
+sink.monitor.freshness.embed.static=false
+sink.monitor.freshness.embed.snapshot=true
+sink.monitor.freshness.embed.tablelist=loanapps,loantrans
+sink.monitor.freshness.verbose=true
+sink.monitor.freshness.timestamp=true
+sink.storage.loop=true
+# Kafka Config
+bootstrap.servers=realtime-kafka-2:29092
+group.id=3078
+auto.offset.reset=earliest
+key.deserializer=org.apache.kafka.common.serialization.StringDeserializer
+#value.deserializer=io.pixelsdb.pixels.writer.deserializer.RowChangeEventAvroDeserializer
+value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer
+# Topic & Database Config
+topic.prefix=postgresql.oltp_server
+consumer.capture_database=pixels_bench_sf1x
+consumer.include_tables=
+sink.csv.path=./data
+sink.csv.enable_header=false
+## Retina Config
+sink.retina.embedded=false
+# stub or stream
+sink.retina.mode=stream
+#writer.retina.mode=stub
+sink.remote.host=localhost
+sink.remote.port=29422
+sink.timeout.ms=5000
+sink.flush.interval.ms=50
+sink.flush.batch.size=10
+sink.max.retries=3
+## writer commit
+# sync or async
+sink.commit.method=sync
+sink.commit.batch.size=10
+sink.commit.batch.worker=32
+sink.commit.batch.delay=3000
+## Proto Config
+sink.proto.dir=file:///home/ubuntu/disk2/hybench/
+sink.proto.data=hybench1000_1
+# sink.proto.data=hybench100_3
+# sink.proto.data=hybench10_10
+sink.proto.maxRecords=100000
+## Flink Config
+sink.flink.server.port=9091
+## Schema Registry
+sink.registry.url=http://localhost:8080/apis/registry/v2
+# Transaction Config
+transaction.topic.suffix=transaction
+#transaction.topic.value.deserializer=io.pixelsdb.pixels.writer.deserializer.TransactionAvroMessageDeserializer
+transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.TransactionJsonMessageDeserializer
+sink.trans.batch.size=100
+
+# Sink Metrics
+sink.monitor.enable=true
+sink.monitor.port=9464
+sink.monitor.report.interval=10000
+sink.monitor.freshness.interval=1000
+
+# Interact with other rpc
+sink.rpc.enable=true
+sink.rpc.mock.delay=20
+# debezium engine config
+debezium.name=testEngine
+debezium.connector.class=io.debezium.connector.postgresql.PostgresConnector
+debezium.provide.transaction.metadata=true
+debezium.offset.storage=org.apache.kafka.connect.storage.FileOffsetBackingStore
+debezium.offset.storage.file.filename=/tmp/offsets.dat
+debezium.offset.flush.interval.ms=60000
+debezium.schema.history.internal=io.debezium.storage.file.history.FileSchemaHistory
+debezium.schema.history.internal.file.filename=/tmp/schemahistory.dat
+debezium.database.hostname=realtime-pg-2
+debezium.database.port=5432
+debezium.database.user=pixels
+debezium.database.password=pixels_realtime_crud
+debezium.database.dbname=pixels_bench
+debezium.plugin.name=pgoutput
+debezium.database.server.id=1
+debezium.schema.include.list=public
+debezium.snapshot.mode=never
+debezium.key.converter=org.apache.kafka.connect.json.JsonConverter
+debezium.value.converter=org.apache.kafka.connect.json.JsonConverter
+debezium.topic.prefix=postgresql.oltp_server
+debezium.transforms=topicRouting
+debezium.transforms.topicRouting.type=org.apache.kafka.connect.transforms.RegexRouter
+debezium.transforms.topicRouting.regex=postgresql\\.oltp_server\\.public\\.(.*)
+debezium.transforms.topicRouting.replacement=postgresql.oltp_server.pixels_bench_sf10x.$1

conf/pixels-sink.flink.properties

@@ -0,0 +1,46 @@
+# engine | kafka | storage
+sink.datasource=storage
+# -1 means no limit, Only implement in retina sink mode yet
+sink.datasource.rate.limit=50000
+# Sink Config: retina | csv | proto | flink | none
+sink.mode=flink
+## batch or single or record, batch is recommend. record is faster, but doesn't have ACID feature
+sink.trans.mode=batch
+sink.monitor.report.enable=true
+sink.monitor.report.file=/home/ubuntu/pixels-sink/tmp/test.csv
+sink.monitor.freshness.file=/home/ubuntu/pixels-sink/tmp/test_freshness.csv
+# trino for freshness query
+trino.url=jdbc:trino://realtime-kafka-2:8080/pixels/pixels_bench_sf10x
+# trino.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench_sf10x
+trino.user=pixels
+trino.password=password
+trino.parallel=1
+# row or txn or embed
+sink.monitor.freshness.level=row
+sink.monitor.freshness.embed.warmup=10
+sink.monitor.freshness.embed.static=false
+sink.monitor.freshness.embed.snapshot=true
+sink.monitor.freshness.embed.tablelist=loanapps,loantrans
+sink.monitor.freshness.verbose=false
+sink.monitor.freshness.timestamp=true
+sink.storage.loop=true
+
+sink.remote.host=localhost
+sink.remote.port=29422
+sink.timeout.ms=5000
+sink.flush.interval.ms=50
+sink.flush.batch.size=10
+sink.max.retries=3
+
+## Proto Config
+sink.proto.dir=file:///home/ubuntu/disk1/hybench/
+sink.proto.data=hybench10_10
+sink.proto.maxRecords=100000
+## Flink Config
+sink.flink.server.port=9091
+
+# Sink Metrics
+sink.monitor.enable=true
+sink.monitor.port=9465
+sink.monitor.report.interval=10000
+sink.monitor.freshness.interval=1000