diff --git a/antora.yml b/antora.yml index 6eb3481..07823f3 100644 --- a/antora.yml +++ b/antora.yml @@ -18,7 +18,7 @@ asciidoc: pulsar: 'Apache Pulsar' pulsar-short: 'Pulsar' pulsar-reg: 'Apache Pulsar(TM)' - pulsar-version: '2.10' #DO NOT INCLUDE PATCH VERSION .. + pulsar-version: '3.1' #DO NOT INCLUDE PATCH VERSION .. debezium-version: '1.7' astra-streaming-examples-repo: 'https://raw.githubusercontent.com/datastax/astra-streaming-examples/master' kafka-for-astra: 'Starlight for Kafka' diff --git a/modules/pulsar-io/examples/connectors/sources/debezium-sqlserver/sample-data.adoc b/modules/pulsar-io/examples/connectors/sources/debezium-sqlserver/sample-data.adoc index 9837e16..c542a82 100644 --- a/modules/pulsar-io/examples/connectors/sources/debezium-sqlserver/sample-data.adoc +++ b/modules/pulsar-io/examples/connectors/sources/debezium-sqlserver/sample-data.adoc @@ -1,19 +1,38 @@ [source,json] ---- -{ - "database.hostname": "localhost", - "database.port": "1433", - "database.user": "sa", - "database.password": "MyP@ssw0rd!", - "database.dbname": "MyTestDB", - "database.server.name": "mssql", - "snapshot.mode": "schema_only", - "topic.namespace": "'$TENANT'/'$NAMESPACE'", - "task.class": "io.debezium.connector.sqlserver.SqlServerConnectorTask", - "value.converter": "org.apache.kafka.connect.json.JsonConverter", +"configs": { + "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", + "database.dbname": "cdc_test", + "database.history": "org.apache.pulsar.io.debezium.PulsarDatabaseHistory", + "database.history.name": "cdc_test_history", + "database.history.pulsar.service.url": "pulsar+ssl://pulsar-azure-westus2.streaming.datastax.com:6651", + "database.history.pulsar.token": "sensitive_data_removed", + "database.history.pulsar.topic": "dbz-stream672-history-topic", + "database.hostname": "kalash-server-stream672.database.windows.net", + "database.include.list": "cdc_test", + "database.names": "cdc_test", + "database.password": "sensitive_data_removed", + "database.port": 1433, + "database.server.name": "kalash-server-stream672", + "database.ssl": true, + "database.ssl.mode": "required", + "database.tcpKeepAlive": true, + "database.user": "debezium_user", + "decimal.handling.mode": "double", + "driver.encrypt": true, + "driver.trustServerCertificate": true, + "include.schema.changes": true, "key.converter": "org.apache.kafka.connect.json.JsonConverter", + "name": "dbz-stream672", + "pulsar.auth.token": "sensitive_data_removed", + "pulsar.service.url": "pulsar+ssl://pulsar-azure-westus2.streaming.datastax.com:6651", + "snapshot.mode": "always", + "table.include.list": "dbo.accounts", + "task.class": "io.debezium.connector.sqlserver.SqlServerConnectorTask", + "task.id": "0", + "topic.namespace": "kalash-stream672/default", + "topic.prefix": "kalash-server-stream672", "typeClassName": "org.apache.pulsar.common.schema.KeyValue", - "database.tcpKeepAlive": "true", - "decimal.handling.mode": "double" -} + "value.converter": "org.apache.kafka.connect.json.JsonConverter" + }, ---- \ No newline at end of file diff --git a/modules/pulsar-io/pages/connectors/sources/debezium-sqlserver.adoc b/modules/pulsar-io/pages/connectors/sources/debezium-sqlserver.adoc index 30b12ff..81e307a 100644 --- a/modules/pulsar-io/pages/connectors/sources/debezium-sqlserver.adoc +++ b/modules/pulsar-io/pages/connectors/sources/debezium-sqlserver.adoc @@ -12,6 +12,32 @@ https://debezium.io/releases/{debezium-version}/[Debezium documentation]. == Get Started +To create a Debezium SQL Server source connector, do the following. +Your deployment's values may differ. + +. Create an Azure SQL Server or compatible SQL Server instance with a database and table you want to capture changes from. +. Choose a connector name. +. For Debezium SQL Server source connectors, you must manually create the required topics in your Pulsar namespace, following Debezium's topic naming conventions: + + * `**CONNECTOR_NAME**-debezium-history-topic` + * `**CONNECTOR_NAME**-debezium-offset-topic` + * `**SERVER_NAME**` + * `**SERVER_NAME**.**DB_NAME**.**TABLE_NAME**` + + +You must create these topics manually. +Autocreation is not available. + +. In the source connector configuration, set the `topic.prefix` configuration value to match `database.server.name`. +In the example configuration, both are set to `kalash-server-stream672`. +. In the source connector configuration, ensure `task.id` is passed as a string (`"0"`), not an integer (`0`); otherwise, the connector will throw a `NullPointerException`. + +. Set your consumers to subscribe to the `events` topic in your Pulsar namespace to receive change data capture (CDC) events. +This topic contains the change data capture (CDC) events emitted by Debezium for the configured tables. + +For more on Debezium topic naming conventions, see the https://debezium.io/documentation/reference/stable/connectors/sqlserver.html#sqlserver-topic-names[Debezium documentation]. + + include::partial$connectors/sources/get-started.adoc[] == Managing the Connector