From beff9669daa5240fb3e32a9523c891fd72f82e5a Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 14 Oct 2025 18:27:22 -0700 Subject: [PATCH] quick clean --- .../functions/pages/astream-functions.adoc | 1 - modules/functions/pages/cast.adoc | 1 - modules/functions/pages/compute.adoc | 1 - modules/functions/pages/deploy-in-sink.adoc | 3 +- modules/functions/pages/drop-fields.adoc | 1 - modules/functions/pages/drop.adoc | 1 - modules/functions/pages/flatten.adoc | 1 - modules/functions/pages/merge-key-value.adoc | 1 - modules/functions/pages/unwrap-key-value.adoc | 1 - modules/pulsar-io/pages/connectors/index.adoc | 1 - .../pages/connectors/sinks/astra-db.adoc | 5 ++-- .../pages/connectors/sinks/cloud-storage.adoc | 1 - .../connectors/sinks/elastic-search.adoc | 1 - .../connectors/sinks/google-bigquery.adoc | 11 ++----- .../connectors/sinks/jdbc-clickhouse.adoc | 1 - .../pages/connectors/sinks/jdbc-mariadb.adoc | 1 - .../pages/connectors/sinks/jdbc-postgres.adoc | 1 - .../pages/connectors/sinks/jdbc-sqllite.adoc | 1 - .../pages/connectors/sinks/kafka.adoc | 1 - .../pages/connectors/sinks/kinesis.adoc | 1 - .../pages/connectors/sinks/snowflake.adoc | 1 - .../connectors/sources/data-generator.adoc | 1 - .../connectors/sources/debezium-mongodb.adoc | 1 - .../connectors/sources/debezium-mysql.adoc | 1 - .../connectors/sources/debezium-oracle.adoc | 1 - .../connectors/sources/debezium-postgres.adoc | 1 - .../sources/debezium-sqlserver.adoc | 1 - .../pages/connectors/sources/kafka.adoc | 1 - .../pages/connectors/sources/kinesis.adoc | 1 - .../astream-subscriptions-exclusive.adoc | 1 - .../pages/astream-subscriptions-failover.adoc | 1 - .../astream-subscriptions-keyshared.adoc | 1 - .../pages/astream-subscriptions-shared.adoc | 1 - modules/subscriptions/pages/index.adoc | 1 - .../partials/subscription-prereq.adoc | 2 +- .../images/csharp-icon.png | Bin 3219 -> 0 bytes .../images/golang-icon.png | Bin 3804 -> 0 bytes .../images/java-icon.png | Bin 4487 -> 0 bytes .../images/node-icon.png | Bin 5084 -> 0 bytes .../images/python-icon.png | Bin 3189 -> 0 bytes .../consuming-change-data.adoc | 16 +++++----- .../pages/change-data-capture/index.adoc | 28 +++++++----------- .../questions-and-patterns.adoc | 4 +-- .../pages/starlight/jms/index.adoc | 1 - 44 files changed, 27 insertions(+), 74 deletions(-) delete mode 100644 modules/use-cases-architectures/images/csharp-icon.png delete mode 100644 modules/use-cases-architectures/images/golang-icon.png delete mode 100644 modules/use-cases-architectures/images/java-icon.png delete mode 100644 modules/use-cases-architectures/images/node-icon.png delete mode 100644 modules/use-cases-architectures/images/python-icon.png diff --git a/modules/functions/pages/astream-functions.adoc b/modules/functions/pages/astream-functions.adoc index ad5c3eb..ec04dbf 100644 --- a/modules/functions/pages/astream-functions.adoc +++ b/modules/functions/pages/astream-functions.adoc @@ -1,6 +1,5 @@ = {pulsar-reg} functions :navtitle: {pulsar-short} functions -:page-tag: astra-streaming,dev,develop,pulsar,java,python Functions are lightweight compute processes that enable you to process each message received on a topic. You can apply custom logic to that message, transforming or enriching it, and then output it to a different topic. diff --git a/modules/functions/pages/cast.adoc b/modules/functions/pages/cast.adoc index 9e679bb..0a21815 100644 --- a/modules/functions/pages/cast.adoc +++ b/modules/functions/pages/cast.adoc @@ -1,6 +1,5 @@ = Cast :functionName: cast -:page-tag: cast, transform-function The cast transform function transforms the data to a target compatible schema. diff --git a/modules/functions/pages/compute.adoc b/modules/functions/pages/compute.adoc index b98b737..1e1518b 100644 --- a/modules/functions/pages/compute.adoc +++ b/modules/functions/pages/compute.adoc @@ -1,6 +1,5 @@ = Compute :functionName: compute -:page-tag: compute, transform-function The `compute` transform function computes field values based on an `expression` evaluated at runtime. + If the field already exists, it will be overwritten. + diff --git a/modules/functions/pages/deploy-in-sink.adoc b/modules/functions/pages/deploy-in-sink.adoc index 48bd53b..1a4e037 100644 --- a/modules/functions/pages/deploy-in-sink.adoc +++ b/modules/functions/pages/deploy-in-sink.adoc @@ -1,7 +1,6 @@ = Deploy transform function in sink -:page-tag: cast, transform-function -As of https://www.datastax.com/products/luna-streaming[Luna Streaming] version 2.10.1.6, transform functions can be deployed inside of a sink process. + +As of https://www.ibm.com/docs/en/supportforpulsar[IBM Elite Support for Apache Pulsar (formerly Luna Streaming)] version 2.10.1.6, transform functions can be deployed inside of a sink process. + Before this update, functions transformed data either after it was written to a topic by a source connector, or before it was read from a topic by a sink connector. + This required either an intermediate topic, with additional storage, IO, and latency, or a custom connector. + Now, functions can be deployed at sink creation and apply preprocessing to sink topic writes. + diff --git a/modules/functions/pages/drop-fields.adoc b/modules/functions/pages/drop-fields.adoc index 856b37e..c9baf3c 100644 --- a/modules/functions/pages/drop-fields.adoc +++ b/modules/functions/pages/drop-fields.adoc @@ -1,7 +1,6 @@ = Drop fields :functionName: drop-fields -:page-tag: drop-fields, transform-function The {functionName} transform function drops fields of structured data (Currently only AVRO is supported). + The cast transform function transforms the data to a target compatible schema. diff --git a/modules/functions/pages/drop.adoc b/modules/functions/pages/drop.adoc index 9aaece8..9161db8 100644 --- a/modules/functions/pages/drop.adoc +++ b/modules/functions/pages/drop.adoc @@ -1,6 +1,5 @@ = Drop :functionName: drop -:page-tag: drop, transform-function The {functionName} transform function drops a record from further processing. + Use in conjunction with `when` to selectively drop records. + diff --git a/modules/functions/pages/flatten.adoc b/modules/functions/pages/flatten.adoc index fca3046..7b23574 100644 --- a/modules/functions/pages/flatten.adoc +++ b/modules/functions/pages/flatten.adoc @@ -1,6 +1,5 @@ = Flatten :functionName: flatten -:page-tag: flatten, transform-function The {functionName} transform function converts structured, nested data into a new single-hierarchy-level structured data. + The names of the new fields are built by concatenating the intermediate level field names. + diff --git a/modules/functions/pages/merge-key-value.adoc b/modules/functions/pages/merge-key-value.adoc index c007d37..8c06acb 100644 --- a/modules/functions/pages/merge-key-value.adoc +++ b/modules/functions/pages/merge-key-value.adoc @@ -1,6 +1,5 @@ = Merge KeyValue :functionName: merge-key-value -:page-tag: merge-key-value, transform-function The {functionName} transform function merges the fields of KeyValue records where both the key and value are structured types of the same schema type. (Currently only AVRO is supported). + The step name is `merge-key-value` and the `UserConfig` is controlled here: `{"steps": [{"type": "merge-key-value"}]}`. diff --git a/modules/functions/pages/unwrap-key-value.adoc b/modules/functions/pages/unwrap-key-value.adoc index 677d6d2..3dc87df 100644 --- a/modules/functions/pages/unwrap-key-value.adoc +++ b/modules/functions/pages/unwrap-key-value.adoc @@ -1,6 +1,5 @@ = Unwrap KeyValue :functionName: unwrap-key-value -:page-tag: unwrap-key-value, transform-function If the record value is a KeyValue, the {functionName} transform function extracts the KeyValue's key or value and makes it the record value. + The step name is `unwrap-key-value`, and the `UserConfig` is controlled here: `{"steps": [{"type": "unwrap-key-value"}]}`. diff --git a/modules/pulsar-io/pages/connectors/index.adoc b/modules/pulsar-io/pages/connectors/index.adoc index 3b08dfc..119d70f 100644 --- a/modules/pulsar-io/pages/connectors/index.adoc +++ b/modules/pulsar-io/pages/connectors/index.adoc @@ -1,6 +1,5 @@ = Connectors :navtitle: Connector Overview -:page-tag: connectors,sinks,sources,astra-streaming,dev,develop,pulsar,go {product} offers fully-managed {pulsar-reg} connectors. diff --git a/modules/pulsar-io/pages/connectors/sinks/astra-db.adoc b/modules/pulsar-io/pages/connectors/sinks/astra-db.adoc index 92b4885..0c2d9a5 100644 --- a/modules/pulsar-io/pages/connectors/sinks/astra-db.adoc +++ b/modules/pulsar-io/pages/connectors/sinks/astra-db.adoc @@ -1,9 +1,8 @@ = {astra-db} ({cass-reg} enhanced) :connectorName: astra-db-sink :connectorType: astra-db -:page-tag: astra-db,cdc,sink-connector -{company} {astra-db} Sink Connector is based on the open-source https://docs.datastax.com/en/pulsar-connector/docs/index.html[{cass-reg} sink connector for {pulsar-reg}]. Depending on how you deploy the connector, it can be used to sink topic messages with a table in {astra-db} or a table in a {cass-short} cluster outside of DB. +{company} {astra-db} Sink Connector is based on the open-source xref:pulsar-connector:ROOT:index.adoc[{cass-reg} sink connector for {pulsar-reg}]. Depending on how you deploy the connector, it can be used to sink topic messages with a table in {astra-db} or a table in a {cass-short} cluster outside of DB. The {product} portal provides simple way to connect this sink and a table in {astra-db} with simply a token. Using `pulsar-admin` or the REST API, you can configure the sink to connect with a {cass-short} connection manually. @@ -140,7 +139,7 @@ These values are provided in the `auth` area of the preceding {cass-short} conne These values are provided in the `topic` area of the preceding {cass-short} connection parameters. -Refer to the official documentation for a https://docs.datastax.com/en/pulsar-connector/docs/cfgRefPulsarDseConnection.html[connection properties reference]. +Refer to the official documentation for a xref:pulsar-connector:ROOT:cfgRefPulsarDseConnection.adoc[connection properties reference]. === Mapping topic data to table columns diff --git a/modules/pulsar-io/pages/connectors/sinks/cloud-storage.adoc b/modules/pulsar-io/pages/connectors/sinks/cloud-storage.adoc index b8b74f6..68979f9 100644 --- a/modules/pulsar-io/pages/connectors/sinks/cloud-storage.adoc +++ b/modules/pulsar-io/pages/connectors/sinks/cloud-storage.adoc @@ -1,7 +1,6 @@ = Cloud Storage :connectorName: cloud-storage-sink :connectorType: cloud-storage -:page-tag: cloud storage,sink-connector, aws, azure, gcp Each public cloud has different ways of persisting data to their storage systems. Each cloud has their own way of formatting and storing the bytes. The Cloud Storage sink connector is a general interface to a chosen cloud storage, that exports data from a {pulsar-short} topic to the given system following a desired format. diff --git a/modules/pulsar-io/pages/connectors/sinks/elastic-search.adoc b/modules/pulsar-io/pages/connectors/sinks/elastic-search.adoc index e8ec874..416c922 100644 --- a/modules/pulsar-io/pages/connectors/sinks/elastic-search.adoc +++ b/modules/pulsar-io/pages/connectors/sinks/elastic-search.adoc @@ -1,7 +1,6 @@ = Elasticsearch :connectorName: es-sink :connectorType: elastic-search -:page-tag: elasticsearch,sink-connector Elasticsearch is the distributed, RESTful search and analytics engine at the heart of the Elastic Stack. diff --git a/modules/pulsar-io/pages/connectors/sinks/google-bigquery.adoc b/modules/pulsar-io/pages/connectors/sinks/google-bigquery.adoc index 4937456..6a0682f 100644 --- a/modules/pulsar-io/pages/connectors/sinks/google-bigquery.adoc +++ b/modules/pulsar-io/pages/connectors/sinks/google-bigquery.adoc @@ -1,19 +1,12 @@ = Google BigQuery :connectorName: bigquery-sink :connectorType: bigquery -:page-tag: bigquery,sink-connector - https://cloud.google.com/bigquery[Google BigQuery] is a fully managed enterprise data warehouse that helps you manage and analyze your data with built-in features like machine learning, geospatial analysis, and business intelligence. BigQuery's serverless architecture lets you use SQL queries to answer your organization's biggest questions with zero infrastructure management. BigQuery's scalable, distributed analysis engine lets you query terabytes in seconds and petabytes in minutes. -BigQuery {pulsar-short} Sink is not integrated with BigQuery directly. It uses {pulsar-short}’s built-in https://pulsar.apache.org/docs/adaptors-kafka/[Kafka Connect adapter] library to transform message data into a Kafka compatible format. Then the https://docs.confluent.io/kafka-connectors/bigquery/current/kafka_connect_bigquery_config.html[Kafka Connect BigQuery Sink] is used as the actual BigQuery integration. The adaptor provides a flexible and extensible framework for data transformation and processing. It supports various data formats, including JSON, Avro, and Protobuf, and enables users to apply transformations on the data as it is being streamed from {pulsar-short}. - -You will notice references to Kafka throughout the configuration. *You don’t need a running instance of Kafka to use this connector.* The Kafka references are used as "translation points” by this connector. +BigQuery {pulsar-short} Sink is not integrated with BigQuery directly. It uses {pulsar-short}'s built-in https://pulsar.apache.org/docs/adaptors-kafka/[Kafka Connect adapter] library to transform message data into a Kafka compatible format. Then the https://docs.confluent.io/kafka-connectors/bigquery/current/kafka_connect_bigquery_config.html[Kafka Connect BigQuery Sink] is used as the actual BigQuery integration. The adaptor provides a flexible and extensible framework for data transformation and processing. It supports various data formats, including JSON, Avro, and Protobuf, and enables users to apply transformations on the data as it is being streamed from {pulsar-short}. -[NOTE] -==== -For more information on the Kafka Connect adapter, see https://www.datastax.com/blog/simplify-migrating-kafka-to-pulsar-kafka-connect-support[Simplify migrating from Kafka to {pulsar-short} with Kafka Connect Support] and the "https://medium.com/building-the-open-data-stack/datastax-presents-snowflake-sink-connector-for-apache-pulsar-53629b196064[{company} Snowflake Sink Connector for {pulsar-reg}]. -==== +You will notice references to Kafka throughout the configuration. *You don't need a running instance of Kafka to use this connector.* The Kafka references are used as "translation points" by this connector. == Get Started diff --git a/modules/pulsar-io/pages/connectors/sinks/jdbc-clickhouse.adoc b/modules/pulsar-io/pages/connectors/sinks/jdbc-clickhouse.adoc index d8fc3c1..c1cc7c6 100644 --- a/modules/pulsar-io/pages/connectors/sinks/jdbc-clickhouse.adoc +++ b/modules/pulsar-io/pages/connectors/sinks/jdbc-clickhouse.adoc @@ -1,7 +1,6 @@ = JDBC Clickhouse :connectorName: jdbc-clickhouse :connectorType: jdbc-clickhouse -:page-tag: jdbc,clickhouse,sink-connector ClickHouse is an open-source column-oriented database management system for online analytical processing that allows users to generate analytical reports using SQL queries in real-time. diff --git a/modules/pulsar-io/pages/connectors/sinks/jdbc-mariadb.adoc b/modules/pulsar-io/pages/connectors/sinks/jdbc-mariadb.adoc index b83aeeb..dfcca38 100644 --- a/modules/pulsar-io/pages/connectors/sinks/jdbc-mariadb.adoc +++ b/modules/pulsar-io/pages/connectors/sinks/jdbc-mariadb.adoc @@ -1,7 +1,6 @@ = JDBC MariaDB :connectorName: jdbc-mariadb :connectorType: jdbc-mariadb -:page-tag: jdbc,mariadb,sink-connector MariaDB is the open source relational database loved by developers all over the world. Created by MySQL’s original developers, MariaDB is compatible with MySQL and guaranteed to stay open source forever. MariaDB powers some of the world’s most popular websites such as Wikipedia and WordPress.com. It is also the core engine behind banking, social media, mobile and e-commerce sites worldwide. MariaDB Connector/J is a Type 4 JDBC driver. It was developed specifically as a lightweight JDBC connector for use with MariaDB and MySQL database servers. It was originally based on the Drizzle JDBC code with numerous additions and bug fixes. Learn more about MariaDB on https://mariadb.org/[their site]. diff --git a/modules/pulsar-io/pages/connectors/sinks/jdbc-postgres.adoc b/modules/pulsar-io/pages/connectors/sinks/jdbc-postgres.adoc index 5845bc6..f7f1c8e 100644 --- a/modules/pulsar-io/pages/connectors/sinks/jdbc-postgres.adoc +++ b/modules/pulsar-io/pages/connectors/sinks/jdbc-postgres.adoc @@ -1,7 +1,6 @@ = JDBC PostgreSQL :connectorName: jdbc-postgres :connectorType: jdbc-postgres -:page-tag: jdbc,postgres,sink-connector PostgreSQL is a powerful, open source, object-relational database system with over 30 years of active development. diff --git a/modules/pulsar-io/pages/connectors/sinks/jdbc-sqllite.adoc b/modules/pulsar-io/pages/connectors/sinks/jdbc-sqllite.adoc index a928cc9..7a7ee1c 100644 --- a/modules/pulsar-io/pages/connectors/sinks/jdbc-sqllite.adoc +++ b/modules/pulsar-io/pages/connectors/sinks/jdbc-sqllite.adoc @@ -1,7 +1,6 @@ = JDBC SQLite :connectorName: jdbc-sqlite :connectorType: jdbc-sqlite -:page-tag: jdbc,sqlite,sink-connector SQLite is the most used database engine in the world. diff --git a/modules/pulsar-io/pages/connectors/sinks/kafka.adoc b/modules/pulsar-io/pages/connectors/sinks/kafka.adoc index 2ac5fde..59aeaa2 100644 --- a/modules/pulsar-io/pages/connectors/sinks/kafka.adoc +++ b/modules/pulsar-io/pages/connectors/sinks/kafka.adoc @@ -1,7 +1,6 @@ = Kafka :connectorName: kafka-sink :connectorType: kafka -:page-tag: kafka,sink-connector Apache Kafka(R) is an open-source distributed event streaming platform used by thousands of companies for high-performance data pipelines, streaming analytics, data integration, and mission-critical applications. diff --git a/modules/pulsar-io/pages/connectors/sinks/kinesis.adoc b/modules/pulsar-io/pages/connectors/sinks/kinesis.adoc index 0ad2ef9..c9aaa95 100644 --- a/modules/pulsar-io/pages/connectors/sinks/kinesis.adoc +++ b/modules/pulsar-io/pages/connectors/sinks/kinesis.adoc @@ -1,7 +1,6 @@ = Kinesis :connectorName: kinesis-sink :connectorType: kinesis -:page-tag: kinesis,sink-connector Amazon Kinesis collects, processes, and analyzes real-time streaming data for timely insights and quick reactions to new information. diff --git a/modules/pulsar-io/pages/connectors/sinks/snowflake.adoc b/modules/pulsar-io/pages/connectors/sinks/snowflake.adoc index 3235255..73ec8e0 100644 --- a/modules/pulsar-io/pages/connectors/sinks/snowflake.adoc +++ b/modules/pulsar-io/pages/connectors/sinks/snowflake.adoc @@ -1,7 +1,6 @@ = Snowflake :connectorName: snowflake-sink :connectorType: snowflake -:page-tag: snowflake,sink-connector A Snowflake database is where an organization's uploaded structured and semi-structured data sets are held for processing and analysis. diff --git a/modules/pulsar-io/pages/connectors/sources/data-generator.adoc b/modules/pulsar-io/pages/connectors/sources/data-generator.adoc index 3f40e2d..df248e0 100644 --- a/modules/pulsar-io/pages/connectors/sources/data-generator.adoc +++ b/modules/pulsar-io/pages/connectors/sources/data-generator.adoc @@ -1,7 +1,6 @@ = Data Generator :connectorName: data-gen-src :connectorType: data-generator -:page-tag: data-generator,source-connector The Data Generator source connector creates fake data on an {pulsar-reg} topic using the https://github.com/Codearte/jfairy[JFAIRY library] to generate a message containing "person" data. diff --git a/modules/pulsar-io/pages/connectors/sources/debezium-mongodb.adoc b/modules/pulsar-io/pages/connectors/sources/debezium-mongodb.adoc index 0327440..d6966e6 100644 --- a/modules/pulsar-io/pages/connectors/sources/debezium-mongodb.adoc +++ b/modules/pulsar-io/pages/connectors/sources/debezium-mongodb.adoc @@ -1,7 +1,6 @@ = Debezium MongoDB :connectorName: debezium-mongo-src :connectorType: debezium-mongodb -:page-tag: debezium,cdc,mongodb,source-connector Debezium’s MongoDB connector tracks a MongoDB replica set or a MongoDB sharded cluster for document changes in databases and collections and records those changes as messages in an {pulsar-reg} topic. diff --git a/modules/pulsar-io/pages/connectors/sources/debezium-mysql.adoc b/modules/pulsar-io/pages/connectors/sources/debezium-mysql.adoc index 2a20622..d51dd47 100644 --- a/modules/pulsar-io/pages/connectors/sources/debezium-mysql.adoc +++ b/modules/pulsar-io/pages/connectors/sources/debezium-mysql.adoc @@ -1,7 +1,6 @@ = Debezium MySQL :connectorName: debezium-mysql-src :connectorType: debezium-mysql -:page-tag: mysql,debezium,cdc,source-connector The Debezium MySQL connector reads the binlog, produces change events for row-level INSERT, UPDATE, and DELETE operations, and emits these change events as messages in an {pulsar-reg} topic. diff --git a/modules/pulsar-io/pages/connectors/sources/debezium-oracle.adoc b/modules/pulsar-io/pages/connectors/sources/debezium-oracle.adoc index 099280f..316faad 100644 --- a/modules/pulsar-io/pages/connectors/sources/debezium-oracle.adoc +++ b/modules/pulsar-io/pages/connectors/sources/debezium-oracle.adoc @@ -1,7 +1,6 @@ = Debezium Oracle :connectorName: debezium-orcl-src :connectorType: debezium-oracle -:page-tag: oracle,debezium,cdc,source-connector Debezium’s Oracle connector captures and records row-level changes that occur in databases on Oracle servers, including tables that are added while the connector is running. diff --git a/modules/pulsar-io/pages/connectors/sources/debezium-postgres.adoc b/modules/pulsar-io/pages/connectors/sources/debezium-postgres.adoc index 4cba4cf..26a92cc 100644 --- a/modules/pulsar-io/pages/connectors/sources/debezium-postgres.adoc +++ b/modules/pulsar-io/pages/connectors/sources/debezium-postgres.adoc @@ -1,7 +1,6 @@ = Debezium PostgreSQL :connectorName: debezium-pg-src :connectorType: debezium-postgres -:page-tag: postgres,debezium,cdc,source-connector The PostgreSQL connector produces a change event for every row-level insert, update, and delete operation that it captures, and sends change event records for each table in a separate {pulsar-reg} topic. diff --git a/modules/pulsar-io/pages/connectors/sources/debezium-sqlserver.adoc b/modules/pulsar-io/pages/connectors/sources/debezium-sqlserver.adoc index 81e307a..28fcf79 100644 --- a/modules/pulsar-io/pages/connectors/sources/debezium-sqlserver.adoc +++ b/modules/pulsar-io/pages/connectors/sources/debezium-sqlserver.adoc @@ -1,7 +1,6 @@ = Debezium SQL Server :connectorName: debezium-mssql-src :connectorType: debezium-sqlserver -:page-tag: sql-server,cdc,debezium,source-connector The Debezium SQL Server connector is based on the change data capture feature available in SQL Server 2016 Service Pack 1 (SP1) and later editions. diff --git a/modules/pulsar-io/pages/connectors/sources/kafka.adoc b/modules/pulsar-io/pages/connectors/sources/kafka.adoc index 0b10b9e..287f4a9 100644 --- a/modules/pulsar-io/pages/connectors/sources/kafka.adoc +++ b/modules/pulsar-io/pages/connectors/sources/kafka.adoc @@ -1,7 +1,6 @@ = Kafka :connectorName: kafka-src :connectorType: kafka -:page-tag: kafka,source-connector The Kafka source connector pulls data from a Kafka topic and persists the data into an {pulsar-reg} topic. diff --git a/modules/pulsar-io/pages/connectors/sources/kinesis.adoc b/modules/pulsar-io/pages/connectors/sources/kinesis.adoc index 64497a0..bb601ea 100644 --- a/modules/pulsar-io/pages/connectors/sources/kinesis.adoc +++ b/modules/pulsar-io/pages/connectors/sources/kinesis.adoc @@ -1,7 +1,6 @@ = Kinesis :connectorName: kinesis-src :connectorType: kinesis -:page-tag: kinesis,source-connector The Kinesis source connector pulls data from Amazon Kinesis and persists data into an {pulsar-reg} topic. diff --git a/modules/subscriptions/pages/astream-subscriptions-exclusive.adoc b/modules/subscriptions/pages/astream-subscriptions-exclusive.adoc index 7dbb782..7677cb1 100644 --- a/modules/subscriptions/pages/astream-subscriptions-exclusive.adoc +++ b/modules/subscriptions/pages/astream-subscriptions-exclusive.adoc @@ -1,6 +1,5 @@ = Exclusive subscriptions in {pulsar-reg} :navtitle: Exclusive -:page-tag: pulsar-subscriptions,quickstart,admin,dev,pulsar *Subscriptions* in {pulsar-reg} describe which consumers are consuming data from a topic and how they want to consume that data. + diff --git a/modules/subscriptions/pages/astream-subscriptions-failover.adoc b/modules/subscriptions/pages/astream-subscriptions-failover.adoc index 80bca04..bcd6e6c 100644 --- a/modules/subscriptions/pages/astream-subscriptions-failover.adoc +++ b/modules/subscriptions/pages/astream-subscriptions-failover.adoc @@ -1,6 +1,5 @@ = Failover subscriptions in {pulsar-reg} :navtitle: Failover -:page-tag: pulsar-subscriptions,quickstart,admin,dev,pulsar *Subscriptions* in {pulsar-reg} describe which consumers are consuming data from a topic and how they want to consume that data. + diff --git a/modules/subscriptions/pages/astream-subscriptions-keyshared.adoc b/modules/subscriptions/pages/astream-subscriptions-keyshared.adoc index c40fe07..a3edc9f 100644 --- a/modules/subscriptions/pages/astream-subscriptions-keyshared.adoc +++ b/modules/subscriptions/pages/astream-subscriptions-keyshared.adoc @@ -1,6 +1,5 @@ = Key_Shared subscriptions in {pulsar-reg} :navtitle: Key Shared -:page-tag: pulsar-subscriptions,quickstart,admin,dev,pulsar *Subscriptions* in {pulsar-reg} describe which consumers are consuming data from a topic and how they want to consume that data. + diff --git a/modules/subscriptions/pages/astream-subscriptions-shared.adoc b/modules/subscriptions/pages/astream-subscriptions-shared.adoc index aec8d20..2d85077 100644 --- a/modules/subscriptions/pages/astream-subscriptions-shared.adoc +++ b/modules/subscriptions/pages/astream-subscriptions-shared.adoc @@ -1,6 +1,5 @@ = Shared subscriptions in {pulsar-reg} :navtitle: Shared -:page-tag: pulsar-subscriptions,quickstart,admin,dev,pulsar *Subscriptions* in {pulsar-reg} describe which consumers are consuming data from a topic and how they want to consume that data. + diff --git a/modules/subscriptions/pages/index.adoc b/modules/subscriptions/pages/index.adoc index 4bd1694..1d6e4c6 100644 --- a/modules/subscriptions/pages/index.adoc +++ b/modules/subscriptions/pages/index.adoc @@ -1,6 +1,5 @@ = Subscriptions in {pulsar-reg} :navtitle: Overview -:page-tag: pulsar-subscriptions,quickstart,admin,dev,pulsar *Subscriptions* in {pulsar-reg} describe which consumers are consuming data from a topic and how they want to consume that data. + diff --git a/modules/subscriptions/partials/subscription-prereq.adoc b/modules/subscriptions/partials/subscription-prereq.adoc index b232f54..6cacdae 100644 --- a/modules/subscriptions/partials/subscription-prereq.adoc +++ b/modules/subscriptions/partials/subscription-prereq.adoc @@ -6,7 +6,7 @@ To run this example, you'll need: * https://openjdk.java.net/install/[Java OpenJDK 11] -* A configured {product} instance with at least *one streaming tenant* and *one topic*. See the https://docs.datastax.com/en/astra-streaming/docs/astream-quick-start.html[{product} quick start] for instructions. +* A configured {product} instance with at least one streaming tenant and one topic. See the xref:astra-streaming:getting-started:index.adoc[{product} quick start] for instructions. * A local clone of the https://github.com/datastax/pulsar-subscription-example[{company} {pulsar-short} Subscription Example repository] diff --git a/modules/use-cases-architectures/images/csharp-icon.png b/modules/use-cases-architectures/images/csharp-icon.png deleted file mode 100644 index ba77d2b03306e0f41e1bb94324f0f950cde4f2af..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3219 zcmV;E3~cj>P)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!~g&e!~vBn4jTXf3@AxNK~!i%#hM9l zRP`Olzq^}kP6D|goJlwW3HN1yP(ejGG*}R=t>DF|WASKh9cQd<9ouPfdI%m>QFIXN zQ51^cfyx;<0+Aye3AxC@W|KX#_xbkd|M$;}kg&V&ZL+bS8JKzh_wwHR{{O%8|5g!X zpq~(2K4?zrp{+E)W%eL#XbOhU9E_Za8Biz{;#IMyUqtzZ02lJz%V%J2wjtoZ@(7h$ ziM+W(G5f(Hq~{Lk7jZEcW!JfI^!2l7JgkM__KBm{l9Zl+2}^RhxVQwBI2s#bCdzH~ zqWbeje806C9=jlpN|TY7hFSL)B5(ds#HnK=uFs-;Zash$^RTu$(3{&SRSFE5 zn2D0drXXwFAb#8z`XtKSWW%A2r_p-afPgO`j`kjPN<7BiFdW5?OhVGY1aUNadL_!P zcj5ew2Athq2d^`{-}IK$!O58YKq1E7I0ErWapI_aq7vn?c~QIfGLFA<0d9*2=#$|_ zhN2=x$9a!V#_$>0ENaU4H`$^zr}Q}b##!iV%m`i;qWc?)jH9j_f|AFlVn9x^I2xJA zqSzTzlNF~wu0{1%O+Tubo~TpfG41XGOkAFiq(PDWEvzV)$-{i69+jUmzlkD5_a_P# z6jO&JV=gnv+&MX6{p}i2!9WlVhgxxb^Ldz9HPD}a(_55rO5_w}pyY|E$SxQpj$Yvh zQS`r|#)OJ3)o47T<>Gn;n@uv0%+UieaCi#RhNp7VQLo=Vy%CGgaL|or z+%q0ie>RR+-+jf|;X(PE=TP%)OH>t#f<0OY2uK^1gkQY53`$+K2|jQ1CPz_d>0jqz z`s(p95kw?sBw)eQ(=q??DcoNqVWt;Q_+fQeVKrN!zhr_*V}+6&HQJz%oSDd%dieA# z9%f?+MYf{bM>AnlBbu^;X?6)tSS%KJ-CkZu``O}9W2%f+P<(P6Zg^$}22V&AM`906 z)P!ZZn9fZ5nkJGR3PCSS789H{CyN7D%#_D*_Ae^Cp;;_`Xd)|Ha>SATgT&0ZrvT$_ z8YvEtsri_xSzxtLqzvUUoeLC|s3C>vC|s7uAMj&9MJz>C^}skdTy~hv76=}e*CdQf zwkV2p^H>un%Yhjb$u8__#NI!jz_zvfuyxg!*s<;)$~K(gtv)rEQrHbnoZ3+ZLz@9^ zms_^)D=I&!3zo4)PYN=*C4ZSCnTIHW*6+KFBmX+f+LKNP?g@3aRB0@HdIpA0&b(&j zDQ{Kpy@Y)mkMq10>K>t>ihj-8rSey*s5DPf5Y1%yrt>)P_mg}}EZoablBZp!FP}Y% zhO$<1v=i+G13rHIEA0BiLDq~qWed6*S)!6N5;1D7ba1HtqKTJUey_a5Nlj%k>k^QXnS{*zK`=I1piYbz zuk4PrD5fQgmG_Z09mztNZ}P+!4FM&q2JBX7>vh~*dy+eADDHiy6qA;XLrQug?tXPC zR&Tlu1G18lH+vYC{H`Qqn=dI!$;6~)Cy7U)5u5_r&Kboc3E12O6#=+>4wyPj5WL|- zw8!p+vC)jgG&S0)jnJOc!Q5)&e%4-NgsH{aIio$06h$kPD6w7Tb@*VpY!{Eh6I3u0 z^1ER&Q#HV)Eh;D zZXB@}Vyoj(IALRjr_ExIs@BZR9mq=|eZ7fKLfViN?sqmFZJdf1^*~Y-{h`!LP!I{} zb8KZ?eg~Q@4XD*zgwd@-Kqapupk%*>`BUki7V>~`*UO9X@F%M<|KS-($wL1)*(sbqekiX0^=mo=Se zmH4G;CSi%Y9kbFh(B9HARZS)$w_BGc8VQ{wo>BSC&+@U){|SBnm_5rIJW zm#1U1%><)M2d~N%wsoO4;>f#aVA0yqez5~zuRn?dubyDnE4W{M{o+yRSg;6%EKyRs zO1uXu&4(jkLxb@mw0aE$A@b^a*?EIi4|Bi>MN%+iwL+2CX>zj_YUO^F%))wGl>wK^ zHR2VakrYK}=>vd-UyZzOFmg)L;BrN`$KV#+xbZi$na?ai_Sp1pzDZGT=X3jUl&u&+ zRmau`cR^pfsOO&y}0(+ZNJPHk&Md?$Eke$m4)F_y_Y%*popMo(ZBY2`+^wb>g zH(ih!b~S=zHMI2*F6wn5OB6Zn*&X$gHyb#30G@pR0Zdw0z(q!aKCoTCx)`h1uV5*o zbEh?FP$G)&os6{~u0+w|F}?o6sCx9SC!TLEZxav0<21VYx1+0PF%(Gy5;1-0B#bD` zfvMHP^B2uDnN^%R4ry7bD43Iv`~J2Hb63q^zI5%2Co^&5Z^}ddoRPeukXx;jZ%{QXYYJ0b^-SkZXqGIqVR zmz$o)RPBckdRH52G*z(JEMcAV%-0w2(I0ogVeX$5 zYi}^%t#uzl*BH8D@cXF`vq0z6vZ}=W!_tshD_Ig(4jReIo46n>z7Ev3SE0?=%Ks=5 zB({9}DZI9JE86P~vLcfl{4AJMmDS;u``$&%`R;ocp%4TCMw=c+uMU2dz$*@FAeFS! z>dmmBrM-z?Bo`9Wc)A4})@;V%k1O~K^#$_RZ7+O|O~3pA9V`{dh2F#CaYJu!$LK{v z`ImkEeM(!W$L1$Kf~HzJUxk7a?Ud3E0Ml+M!jREfNJvrh z4H8Pe{6ibdUF7@}a4L3H@)Y55hTb2Pg2ZHvO2^_y7ozy5(|-6FP`7Wsloy(rEr0fp zZ*bvItvK3yNDx&Ri75$O6q%^U;ep-cWR-@zW-%O8g)O{yE*3pB9~r~CfA{Ssstc;f zD?U4eJ#QR>wpJIjKoAR5tQ0DoKMqUQE#OH#6d^@*;j+1L=>2jW+kA_O;w+*lg3giy?|I`Oj(&PF>dnenkig8h^j9~bc*QiNW~7Lt;gJ;8 z1^L^BvRdqU{UGX2H2o+h8J`fxqFNDdeey=+jL8;9rI9VF3!le_lV6_0?v49cDbvBl z#u`x&q%o5DJZ88#5d-t2a zUW@9Y`CKb@ZTJ?|ti18{Axx8i;uX`eblpPULHC6|iy}x&?Xd=IU%v;fHSJNY9Q9lU ztk+uk>{9MGw1d=FVj`*wsuarqdm6i059G8;?qNQqiztFju;siKyP3afF1Ewx^&ufyjR_@V zvHYn;NX<-<+;QqH{U)jl5@%_*@j{mBK-x`;)Qa~V;D2Ck6BBE1ZQLxAY_vg;$?V;oX(lt{{G@3qJ`_O^!KmPq;Qw2}Cn0L9?Tam?khm;cmfj@-EE=vY<#k5o0f^m&*sM6pBI0EL{uRZtU4R5Z z`^X8`V)zNfJ8YaF;u6EP09P2E5Ad?jnUOMxUqvh^s*L``+lU9}BlcPZ2XpY-jyyG} zttleQMaed+noc9ry@UIIwng1*#M^^UkFzj+6yF25_LnMhmf?pGpH<}CbcnAmX|u%; z|7E#p3GpKUUAw%6atiT2#A8)^yd3Cssj4^8z$>TJnP(8+gLrkpJj%xIMEtceUz`ec zeo~wa<_{hK;NmJRuzL3+9o0iVC08s$)vACv7|KExH6jmbd4gn8^0FkllzkRT6fd|P0n zrwSVAXrbt(XrcJk;OyVRDf#{Yw^VBW$m{6QFhZvPBk<6w!tZERBfJcLd1$M3&)TehHS09YRtkN(Z(GDi^F2;vKBX;HY+Q*UEnTqUvgh@A9J5aYSek}G;8j! zqp=F+X_rylpF&#LEj%b>g+0Q~5^H*5N-?MwSLAe5T{?GXqX;9WfX;*4pXw6&V*io?m|<6;W0GAb4f<-njQ>$>Q>9MPr@ zwU=n+0I+*9IKpp9{{0u*d8K7@p9VPmCp-6AwmA>rtOL;dXv;Q$XXCx^0IWWyc=l3w zC&Xgk0yx!!!@*Zi8qcdA>-AfBZYACT*nGWS|NH%Vee(m<-vGFv{{#Oecnig2*2@3@ z061k>NoGw=04e|g00;m9hiL!=000010000Q0000000N)_00aO40096105+fl00aO4 z0096106G8w000$x;hz8i3PnjoK~!i3)mm$86xS7=JMW!+d+oK?Z^uvSCv3rp5X_@c zpj1$`Dos;bqVz|l(x?xmmHMNtR0*X1K#CAm`>To6fAcO#RZEP@(jltOX zwXo~gzISGJ9zE-|*XxzNJHQI1_lNh+%(>@$_sn_Cxkjq0Vqflof4Sks(ET4kB%fc4 zQY1X*_09PGvw>hhjAEF|kfffXECzk9*=)61F^YN~Dd?GPamM|mYw*Es&-92O%@LHS z<0Ohf3Nj5CxWx4 zmtkmJRgnU2r(y}`BS}$~qr4!=k}Qg>+qFsIU*Ec!Vc6cm`#{%NU1iR(X#uSX;+@{R z7yJ6XvRZ966wuU}nZQ(G850hf!a=0SK~{%5idmfwlm{XaLQxfqjttcb!77sh?%ekW zA6V>))n@GN6z1lHwd)lzYIcqOa(%P2r6nWE%n+~KxOsfo#Wby>7-sy^zgeS+D=c2? zezd>7uA;CIM`+ybK6|^b$6jJ6Z~#hx$=<#7W>p#{|SzA||OaSwD{`t{)Q-RTDyZPH+@iWug{`&6h_}EWbW$%_( zQYcd{GX$M+kH??q2n`S2`qLjnu8Eu$r(t~f)lIGMojKFf-JRftDEj-YPUIRF6}1UT ze87L;z=6}JPbUaq(pFbPOnbmq8-DU;-dZPEEl$y*tR&N{IL`pltk>gyuc@lK?uT#G z9eN`w@RbJc-23Ogd+^|o4j-PLo=z~$&&{<+QBe>|O@_BmoVak|!r8ND1wlv&ekoOfJcw_!tTa=?-# zDT?D1%Z>&@+qZ5lD=XWxdp8-k#d+aRW@aJ;1@aJ%6FQE*_Mf*Js!K{sS3{P#pDA?X z_~G5VimqNg?h+BrkcxaisvO#}-V-4-|6Wwn$0-A zpn_;9^pEadyrxN2Wbk>?V*P=QYi(&xEKf7Z1QC|vcvX41&M#q#g2}R4QvR1acmJ?! ztHqYPz*mtwdhYz~iFfpN_rymZ;CjPvUf%N7zTKOuD-8y{D2hYl6Q}O<50+Hw8AeqU z&^>9gJamtwftCD714Ima36>{R#B z-^W?!F!;&V&8J))29Xyqg#Go?sh#vC|KJ>&!@x$d18VP1_ls{@TSDA)h8K(PJAJO6Up%70Cz;-1JNvmuiWrZe)EP+&6T1kxy++YVGy|K5ayK`u0 zc<>${6*-n#4PSf>R80^{Q_!qDQICWt1E!=rPM6;c*2zf{%ui^~WOmO)Jme-icYaz-gvnrsoW0^<2h5K}m3?NSp& zj%8W2RP$#=I203E)uP@=0mQ68TVMjwA3^%l!ZL#`@`Bjg^Iy+!e?-ueIKni!pcc81 z1o2Nl{QHxKO=~yqu-U9>)}-4L@w(iI=URVu++$ij{hL;L6z( zUY|b=<8%s|cc**3`Y#^r8(c1AWA7 z7GnrkBjJ3bG5t>BPc=czF_};ln%LQU*Epg=ffD;tzanBGQ;Qo9AHW?0uozZY@b(Ka=S|QpC3Jw-ngOd{| zsjI0`MlzX9kouv4A8+2Mr6(3&s1+hmA86as=AW9GbU|l3S*pf4)D03rc|ZI3+Xk|* zq(tK?AbSdkqhsUs!TxUyydUiAk#RDnPHm;;q_KD%N?5HL!X3Lki|0qDp=gI=1O&GSxIdp)ga zVB3buvBxu=V`%T8U+Y*+Z$nd#2H76hsMkFbDXR-k&H1o`qp$9kNV;QWtncxt3z1=( zk!e~XfE&0xbRQK$j*^jZjv|#4*m{S3!&f&OtvNTZUG5&dB18gH9vQm#7^VrLRxSGd z{&VlXU2Y8W#;?4x=b(X$H9uoRmf$OF+8>zl`Gb*&7?lYEaSUTI8f*rG&1#{S_hCD4 zT%CN-^-{}@rsm8?YjFu$Ap$j2pl-Ljq_`-)@`e3<3WzTZQ;MLc@aUA4FYFQ>Ir8&) zUj%<#N^Y^{_WX!_2tnr{HnLeUrr{QkWAj26Btmqv!Vv?Uc|ltjegP2-tJM;oAJ7tm zr-BfLr}4#EQQrKKaE#NUV&d9~UoG4N7$6#ik{rjuthu?7rYXPQM^g-R$!lvX;P^ch z^wW%PeB9OE-UVkZ*@6fMgXLwVaM};5(&-pM5aCEh5To$l0TfAadOaL&A_VOj^k8Th z!YVSEiU6BE1ZQLxAY_vg;$?V;oX(lt{{G@3qJ`_O^!KmPq;Qw2}Cn0L9?Tam?khm;cmfj@-EE=vY<#k5o0f^m&*sM6pBI0EL{uRZtU4R5Z z`^X8`V)zNfJ8YaF;u6EP09P2E5Ad?jnUOMxUqvh^s*L``+lU9}BlcPZ2XpY-jyyG} zttleQMaed+noc9ry@UIIwng1*#M^^UkFzj+6yF25_LnMhmf?pGpH<}CbcnAmX|u%; z|7E#p3GpKUUAw%6atiT2#A8)^yd3Cssj4^8z$>TJnP(8+gLrkpJj%xIMEtceUz`ec zeo~wa<_{hK;NmJRuzL3+9o0iVC08s$)vACv7|KExH6jmbd4gn8^0FkllzkRT6fd|P0n zrwSVAXrbt(XrcJk;OyVRDf#{Yw^VBW$m{6QFhZvPBk<6w!tZERBfJcLd1$M3&)TehHS09YRtkN(Z(GDi^F2;vKBX;HY+Q*UEnTqUvgh@A9J5aYSek}G;8j! zqp=F+X_rylpF&#LEj%b>g+0Q~5^H*5N-?MwSLAe5T{?GXqX;9WfX;*4pXw6&V*io?m|<6;W0GAb4f<-njQ>$>Q>9MPr@ zwU=n+0I+*9IKpp9{{0u*d8K7@p9VPmCp-6AwmA>rtOL;dXv;Q$XXCx^0IWWyc=l3w zC&Xgk0yx!!!@*Zi8qcdA>-AfBZYACT*nGWS|NH%Vee(m<-vGFv{{#Oecnig2*2@3@ z061k>NoGw=04e|g00;m9hiL!=000010000Q0000000N)_00aO40096104<;c00aO4 z0096107w7;001+G4#kcCMwgg}Dq1ld8wHN{ri zgGE%%DFqa@QfphQo}-?g7H#c8kG9&P6;RwbDsn`SgQzHr5D3T?!V(BcNXVYdOlEug z>vw0uV+NBjLyh>(EcdqG_epd@0 zoT)eA@6>%iIMk+-!UvDbM>aG43vG+vW-i<6gK@s zYprI7Otvjp0frjFa9!nesqSlJa)-0|#dOc+r~jXIPHR3DS@)EDcoP(xCe5|ZT}G#6 z#XzWaN2J|vBe3!#Z{erRV_dlgxb6$Uae0@x`yKz|Mc#W&{y!CnJ67qM7VAeoQx;r) zm+uizX!QeXYju*HQ%TAth)F{DgU7tTae5b91Iunz>dRxbTChF5=8xV7o&Kjs%g4S< zXmY8W)N#mS=#9nR`|Z9592mD2^2SuCEmeW%W_TB}!GHZi^EY2g@5S{g^Hluu6=}~J zn&sHRlWhxE(3UjJw;u9_S1we}lyQYqZTGFFZLaP~(DT#OaA0F0?s$VnY~W0mUp^ZH zfzG8JE0yYf6w8}tEWQj7=pnWBRu6XQVJ{YRc3?3#XhPTUbX`8SmFCR2rA^kZhbjK0 zhJ$u_7&7QI2gjs*)_EjDzN_ zw)wBJ`QwCjPZF*~FUB*}GJ7ed!9p}8i{UlD*ZoaNBIxQNCFz(j?aHzLb{%a_i@d$W z*q39&VVU!HN>|}ESZn0UohkL4vOT)syK?Q1Yyc;OPnH_`%r`CJC(O}h0nvpJIhE@& zaXmPpH~Obs;j~m>P|2>h-p)`kA}Fesk?wHX6M}?V z^DSBHdGav)?!#L1Y5V=}(4cMNrE1#A+NM*rjkQhff+&v~a@F+WqI8EXCe<5`Kv<}# zs^534y4@G(pOHRnP;NnPW=@vd%y$=sbsbKmoZfA^;r?!;nXIVI?S8X~4~B)W_aA$6 zm$};R662!2mLN{D}IBjfE_c$K9^XAz%j<0EGt8eaTX!F9J4Xs{DR%BI!daT*#lwoL= zVW14aEGFJ*w>TZPESDq8o$JYPWx5@i>5jBSCdET(@pg8G1vJF$;Cc5rP?3}A!Y95Z zl;Z`BY`Z8O1y~zbyg>40%-X6?wu5%_Z@;!MuRj_l<+le;)cpIyFJOGqAZi8%f0qja z{rqj!hAq3x!MxwyT@vf3M|&89D?M`R;brfn*)5)a=|%Y-I4x$~#B&_W&JL&I&a1`!;I(M`-@SH0w!|h z$J=%tJ_VNG76{-GaD+Ps0OV~VDYEd;CLwOkrUOd4?v9aW!eZEW-V0wFl02$h5@6vr^|BD>AvO0#9U z(%=Q2%=9d`)179+^u?NoA|jZc;Y#c0w8yJA%E{s|8uU>iDV{=iQ@j7A_dY{hNd`xZ z(aA^}GRfLw>5BDhJ}WKE%N|iMAScTO`=^7j(XfWTz>GjxZ1)FY{N@gSt2c1Xut6nL zMuVgfF_iUExU|2-8pF}S|Gnj>zbvajD1)=XU9ce~&Wm`W!)eGH9fT3tpc`HW0%0{6 zl9%0!dZVfvdtgO4Uegd=7pRQ%G>k0W(Y3%fwId8SRn<42YxN-tMW9M0m_UBJ)dZeq zyPf$tnFYC7d7g}3ECJ9M3|y}p9H{u_aOI|*hXqNRUOcpT=s-`V3!#F8a|X=LjLb zV7R`e<9KyL^|_X*ql*j>knEZlP;BYzA7eVg5h)_cc8dwY2Ns4+JQ>dHe(8uySd%Pf zJ{HG70@9c)ID{Vfg*hcf#XL!}jQPT%k9`KZMX>P(!U#X8GV^?%r{9>Oyon)0oib|3>>DRQJ^=@WF1iq> zMM;4+SQSvPtH5v(!a+|k17Bc56D&3{JK3#ftJy?Ceq7i_Fa_)Dnma)7Q)inR+kG&` zn8Li|BT*h5DxFvWI55rOQ9I)f2x zE5m{a5U_uA#j#w~KcgtG|ImC7rW^E+AM+(}#6&TAbQ%Z=_{FTFcQ622VH@ZmsSLn@ zRah1)Fh~FzVJ&o6%}$3k-EMQ(EhzWyQ%T{72i`~J1mpCPj%2#M5YZqlm#tOm4$`pe zO-pD;Ml$WcFPTIfhc}A5-^J;PY4SZkFpk7|;mowzPMd59~-<5W~MRUAm?(=x{l@D!_%QtIv zM>Jm}rHB;6)9KmV@SDsfk1$U1(txX19PPE@mY3N6g>1nj+UC~9fKs_ruGlJ{*kNC^ zlAkz-9WY!wSD_05Y0rDsI2_9Huapzp8242qNkq}&w*P94C)m6(eExW(`nBdtrSco` z>>;)8kbTKVga*Ci;08}hTY8W{K5tvCy*BvISz1e#RQg^N$0%;-wMZ7?V+pp=j%bip zBpg4$?ChZNbac4(MXM;YjW#=?BPTz?hNU3GsBA6cGB_ZT3EbVfd# zJCdwaVL!JDqZ1|*C$Zv)sG&1rQJyf=Q}^hDV3jI{TCqA(bAEzdIkOip!KMj!kP+i;uV#OE z#R72L^0UX0^(J{UhE-1P2)&F6B^#*N)dLQO)dFow`4-TPpLl0NvMbvA`R3%G8+sj+ zJw$Chshr$SDS{qoV=@ME#j|2E@`3e8Z!($r=8`|qmh)7NCc)!7xnSY=rx-WcVc}vp zq`!12Ov>2u9o?eV9ze2|+@8M>l(^A9!JS%NCT@Ej1=EB%MqnVD0-DjoKDDYe)^StxEZ4dY}XGiV)@Q7obL1|5uz#CX8laG3H+P*5h3Yv}Lje8#uTjQ;^;izkW?? zIiogJGR|zv%t!fYzmj&WLSBz?4~R)#*ayKmgRjiS$%gsbUt8zDYIH=rTtaG=MH>IY z!_-SUd_>oyWbdYN)E?OGAKw~&|1mw#&U%W>w>_31NcMVNSbawOXDfI0^)aXKYpvhy{{lb0TlYgg=_!8vkQj)r Z{{gcVTi0kXr~Ci_002ovPDHLkV1m8rrYHaa diff --git a/modules/use-cases-architectures/images/node-icon.png b/modules/use-cases-architectures/images/node-icon.png deleted file mode 100644 index c0bd1ea6393957cdb4cae471cb032b22a3a5fd3d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5084 zcmV<26C>=2P)BE1ZQLxAY_vg;$?V;oX(lt{{G@3qJ`_O^!KmPq;Qw2}Cn0L9?Tam?khm;cmfj@-EE=vY<#k5o0f^m&*sM6pBI0EL{uRZtU4R5Z z`^X8`V)zNfJ8YaF;u6EP09P2E5Ad?jnUOMxUqvh^s*L``+lU9}BlcPZ2XpY-jyyG} zttleQMaed+noc9ry@UIIwng1*#M^^UkFzj+6yF25_LnMhmf?pGpH<}CbcnAmX|u%; z|7E#p3GpKUUAw%6atiT2#A8)^yd3Cssj4^8z$>TJnP(8+gLrkpJj%xIMEtceUz`ec zeo~wa<_{hK;NmJRuzL3+9o0iVC08s$)vACv7|KExH6jmbd4gn8^0FkllzkRT6fd|P0n zrwSVAXrbt(XrcJk;OyVRDf#{Yw^VBW$m{6QFhZvPBk<6w!tZERBfJcLd1$M3&)TehHS09YRtkN(Z(GDi^F2;vKBX;HY+Q*UEnTqUvgh@A9J5aYSek}G;8j! zqp=F+X_rylpF&#LEj%b>g+0Q~5^H*5N-?MwSLAe5T{?GXqX;9WfX;*4pXw6&V*io?m|<6;W0GAb4f<-njQ>$>Q>9MPr@ zwU=n+0I+*9IKpp9{{0u*d8K7@p9VPmCp-6AwmA>rtOL;dXv;Q$XXCx^0IWWyc=l3w zC&Xgk0yx!!!@*Zi8qcdA>-AfBZYACT*nGWS|NH%Vee(m<-vGFv{{#Oecnig2*2@3@ z061k>NoGw=04e|g00;m9hiL!=000010000Q0000000N)_00aO40096107IYy00aO4 z00961051Rl001pv+mZkP4@F5tK~!i3wVHX1-bZ=I-{19~`yTu5yQkx|6UQNrf}QwE zou(v>2(5|&LP+ojQX#DZ1{8r5klKbKs0u}b0;zh?mK!NWqE!Jkf&|1-f*}dP#Ez5L zv16~--h21m_r87ypLw5m$3O3`W3w?MXEMK;XJ)?7InT_yMi2x`OG}1f*tXqhG;+Dz z{QUgz@bJvc%*e>d?Ck8|;2;EREy8N}hgYW;7?GKd}TLVMcxw*N4fq`8x7K;YBp@5EJnkFm5lIyy3w%ctmu?ju(GYU;4 z5`pb783YiA$t)w<@meQJ|95ES)3enhmT9|zJC@pWVE99?$-U9GownOxHQ)DHib4Ys zvFR&s9Y)A@8XFUtZiqs_rwdC9VtT@DO%QGffv$~+Wn0mgi%E|nt96JB`QhVAl zZDTbL7dQ$ce#i3$l3RA?-@kkKok=Ixa%*71m}NB$K(B~oT3wiZE$5i&)=Xtj)~sVOESX-tSYbTPnM zeSLk^YL!~n1vm2OswAKRSt^-sHri$L#qXc^+tby@!!B`Y{HsU2paa4i^KZUu{Gn{* zhIk@csh0HCpu{xL6;{AZ5NZj6ikL>k1H6zKV4@c~sF{N>U;rZghfoVC>*^Je3dQYs zeaYOp#pjMzzxedjmpXpUHrKkuYS5SdR?ZW%GrNcWVpsP5&4V{B7UwbIj_Kl)VZ!$E@-nq1Wi;>uGZOW!#?L4I?Kj1LUv5s? zgujR~;#XP3dHUm9_7xtuYE&fJ2qT0H1sHG%_c#DLK|hFr z0eI!C>F@6sHmbF1GLiH;SgLWV_Slcke0-*U#KLD(bh?^FoM)_W-#yoUY{)#I1YaqF zUUY(0!xg*%7=#EKgy5tNd&cDCBvvFkL63Ctsfn~s+D(79`Rp%dKXZKPyU0(U5?7Ot zm9=mlBVz9uc*~yt2d*DFP_38XfkCa4wFpsw(g}_sK}q~XI7S=rDmrO5+-H{_`N8Ce zJAU=mxx@eP|7L>l}Az=BO9#3@X&73@kvaM%jr8#bux zwT5d|4$b_78^8t!_Nv3~@}M?EVK}Da29>85KVPiP5ZQ?|gkTVAPZoq&AdJy(=JR>1 zkbo^5<2vF}Xhw*z%UY}0^vYLwP;3bI`jgnP*;#gC=tC9p6CJz42=QG8Ya{HmFi5%HR_z$Gv1<$IUp$WS?+<3wQ^kb+Q-Z^0&vQDb-}UJeCoL5jl= zA0U;QVcJMSr62~YUUJ9ZX~49+>Zwf=|MQ2niIK}l2hDX=folR1u0RML%nWSQ8ZG@0 z^v*SfwL1ZpXp0P(Ed2`WRdx0HRR7fDw|I^CWz9?gfCLDFhgx6@PHO4t-JhTXz0}YN zKT09cbQ;ZHs{hs%&-m|*3J%BV1dFC8BXTr;(0^WbeK5+jSq zfqhjJO(?#RJ)l=1lvs?=ZC*6M&7yKLG5`ZAbz8~mGwE!#JdI93GxO7%i$}I9^?=+n zmtWXaIXyIX&yl`uld;IaJ6`+W*XQpPeB11 zuZ3u&8NgEM#mzH6zS(U=Sr)^qo+;GNjsrp98;zM9!ii=YpfBt@Wny=YXmcUM_DOqiOZEWh=Ibwg=lFv! z7yqAjA$fg1v6Tm-cCoL48ZAG)b$;}&W6^8_$#hx4l7jkl-mS;Xn48@(J+OPiPPM$I z-I!Bsa{vH|-6@!sKHw!S1ShFiFI(ZU2(95Nk_(^{875m6W-u_$90y z7DQtCnw@NQseNA5Zbx9lsPOaq-Ddp_I(wF@C|AwT?YiACYr6c7PxOJep+6o+?p!b+jb7Wb;sZv9W$;v-D1JwEu~SWUN-5vvQPR)um>RwN zsGVvM(-R}hiJ?ksxzDY{Tk~1mrf=)Km25`*h+`#pWZsc*GDHac1gIJz9)g*PC*waV z#xVvb;!>Ja=gfVn%zWkKmtS~Y+b^jLuc@Bel)xxhKazHN_)vzz-PQAh@!TKm8~w`{ z7ax6L{yX$r>Pn*uI)t+H;&}DUVE@iDsqs00A`uzonDSb&;?b>M)9K%N8V)fzd@pY0 z-+Svb(IAIHIZrUKRS3a}me7@Yxnm4JL5~=ikV?#kLqZ-;K}}=sKcBn3)miQty=;Do zn!78}0THfyyX+l#D~5W}8cHt8^SY;#vRFVRoPqmp{8=_WhWy0o^|1&x5TXr~f#KQs z3R+{D=!7l?0w*Bcff)PZaKf<7*Zk_qzr`XtDDuYW#N)NNKZL%?;#vdUWHH?12g?5|G}=}zndFV z!xH)l*S(#tOedFwwb21>(t>amHgM|xp3kl2WTl65gKR;cCL2YxfqBZhRPryfW&s+Y z67*WuJx)*%EeZqZNAgIl3x`2#!~oapMD4U=CVQv#1^{hz3SJ(wvZfy+et=M1MG%4w zgyIGCpeS{Zhk`;|h9x-QKg2ZJj$_Bo{@aE=Xc|e^CpKK;z2^ixGP7WJE+zV`&gE9M0-BsS5Lv;B@WrpG^M!RQ2$c|5S2W z`D+4owg$oG%x!mG^U$W~AElBho>B?8h#Z|@>S#l!!*ak=OIE-BTRp?%0+MXfhI{x% ziOL9rXkhbJ|Al83|FzscZ<|-`t={#$Ty*pPkq;c$e1EeVq|!0YZ*??xf4YHNj32H* z0(3GdQgsl5Q^;1o)uRCh7$$?%Z}r>|kOUf2j>SEp?PgN_XBVG4QvQ!$&wjP-y~@Yp z?(Zj7dRO6HyYmOf2X9(fn$6~VPtt6S8ZQV|mcmZ}p-RER}+C z8jX4;-CwQLE7tKJzWni%rSB05HsCznXL;*21NXje?5}gij#N5cTwc^$V-DaB>jbXk zE{IMv8EvfQ!dpFlf)$FNkYF;5W~g;>qhGT;ESvQL9hbz}Q4E#0{+zZj7!x(JH*5W4;T_tvbP;U(k$PE7EgfW5=)UqO<@Wv`qLj-&5L=c3GBFR6z zF~ll*OUOEHFCI&tSw8gBbDx-~{t#k)Kk5FBgt;Yo&!67#cX{(>(~5N53y<*l9x|1! z!)uBJ!{-S|*l5+(kQs2M6Bc2HPu67u!?a;f(iTR$X-)5sRwSd5bu}?eszs~>uM@`> zzRmAAQ@J>M%HCMt5f3}^iV^i3HS`Rg&<4@v27-rZt<7%5C7u)Ki{$G!94>9`(pW) yTUT!z-RaOs32&{pll65CrSKfyS`bpR^=|=4K%$(btH@&j0000BE1ZQLxAY_vg;$?V;oX(lt{{G@3qJ`_O^!KmPq;Qw2}Cn0L9?Tam?khm;cmfj@-EE=vY<#k5o0f^m&*sM6pBI0EL{uRZtU4R5Z z`^X8`V)zNfJ8YaF;u6EP09P2E5Ad?jnUOMxUqvh^s*L``+lU9}BlcPZ2XpY-jyyG} zttleQMaed+noc9ry@UIIwng1*#M^^UkFzj+6yF25_LnMhmf?pGpH<}CbcnAmX|u%; z|7E#p3GpKUUAw%6atiT2#A8)^yd3Cssj4^8z$>TJnP(8+gLrkpJj%xIMEtceUz`ec zeo~wa<_{hK;NmJRuzL3+9o0iVC08s$)vACv7|KExH6jmbd4gn8^0FkllzkRT6fd|P0n zrwSVAXrbt(XrcJk;OyVRDf#{Yw^VBW$m{6QFhZvPBk<6w!tZERBfJcLd1$M3&)TehHS09YRtkN(Z(GDi^F2;vKBX;HY+Q*UEnTqUvgh@A9J5aYSek}G;8j! zqp=F+X_rylpF&#LEj%b>g+0Q~5^H*5N-?MwSLAe5T{?GXqX;9WfX;*4pXw6&V*io?m|<6;W0GAb4f<-njQ>$>Q>9MPr@ zwU=n+0I+*9IKpp9{{0u*d8K7@p9VPmCp-6AwmA>rtOL;dXv;Q$XXCx^0IWWyc=l3w zC&Xgk0yx!!!@*Zi8qcdA>-AfBZYACT*nGWS|NH%Vee(m<-vGFv{{#Oecnig2*2@3@ z061k>NoGw=04e|g00;m9hiL!=000010000Q0000000N)_00aO40096107{?(00aO4 z0096104e|g004wH&_4hG2i{3UK~!i3-I;rc6;&L^e`jXyV_)vB?yicDwN>-c!wM3L z%HAdsLKaj6859*1)n5@*L_~l1rxij`BqqIIR6;};p&%t?8|apg%1!s7t8ROD@7{Tw z)Ax7I%-p-HmUo;R)M2=D=FFLMexLdN&hMNvS16_E|C_A;FB+3MHN}Ht^bHjb4HX8C zkN!DYQj%T8v8}01`I*xiX17ggZOYfwfLh^=jJY3d+xx}N-wyPhK;92z5CkNphEbMK z*$g1-mUVTkS+V4ri|5q{Qq*GX9q1o^e8bis_YFd=jAPYh9qvYjY{VIsQA+8#o+W7Y z6^oy}_u7WKOzM#7s|TT;-@K!H|KOaCDR*DDq)>D>f3>?*a)l6Vy=KE1z&4JItw_DQ zwR=i^=IOQ9{s-{jiaoy%*_QR_t(V+(<+-rf^@7*7?8;{yQUUlh1E7#mpcEt@sU18puN5Gq+! zC2f!}zlKBxY=_8oJ5ADmniux^BUzQ+z- zgOmY%@KHG8NnT^|rM@t!-;{u#7CMn0sav>Czm@s%4 zhHTIcUJA-s?0_I9SzNA$sQ(RB{7rPNBfG9_ay7Z+69W``>Wyv1l2@N|urzVtO%|^w z`LL8@3!RAy9bEEV!LcOPEgqJU$+(G$(9jB-bvw3I4ZIoMuKJu14+3oXbQgj?EJmLk zG$Ie2pqp7_#&Ocb681?RZ?Um@#b#vWw)Bjz8M%Ij2Hq!sIIdh(CL!Lh{U>(p>C0ub zZ&$jXLVVYr78h2s!6$J@IEo?cU zksqQ5syipd+xy3H7y-7>k@(^?Y_JEmqMtBMR*W~To9p>ljy`nT3R@kf;yzwTO<=>T z5j6Hkby!jDgpqS_c$`I0=IvClaUla>f9GPdHaeIod!kRW=6budXx@y+@4RlsMV;#4 z^W^=hXN9m4HWH{v?DI)dLcDQoa1dTNO=Q!tq~v*BGg~gdpmXl@wrnP2>cNqDGBYPU zrc^GQ>6$g|g08llGpY`)rz0Pqf{kAgXp^wvnS^+7G@aN%!)!F(yuiQbri&lB{i^1A zMgC7z+D)>^_OXc-W||U&(8IkPIf@5Q&4yPJcM$80Nz}rE{!9E=;m5DxS+Gcl%X2L4C!m0`J+MDZOZzz5g zV3d$K$GY#9%gF6fy-$)HBavm(4I45>bPOlXz;2V#arI<2Jf&S-S4;SvB4Ok_XGU}A zXt9Pu27WN7qj~Y%HafbAWP#a`AVLs1u(@QntW;jRA4*K(u!#yWo330xKfGJzDWQ57 z&hMztJNWt)1{lHSV(HS-RG-fj)%aeb=zUlEBZ?=D9jvYc^vme=ky=kmKP83BO{zmDwtVI*E<3Q8h}^rN;9`)3WHgx^qH2 zEH&4zx=4>6?PkVwu!SKNknqXBWy3n=bmJ|?!%_ho{mzG3z(gq_9{O5&>HM{;mW|;X z5Py4+*vw+>$m^Ej|l!wi9vpu~AB060I!R12P=saGaKzZee-Z#Z`4e9o9cq z8xhr^hDBn=8fsh`RZ4P3Xh6efl}esJG(5(CA1KOZosK47{c;xt$Mvo>jwL{pWH*U( zU#Cp_sr$2Xyq*qI=;hI6!gC6ElY0(^Lp-cV!#Eko66|@~7@Nh?KYZ3RzHr3S7pbmu za%*RIo!mM9p@54I@mrpQN}0{RsO@wPnojeXmgeM)IJHt;0QUV|fBZ;+>&( zUAU-nIgmDNuE^518 zOkYh-bLwn0#xwR}dF39a(tz@gW802DPZVM%=J#wn<)#yUYnf`PUM=lD!?tS>InS`? bf2G=A+$XVl6klV_00000NkvXXu0mjf=N%aF diff --git a/modules/use-cases-architectures/pages/change-data-capture/consuming-change-data.adoc b/modules/use-cases-architectures/pages/change-data-capture/consuming-change-data.adoc index a9dba46..56a295b 100644 --- a/modules/use-cases-architectures/pages/change-data-capture/consuming-change-data.adoc +++ b/modules/use-cases-architectures/pages/change-data-capture/consuming-change-data.adoc @@ -17,11 +17,11 @@ Below are example implementations for each runtime consuming messages from the C While these examples are in the `astra-streaming-examples` repository, they are not {product}-specific. You can use these examples to consume CDC data topics in your own {cass-short}/{pulsar-short} clusters. -* image:csharp-icon.png[] https://github.com/datastax/astra-streaming-examples/blob/master/csharp/astra-cdc/Program.cs[{csharp} CDC project example] -* image:golang-icon.png[] https://github.com/datastax/astra-streaming-examples/blob/master/go/astra-cdc/main/main.go[Golang CDC project example] -* image:java-icon.png[] https://github.com/datastax/astra-streaming-examples/blob/master/java/astra-cdc/javaexamples/consumers/CDCConsumer.java[Java CDC consumer example] -* image:node-icon.png[] https://github.com/datastax/astra-streaming-examples/blob/master/nodejs/astra-cdc/consumer.js[Node.js CDC consumer example] -* image:python-icon.png[] https://github.com/datastax/astra-streaming-examples/blob/master/python/astra-cdc/cdc_consumer.py[Python CDC consumer example] +* svg:common::icons/logos/csharp.svg[role="icon text-xl",name="C#"] https://github.com/datastax/astra-streaming-examples/blob/master/csharp/astra-cdc/Program.cs[{csharp} CDC project example] +* svg:common::icons/logos/go.svg[role="icon text-xl",name="Go"] https://github.com/datastax/astra-streaming-examples/blob/master/go/astra-cdc/main/main.go[Golang CDC project example] +* svg:common::icons/logos/java.svg[role="icon text-xl",name="Java"] https://github.com/datastax/astra-streaming-examples/blob/master/java/astra-cdc/javaexamples/consumers/CDCConsumer.java[Java CDC consumer example] +* svg:common::icons/logos/nodejs.svg[role="icon text-xl",name="Node.js"] https://github.com/datastax/astra-streaming-examples/blob/master/nodejs/astra-cdc/consumer.js[Node.js CDC consumer example] +* svg:common::icons/logos/python.svg[role="icon text-xl",name="Python"] https://github.com/datastax/astra-streaming-examples/blob/master/python/astra-cdc/cdc_consumer.py[Python CDC consumer example] == {pulsar-short} functions @@ -29,9 +29,9 @@ It is very common to have a function consuming the CDC data. Functions usually p While these examples are in the `astra-streaming-examples` repository, they are not {product}-specific. You can use these examples to consume CDC data topics in your own {cass-short}/{pulsar-short} clusters. -* image:golang-icon.png[] https://github.com/datastax/astra-streaming-examples/blob/master/go/astra-cdc/main/main.go[Golang CDC project example] -* image:java-icon.png[] https://github.com/datastax/astra-streaming-examples/blob/master/java/astra-cdc/javaexamples/functions/CDCFunction.java[Java CDC function example] -* image:python-icon.png[] https://github.com/datastax/astra-streaming-examples/blob/master/python/cdc-in-pulsar-function/deschemaer.py[Python CDC function example] +* svg:common::icons/logos/go.svg[role="icon text-xl",name="Go"] https://github.com/datastax/astra-streaming-examples/blob/master/go/astra-cdc/main/main.go[Golang CDC project example] +* svg:common::icons/logos/java.svg[role="icon text-xl",name="Java"] https://github.com/datastax/astra-streaming-examples/blob/master/java/astra-cdc/javaexamples/functions/CDCFunction.java[Java CDC function example] +* svg:common::icons/logos/python.svg[role="icon text-xl",name="Python"] https://github.com/datastax/astra-streaming-examples/blob/master/python/cdc-in-pulsar-function/deschemaer.py[Python CDC function example] == See also diff --git a/modules/use-cases-architectures/pages/change-data-capture/index.adoc b/modules/use-cases-architectures/pages/change-data-capture/index.adoc index 1005e98..acc3434 100644 --- a/modules/use-cases-architectures/pages/change-data-capture/index.adoc +++ b/modules/use-cases-architectures/pages/change-data-capture/index.adoc @@ -6,27 +6,25 @@ Change Data Capture (CDC) is a design pattern used in software development to ca Before we get into the specifics of CDC, let’s first understand the resources needed to complete the flow. -== {pulsar-reg} source connectors - -[NOTE] +[TIP] ==== Throughout this document, "Source Connector" will refer to the CDC Source Connector component, while "source connector" refers to any other source connector in a {pulsar-short} deployment. + +For more information about the CDC Source Connector component, see the xref:cdc-for-cassandra:ROOT:index.adoc[{company} CDC for {cass-reg} connector documentation]. ==== +== {pulsar-reg} source connectors + Source connectors in {pulsar} are responsible for ingesting data from external sources into the {pulsar-short} system. They can be used to collect data from a variety of sources including databases, message queues, and file systems. When the source connector “sees” data, it streams the data to a {pulsar-short} topic. This enables users to easily integrate data from disparate sources into their {pulsar-short}-based applications. Source connectors make it easy to ingest, process, and analyze large volumes of data from a variety of sources into {pulsar-short}. {pulsar-short} offers extensible APIs where developers can use a defined interface to develop their own connector. The interface takes much of the boilerplate burdens away from a developer and gets them right to the purpose of the connector. Creating a connector means adding in the know-how to work with data from the source and adapt it to produce a compliant message with the {pulsar-short} client. As you’ll learn in the next section, among the processes needed to capture change data, the {cass-short} Source Connector is a very important part. To run a source connector, you provide configuration about what data will be selected, how to connect with the upstream system, and the destination topic for the new message. The source connector takes care of producing the message. {pulsar-short} source connectors run as {pulsar-short} functions within the cluster, so many of the features of functions apply (like the number of instances to run and how to configure the function instance running environment). Metrics and logs for a source connector are automatically made a part of the cluster. -[discrete] === Monitoring source connectors -Monitoring a source connector includes two areas: health and performance. Every connector in {pulsar-short} emits basic metrics about its health, including stats like the number of records received from the source, and the number of messages written to the destination topic. Connectors also emit debugging metrics like the number of exceptions thrown by the source. Refer to the https://pulsar.apache.org/docs/reference-metrics/#connectors[connectors area of {pulsar-short} metrics] for a complete list and explanation of metrics. - -Performance metrics include health metrics as well as specific knowledge about the source. The {cass-short} Source Connector includes quite a few performance metrics. Refer to the https://docs.datastax.com/en/cdc-for-cassandra/docs/latest/monitor.html[Monitoring CDC for {cass-short}] reference. +Monitoring a source connector includes two areas: health and performance. Every connector in {pulsar-short} emits basic metrics about its health, including stats like the number of records received from the source, and the number of messages written to the destination topic. Connectors also emit debugging metrics like the number of exceptions thrown by the source. Performance metrics include health metrics as well as specific knowledge about the source. Refer to the https://pulsar.apache.org/docs/reference-metrics/#connectors[connectors area of {pulsar-short} metrics] for a complete list and explanation of metrics. -[discrete] === Source connector logs Most {pulsar-short} source connectors emit logs that show lifecycle events as well as custom events specific to the connector type. All logs are handled the same way core cluster logs are handled. By default, they are written to the console and collected by log4j destinations. If you are using function workers, you can access log files on their disk. Refer to {pulsar-short}'s https://pulsar.apache.org/docs/io-debug/[connector debugging guide] for more information. @@ -41,9 +39,8 @@ Schemas follow a primitive or complex type. Primitive schemas are simple data ty Complex schemas introduce a more structured way of messaging. The two types of complex messages are KeyValue and Struct. KeyValue is JSON formatted text that offers a separation of custom labels and their values. Struct is a custom class definition set as Avro, Json, or Protobuf. -KeyValue offers an interesting way to encode a message called “Separated”. This option separates a message key and the message payload. This in turn has the option to store message key information as a different data type than the message payload. It also offers special compression capabilities. CDC takes advantage of separating KeyValue messages when it produces both the event and data topic. Learn more in the "https://docs.datastax.com/en/cdc-for-cassandra/docs/latest/cdc-cassandra-events.html[CDC for {cass-short} Events]" reference. +KeyValue offers an interesting way to encode a message called “Separated”. This option separates a message key and the message payload. This in turn has the option to store message key information as a different data type than the message payload. It also offers special compression capabilities. CDC takes advantage of separating KeyValue messages when it produces both the event and data topic. -[discrete] === Namespace schema configurations In the context of CDC there are a few schema configurations of note. All of these are specific to the namespace where the event and data topics are logically located. @@ -56,15 +53,12 @@ In the context of CDC there are a few schema configurations of note. All of thes - *schema-validation-enforce*: this flag limits how producers and consumers are allowed to be configured. When enabled (true) producer and consumer clients must have a schema set before sending the message. When disabled (set to false) {pulsar-short} will allow producers and consumers without a set schema to send or receive messages. CDC disables this option (set to false), so producers and consumers do not have to know the message schema ahead of time. -== {cass-short} change data capture (CDC) agent - -The {cass-short} CDC agent is a process running on each node in a {cass-short} cluster that watches for data changes on tables that have enabled the CDC feature. Using {cass-short}'s https://cassandra.apache.org/doc/4.0/cassandra/configuration/cass_yaml_file.html#commitlog_sync[commitlog_sync option], the agent periodically syncs a separate log in a special “cdc_raw” directory. Each log entry is a CDC event. The CDC agent creates a new event message containing the row coordinates of the changed data and produces the message to a downstream {pulsar-short} cluster. For more information about the agent, how to include its configuration in cassandra.yaml, and event data specifics read the "https://docs.datastax.com/en/cdc-for-cassandra/docs/latest/index.html[{company} CDC for {cass-reg} Documentation]". +== {cass-short} change data capture (CDC) agent and {cass-short} Source Connector for {pulsar} -== {cass-short} Source Connector for {pulsar} +The {cass-short} CDC agent is a process running on each node in a {cass-short} cluster that watches for data changes on tables that have enabled the CDC feature. Using {cass-short}'s https://cassandra.apache.org/doc/4.0/cassandra/configuration/cass_yaml_file.html#commitlog_sync[commitlog_sync option], the agent periodically syncs a separate log in a special “cdc_raw” directory. Each log entry is a CDC event. The CDC agent creates a new event message containing the row coordinates of the changed data and produces the message to a downstream {pulsar-short} cluster. -Each table that has CDC enabled also has a corresponding Source Connector in {pulsar-short}. This is unlike the CDC agent where the process runs on each {cass-short} node, keeping a log of all table changes. Each table-specific Source Connector subscribes to the events topic the agent is producing messages to. When the connector “sees” a message for its table, it uses the row coordinates within the message to retrieve the mutated data from {cass-short} and create a new message with the specifics. That new message is written to a data topic where others can subscribe and receive CDC messages. For more information about the {cass-short} Source Connector, its configuration, and how to create it read the "https://docs.datastax.com/en/cdc-for-cassandra/docs/latest/index.html[{company} CDC for {cass-reg} Documentation]". +Each table that has CDC enabled also has a corresponding Source Connector in {pulsar-short}. This is unlike the CDC agent where the process runs on each {cass-short} node, keeping a log of all table changes. Each table-specific Source Connector subscribes to the events topic the agent is producing messages to. When the connector “sees” a message for its table, it uses the row coordinates within the message to retrieve the mutated data from {cass-short} and create a new message with the specifics. That new message is written to a data topic where others can subscribe and receive CDC messages. -[discrete] === Event deduplication A particular advantage in the Source Connector is its deduplication feature. You might have read about {pulsar-short}'s built in https://pulsar.apache.org/docs/2.11.x/concepts-messaging/#message-deduplication[deduplication capabilities] - this is *not* utilized in the message flow because CDC needs a finer grain control to detect duplicates. As the CDC agent discovers a new commit log, an authentic identifier is created using the MD5 hash algorithm. That key identifier is added to the event message. @@ -93,4 +87,4 @@ Now that you understand the different resources used in this CDC pattern, let’ == Next steps -With a solid understanding of the resources and flow used within the CDC pattern, let's move on to the next section to learn about xref:use-cases-architectures:change-data-capture/table-schema-evolution.adoc[]. +* xref:use-cases-architectures:change-data-capture/table-schema-evolution.adoc[] \ No newline at end of file diff --git a/modules/use-cases-architectures/pages/change-data-capture/questions-and-patterns.adoc b/modules/use-cases-architectures/pages/change-data-capture/questions-and-patterns.adoc index 3386602..f933483 100644 --- a/modules/use-cases-architectures/pages/change-data-capture/questions-and-patterns.adoc +++ b/modules/use-cases-architectures/pages/change-data-capture/questions-and-patterns.adoc @@ -2,7 +2,7 @@ :navtitle: Questions and patterns :description: This article describes how table schema changes are handled in the {cass-reg} Connector for {pulsar-reg}. -We have collected common questions and patterns from our customers that are using CDC. We hope this will help you in your journey of getting the most out of this feature. Please also refer to the https://docs.datastax.com/en/cdc-for-cassandra/docs/latest/faqs.html[CDC for {cass-short} FAQs] in the official documentation for more information. +We have collected common questions and patterns from our customers that are using CDC. We hope this will help you in your journey of getting the most out of this feature. Please also refer to the xref:cdc-for-cassandra:ROOT:faqs.adoc[CDC for {cass-short} FAQs] in the official documentation for more information. .How do I know if CDC is enabled on a table? [%collapsible] @@ -57,7 +57,7 @@ Another scenario might be the {pulsar-short} broker is too busy to process messa The design of CDC in {cass-short} assumed that when table changes are sync’d to the raw_cdc log, another process will be draining that log. There is a max log size setting that will disable writes to the table when the set threshold is reached. If a connection to the {pulsar-short} cluster is needed for the log to be drained, and it’s not responsive, the log will being to fill, which can impact a table’s write availability. -For more, see the https://docs.datastax.com/en/cdc-for-cassandra/docs/latest/install.html#scaling-up-your-configuration[Scaling up your configuration] section in the official documentation. +For more, see the xref:cdc-for-cassandra:ROOT:install.adoc#scaling-up-your-configuration[Scaling up your configuration] section in the official documentation. ==== .Does the {cass-short} Source Connector use a dead-letter topic? diff --git a/modules/use-cases-architectures/pages/starlight/jms/index.adoc b/modules/use-cases-architectures/pages/starlight/jms/index.adoc index e925b54..2758aae 100644 --- a/modules/use-cases-architectures/pages/starlight/jms/index.adoc +++ b/modules/use-cases-architectures/pages/starlight/jms/index.adoc @@ -1,7 +1,6 @@ = Getting started with Starlight for JMS :navtitle: Starlight for JMS :description: Learn how to get started using the Starlight for JMS API and get hands on with a producer and consumer interacting with a topic. -:page-tag: starlight-jms,dev,quickstart,pulsar,jms Starlight for JMS is a highly compliant JMS implementation designed to run on a modern streaming platform. This guide will get you up and running with a simple Java JMS client that can talk to an {pulsar-reg} streaming instance.