diff --git a/.envrc.example b/.envrc.example index 7ee0c5e..3995ae6 100644 --- a/.envrc.example +++ b/.envrc.example @@ -20,7 +20,9 @@ export SNOWSQL_SCHEMA=public export RPK_BROKERS=localhost:19092 export COMPOSE_PROJECT_NAME=grpc-todo-app # The redpanda schema registry (internal to docker ) -export RP_SCHEMA_REGISTRY_INTERNAL=http://redpanda-0:8081 +export RP_SCHEMA_REGISTRY_INTERNAL=http://redpanda-0:18081 +# The redpanda schema registry (external to docker) +export SCHEMA_REGISTRY=http://localhost:18081 # Kafka Connect URL export KAFKA_CONNECT_URI=localhost:18083 # gRPC service port @@ -28,7 +30,7 @@ export PORT=9090 # Redpanda Brokers export BROKERS="$RPK_BROKERS" # Topic to store the Todo -export TOPICS=todo-list +export TOPIC=todo-list # Running environment, typically used for grpcURL export ENV=dev # The consumer group used while consuming messages diff --git a/README.md b/README.md index 003139c..4899a68 100644 --- a/README.md +++ b/README.md @@ -294,7 +294,7 @@ An output similar to this, > **IMPORTANT**: Ensure the required environment variables are set. The following command expects the following variables > -> - `$TOPICS` +> - `$TOPIC` > - `$SNOWSQL_ACCOUNT` > - `$TODOAPP_USER` > - `$TODOAPP_USER_RSA_PRIVATE_KEY` @@ -371,7 +371,7 @@ snowsql -u $TODOAPP_USER \ --private-key-path="$DEMO_HOME/keys/rsa_key.p8" \ --warehouse $SNOWSQL_WAREHOUSE \ --dbname $TODOAPP_DATABASE \ ---variable redpanda_topic=$TOPICS \ +--variable redpanda_topic=$TOPIC \ --filename "$DEMO_HOME/etc/snowflake/todo_list_query.sql" ``` @@ -429,7 +429,7 @@ snowsql -u $TODOAPP_USER \ --warehouse $SNOWSQL_WAREHOUSE \ --dbname $TODOAPP_DATABASE \ --variable todo_warehouse=$SNOWSQL_WAREHOUSE \ ---variable topic_name=$TOPICS \ +--variable topic_name=$TOPIC \ --variable table_name=$TODOS_TABLE \ --filename "$DEMO_HOME/etc/snowflake/dynamic_table.sql" ``` diff --git a/cmd/server/main.go b/cmd/server/main.go index 600878f..cb4a5a3 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -42,7 +42,7 @@ func main() { config := config.New() client, err := kgo.NewClient( kgo.SeedBrokers(config.Seeds...), - kgo.ConsumeTopics(config.Topics...), + kgo.ConsumeTopics(config.Topic), kgo.DefaultProduceTopic(config.DefaultProducerTopic()), kgo.ConsumerGroup(config.ConsumerGroupID), kgo.AllowAutoTopicCreation(), diff --git a/config/config.go b/config/config.go index b4739b6..f966563 100644 --- a/config/config.go +++ b/config/config.go @@ -11,7 +11,7 @@ type Config struct { ConsumerGroupID string `env:"CONSUMER_GROUP_ID"` Port uint16 `env:"PORT" envDefault:"9090"` Seeds []string `env:"BROKERS" envSeparator:"," envDefault:"localhost:19092"` - Topics []string `env:"TOPICS" envSeparator:","` + Topic string `env:"TOPIC" ` SchemaRegistry string `env:"SCHEMA_REGISTRY" envDefault:"localhost:18081"` SchemaURL string `env:"SCHEMA_URL" envDefault:"https://raw.githubusercontent.com/kameshsampath/demo-protos/main/todo/todo.proto"` } @@ -36,5 +36,5 @@ func New() *Config { // DefaultProducerTopic gets the default topic that will be used as the producer topic func (c *Config) DefaultProducerTopic() string { - return c.Topics[0] + return c.Topic } diff --git a/docker-compose.yml b/docker-compose.yml index a4708a3..9a052c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.7" name: grpc-todo-app networks: _network: @@ -124,7 +123,7 @@ services: environment: - PORT=9090 - BROKERS=redpanda-0:9092 - - TOPICS=todo-list + - TOPIC=todo-list - ENV=dev - CONSUMER_GROUP_ID=grpc-todo-app - SCHEMA_REGISTRY=redpanda-0:8081 diff --git a/etc/config/todolist-snow-sink.json b/etc/config/todolist-snow-sink.json index afd4262..3c9ac63 100644 --- a/etc/config/todolist-snow-sink.json +++ b/etc/config/todolist-snow-sink.json @@ -3,8 +3,8 @@ "config": { "connector.class": "com.snowflake.kafka.connector.SnowflakeSinkConnector", "tasks.max": "8", - "topics": "${TOPICS}", - "snowflake.topic2table.map": "${TOPICS}:${TODO_LIST_TABLE}", + "topics": "${TOPIC}", + "snowflake.topic2table.map": "${TOPIC}:${TODO_LIST_TABLE}", "buffer.count.records": "10", "buffer.flush.time": "10", "buffer.size.bytes": "500",