diff --git a/docs/docs/index.md b/docs/docs/index.md index 91f12c935..bc17b4db1 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -1 +1 @@ -# Coming soon +# What is streams-bootstrap? diff --git a/docs/docs/user/concepts/common.md b/docs/docs/user/concepts/common.md new file mode 100644 index 000000000..4361c36f3 --- /dev/null +++ b/docs/docs/user/concepts/common.md @@ -0,0 +1,16 @@ +# Common concepts + +## Application types + +- App +- ConfiguredApp +- ExecutableApp + +## Application lifecycle + +- Running an application +- Cleaning an application + +## Configuration + +## Command line interface diff --git a/docs/docs/user/concepts/producer.md b/docs/docs/user/concepts/producer.md new file mode 100644 index 000000000..307bbcaf2 --- /dev/null +++ b/docs/docs/user/concepts/producer.md @@ -0,0 +1,15 @@ +# Producer apps + +Producer apps are applications that generate data and send it to a Kafka topic. +They can be used to produce messages from various sources, such as databases, files, or real-time events. + +## Application lifecycle + +- Running an application +- Cleaning an application + +## Configuration + +## Command line interface + +## Deployment diff --git a/docs/docs/user/concepts/streams.md b/docs/docs/user/concepts/streams.md new file mode 100644 index 000000000..0924695cf --- /dev/null +++ b/docs/docs/user/concepts/streams.md @@ -0,0 +1,36 @@ +# Streams apps + +Streams apps are applications that process data in real-time as it flows through Kafka topics. +They can be used to filter, transform, aggregate, or enrich data streams. +Streams apps can also produce new messages to other topics based on the processed data. + +## Application lifecycle + +- Running an application +- Resetting an application +- Cleaning an application + +## Configuration + +- Topics +- Application id +- Kafka properties +- Lifecycle hooks + - Setup + - Clean up +- Execution options + - On start + - Application server + - State listener + - Uncaught exception handler + - Closing options + +## Command line interface + +## Deployment + +## Kafka streams extensions + +- Simple topic access +- Error handling +- Serde auto configuration diff --git a/docs/docs/user/deployment/kubernetes.md b/docs/docs/user/deployment/kubernetes.md new file mode 100644 index 000000000..87c833da9 --- /dev/null +++ b/docs/docs/user/deployment/kubernetes.md @@ -0,0 +1,5 @@ +# Deployment to Kubernetes + +- Autoscaling +- Monitoring +- Persistence diff --git a/docs/docs/user/deployment/local.md b/docs/docs/user/deployment/local.md new file mode 100644 index 000000000..74626da4f --- /dev/null +++ b/docs/docs/user/deployment/local.md @@ -0,0 +1 @@ +# Local deployment diff --git a/docs/docs/user/examples/interactive-queries.md b/docs/docs/user/examples/interactive-queries.md new file mode 100644 index 000000000..a10120417 --- /dev/null +++ b/docs/docs/user/examples/interactive-queries.md @@ -0,0 +1 @@ +# Interactive queries diff --git a/docs/docs/user/examples/word-count.md b/docs/docs/user/examples/word-count.md new file mode 100644 index 000000000..77509ef90 --- /dev/null +++ b/docs/docs/user/examples/word-count.md @@ -0,0 +1 @@ +# Word count diff --git a/docs/docs/user/extensions/large-messages.md b/docs/docs/user/extensions/large-messages.md new file mode 100644 index 000000000..31ce42dec --- /dev/null +++ b/docs/docs/user/extensions/large-messages.md @@ -0,0 +1 @@ +# Large messages diff --git a/docs/docs/user/getting-started/quick-start.md b/docs/docs/user/getting-started/quick-start.md new file mode 100644 index 000000000..754bef8a6 --- /dev/null +++ b/docs/docs/user/getting-started/quick-start.md @@ -0,0 +1 @@ +# Quick start diff --git a/docs/docs/user/getting-started/setup.md b/docs/docs/user/getting-started/setup.md new file mode 100644 index 000000000..feae8cb57 --- /dev/null +++ b/docs/docs/user/getting-started/setup.md @@ -0,0 +1 @@ +# Setup diff --git a/docs/docs/user/monitoring.md b/docs/docs/user/monitoring.md new file mode 100644 index 000000000..31b5a7045 --- /dev/null +++ b/docs/docs/user/monitoring.md @@ -0,0 +1 @@ +# Monitoring diff --git a/docs/docs/user/testing.md b/docs/docs/user/testing.md new file mode 100644 index 000000000..f00b526a9 --- /dev/null +++ b/docs/docs/user/testing.md @@ -0,0 +1 @@ +# Testing diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 1b2fed48d..f7b583021 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -21,5 +21,24 @@ plugins: markdown_extensions: - attr_list nav: - - Index: index.md + - User guide: + - What is streams-bootstrap: index.md + - Changelog: user/changelog.md + - Getting Started: + - Setup: user/getting-started/setup.md + - Quick start: user/getting-started/quick-start.md + - Concepts: + - Common concepts: user/concepts/common.md + - Streams concepts: user/concepts/streams.md + - Producer concepts: user/concepts/producer.md + - Testing: user/testing.md + - Monitoring: user/monitoring.md + - Extensions: + - Large messages: user/extensions/large-messages.md + - Deployment: + - Local deployment: user/deployment/local.md + - Kubernetes: user/deployment/kubernetes.md + - Examples: + - Word count: user/examples/word-count.md + - Interactive queries: user/examples/interactive-queries.md - Javadoc: javadoc/index.html diff --git a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/KafkaApplication.java b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/KafkaApplication.java index 66941b679..f260f63f9 100644 --- a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/KafkaApplication.java +++ b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/KafkaApplication.java @@ -50,8 +50,6 @@ * This class provides the following configuration options: * @@ -82,11 +80,6 @@ public abstract class KafkaApplication activeApps = new ConcurrentLinkedDeque<>(); - @CommandLine.Option(names = "--output-topic", description = "Output topic") - private String outputTopic; - @CommandLine.Option(names = "--labeled-output-topics", split = ",", - description = "Additional labeled output topics") - private Map labeledOutputTopics = emptyMap(); @CommandLine.Option(names = {"--bootstrap-servers", "--bootstrap-server"}, required = true, description = "Kafka bootstrap servers to connect to") private String bootstrapServers; diff --git a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/producer/KafkaProducerApplication.java b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/producer/KafkaProducerApplication.java index 925e8ee79..69ff74fbd 100644 --- a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/producer/KafkaProducerApplication.java +++ b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/producer/KafkaProducerApplication.java @@ -24,19 +24,27 @@ package com.bakdata.kafka.producer; +import static java.util.Collections.emptyMap; + import com.bakdata.kafka.KafkaApplication; +import java.util.Map; import java.util.Optional; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; +import picocli.CommandLine; import picocli.CommandLine.Command; /** *

The base class for creating Kafka Producer applications.

- * This class provides all configuration options provided by {@link KafkaApplication}. + * This class provides the following configuration options in addition to those provided by {@link KafkaApplication}: + * * To implement your Kafka Producer application inherit from this class and add your custom options. Run it by * creating an instance of your class and calling {@link #startApplication(String[])} from your main. * @@ -51,6 +59,11 @@ public abstract class KafkaProducerApplication extends KafkaApplication, ConfiguredProducerApp, ProducerTopicConfig, T, ProducerAppConfiguration> { + @CommandLine.Option(names = "--output-topic", description = "Output topic") + private String outputTopic; + @CommandLine.Option(names = "--labeled-output-topics", split = ",", + description = "Additional labeled output topics") + private Map labeledOutputTopics = emptyMap(); /** * Delete all output topics associated with the Kafka Producer application. @@ -69,8 +82,8 @@ public final Optional createExecutionOptions() { @Override public final ProducerTopicConfig createTopicConfig() { return ProducerTopicConfig.builder() - .outputTopic(this.getOutputTopic()) - .labeledOutputTopics(this.getLabeledOutputTopics()) + .outputTopic(this.outputTopic) + .labeledOutputTopics(this.labeledOutputTopics) .build(); } diff --git a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/streams/KafkaStreamsApplication.java b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/streams/KafkaStreamsApplication.java index 3b028c343..847b682e5 100644 --- a/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/streams/KafkaStreamsApplication.java +++ b/streams-bootstrap-cli/src/main/java/com/bakdata/kafka/streams/KafkaStreamsApplication.java @@ -51,6 +51,8 @@ *

The base class for creating Kafka Streams applications.

* This class provides the following configuration options in addition to those provided by {@link KafkaApplication}: *
    + *
  • {@link #outputTopic}
  • + *
  • {@link #labeledOutputTopics}
  • *
  • {@link #inputTopics}
  • *
  • {@link #inputPattern}
  • *
  • {@link #errorTopic}
  • @@ -84,6 +86,11 @@ public abstract class KafkaStreamsApplication extends @CommandLine.Option(names = "--labeled-input-patterns", split = ",", description = "Additional labeled input patterns") private Map labeledInputPatterns = emptyMap(); + @CommandLine.Option(names = "--output-topic", description = "Output topic") + private String outputTopic; + @CommandLine.Option(names = "--labeled-output-topics", split = ",", + description = "Additional labeled output topics") + private Map labeledOutputTopics = emptyMap(); @CommandLine.Option(names = "--volatile-group-instance-id", arity = "0..1", description = "Whether the group instance id is volatile, i.e., it will change on a Streams shutdown.") private boolean volatileGroupInstanceId; @@ -135,8 +142,8 @@ public final StreamsTopicConfig createTopicConfig() { .labeledInputTopics(this.labeledInputTopics) .inputPattern(this.inputPattern) .labeledInputPatterns(this.labeledInputPatterns) - .outputTopic(this.getOutputTopic()) - .labeledOutputTopics(this.getLabeledOutputTopics()) + .outputTopic(this.outputTopic) + .labeledOutputTopics(this.labeledOutputTopics) .errorTopic(this.errorTopic) .build(); }