Skip to content

Commit f76149d

Browse files
bufdevrodaine
bufdev
and
rodaine
authored
Cleanup (#11)
* commit * commit * commit * commit * commit * commit * commit * commit * commit * commit * commit * comments * commit * docs * commit * more docs * commit * commit * commit * docs * docs * docs * Update proto/bufstream/demo/v1/demo.proto Co-authored-by: Chris Roche <github@rodaine.com> * commit * with CSR * more work * restore schema * commit * commit * commit * Prepare for new demo.buf.dev instance * commit --------- Co-authored-by: Chris Roche <github@rodaine.com>
1 parent ff0c262 commit f76149d

File tree

72 files changed

+1362
-757
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

72 files changed

+1362
-757
lines changed

‎Dockerfile

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎Dockerfile.consume

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# This Dockerfile builds a small image that just contains a binary built from
2+
# cmd/bufstream-demo-consume.
3+
#
4+
# This allows users to try out this demo without needing to have Go installed, as well
5+
# as makes the demo runnable within docker compose.
6+
FROM golang:1.23-alpine3.20 AS builder
7+
8+
WORKDIR /app
9+
10+
COPY go.mod go.sum /app/
11+
RUN go mod download
12+
13+
COPY cmd /app/cmd
14+
COPY gen /app/gen
15+
COPY pkg /app/pkg
16+
RUN go build -ldflags "-s -w" -trimpath -buildvcs=false -o /go/bin/bufstream-demo-consume ./cmd/bufstream-demo-consume
17+
18+
FROM alpine:3.20.3
19+
20+
RUN apk add --update --no-cache \
21+
ca-certificates && \
22+
rm -rf /var/cache/apk/*
23+
24+
COPY --from=builder /go/bin/bufstream-demo-consume /bufstream-demo-consume
25+
ENTRYPOINT ["/bufstream-demo-consume"]

‎Dockerfile.produce

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# This Dockerfile builds a small image that just contains a binary built from
2+
# cmd/bufstream-demo-produce.
3+
#
4+
# This allows users to try out this demo without needing to have Go installed, as well
5+
# as makes the demo runnable within docker compose.
6+
FROM golang:1.23-alpine3.20 AS builder
7+
8+
WORKDIR /app
9+
10+
COPY go.mod go.sum /app/
11+
RUN go mod download
12+
13+
COPY cmd /app/cmd
14+
COPY gen /app/gen
15+
COPY pkg /app/pkg
16+
RUN go build -ldflags "-s -w" -trimpath -buildvcs=false -o /go/bin/bufstream-demo-produce ./cmd/bufstream-demo-produce
17+
18+
FROM alpine:3.20.3
19+
20+
RUN apk add --update --no-cache \
21+
ca-certificates && \
22+
rm -rf /var/cache/apk/*
23+
24+
COPY --from=builder /go/bin/bufstream-demo-produce /bufstream-demo-produce
25+
ENTRYPOINT ["/bufstream-demo-produce"]

‎Makefile

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
BUFSTREAM_VERSION := 0.1.0
2+
3+
.DEFAULT_GOAL := run-docker-compose
4+
5+
## Demo run commands
6+
7+
.PHONY: run-docker-compose
8+
run-docker-compose: # Run the demo within docker compose.
9+
docker compose up --build
10+
11+
.PHONY: run-bufstream
12+
run-bufstream: # Run Bufstream within Docker.
13+
docker run --rm -p "9092:9092" -v "./config/bufstream.yaml:/bufstream.yaml" \
14+
us-docker.pkg.dev/buf-images-1/bufstream-public/images/bufstream:$(BUFSTREAM_VERSION) -c "/bufstream.yaml"
15+
16+
.PHONY: run-consume
17+
run-consume: # Run the demo consumer within Docker.
18+
docker build -t bufstream/demo-consume -f Dockerfile.consume .
19+
docker run --rm bufstream/demo-consume
20+
21+
.PHONY: run-produce
22+
run-produce: # Run the demo producer within Docker.
23+
docker build -t bufstream/demo-produce -f Dockerfile.produce .
24+
docker run --rm bufstream/demo-produce
25+
26+
.PHONY: clean-docker-compose
27+
clean-docker-compose: # Cleanup docker compose assets.
28+
docker compose down --rmi all
29+
30+
## Development commands
31+
32+
.PHONY: build
33+
build: # Build all code.
34+
go build ./...
35+
buf build
36+
37+
.PHONY: lint
38+
lint: buf # Lint all code.
39+
buf lint
40+
41+
.PHONY: generate
42+
generate: buf # Regenerate and format code.
43+
buf generate
44+
buf format -w
45+
gofmt -s -w .
46+
47+
.PHONY: upgrade
48+
upgrade: # Upgrade dependencies.
49+
go get -u -t ./...
50+
go mod tidy -v
51+
buf dep update
52+
53+
.PHONY: buf
54+
buf: # Install buf.
55+
go install github.com/bufbuild/buf/cmd/buf@latest

‎README.md

Lines changed: 51 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -2,56 +2,56 @@
22

33
## What is Bufstream?
44

5-
[Bufstream](https://buf.build/product/bufstream) is a drop-in replacement for
6-
Apache Kafka that's 10x cheaper to operate.
7-
While Bufstream works well for any Kafka workload, it excels when paired with
8-
Protobuf.
9-
By integrating with the Buf Schema Registry, Bufstream can enforce data quality
10-
and governance policies without relying on opt-in client configuration.
5+
[Bufstream](https://buf.build/product/bufstream) is a drop-in replacement for Apache Kafka that's
6+
10x cheaper to operate. While Bufstream works well for any Kafka workload, it excels when paired
7+
with Protobuf. By integrating with the Buf Schema Registry, Bufstream can enforce data quality and
8+
governance policies without relying on opt-in client configuration.
119

12-
In this demonstration, you'll run a Bufstream agent locally, use the `franz-go`
13-
library to publish and consume messages, and explore Bufstream's schema
14-
enforcement features.
10+
In this demonstration, you'll run a Bufstream agent locally, use the
11+
[`franz-go`](https://github.com/twmb/franz-go) library to publish and consume messages, and explore
12+
Bufstream's schema enforcement features.
1513

1614
## Requirements
1715

1816
- Git
19-
- [Docker](https://docs.docker.com/engine/install/)
17+
- [Docker](https://docs.docker.com/engine/install)
2018
- [Buf CLI](https://buf.build/docs/installation)
2119
- [Go 1.22+](https://go.dev/doc/install) _[optional, for local development]_
2220

2321
## Getting started
2422

25-
Before starting, perform the following steps to prepare your environment to run
26-
the demo.
23+
Before starting, perform the following steps to prepare your environment to run the demo.
2724

2825
1. Clone this repo:
2926

3027
```shellsession
3128
$ git clone https://github.com/bufbuild/bufstream-demo.git
32-
$ cd bufstream-demo
33-
```
34-
35-
1. Start downloading Docker images: _[optional]_
36-
37-
```shellsession
38-
$ docker compose pull
29+
$ cd ./bufstream-demo
3930
```
4031

4132
## Try out Bufstream
4233

43-
The Bufstream demo application simulates a small application that produces and
44-
consumes `EmailUpdated` event messages. The demo publishes these events to a
45-
Bufstream topic, followed by a separate consumer that fetches from the same
46-
topic and "verifies" the change. The demo app uses an off-the-shelf Kafka client
47-
([`franz-go`](https://github.com/twmb/franz-go)) to interact with Bufstream.
34+
The Bufstream demo application simulates two small applications that produce and consume
35+
[`EmailUpdated`](proto/bufstream/demo/v1/demo.proto) event messages. The demo
36+
[producer](cmd/bufstream-demo-produce/main.go) publishes these events to a Bufstream topic, followed
37+
by a separate [consumer](cmd/bufstream-demo-consume/main.go) that fetches from the same topic and
38+
"verifies" the change. The demo app uses the off-the-shelf Kafka client
39+
[`franz-go`](https://github.com/twmb/franz-go) to interact with Bufstream.
4840

4941
The demo attempts to publish and consume three payloads:
5042

5143
- A semantically valid, correctly formatted version of the `EmailUpdated` message.
5244
- A correctly formatted, but semantically invalid version of the message.
5345
- A malformed message.
5446

47+
Some relevant sections to check out:
48+
49+
- [cmd/bufstream-demo-produce](cmd/bufstream-demo-produce): The demo producer application.
50+
- [cmd/bufstream-demo-consume](cmd/bufstream-demo-consume): The demo consumer application.
51+
- [proto/bufstream/demo/v1/demo.proto](proto/bufstream/demo/v1/demo.proto): Defines the
52+
`EmailUpdated` message type, which is produced and consumed from the `email-updated` topic.
53+
- [pkg](pkg): Contains helper packages for both the producer and consumer.
54+
5555
### Replace Apache Kafka
5656

5757
1. Boot up the Bufstream and demo apps:
@@ -60,55 +60,52 @@ The demo attempts to publish and consume three payloads:
6060
$ docker compose up --build
6161
```
6262

63-
1. The app logs both the publishing of the events on the producer side and
64-
shortly after, the consumption of these events. The consumer deserializes
65-
the first two messages correctly, while the final message fails with a
66-
deserialization error.
63+
1. The app logs both the publishing of the events on the producer side and shortly after, the
64+
consumption of these events. The consumer deserializes the first two messages correctly, while
65+
the final message fails with a deserialization error.
6766

6867
At this point, you've used Bufstream as a drop-in replacement for Apache Kafka.
6968

7069
### Data quality enforcement
7170

72-
Thus far, Bufstream hasn't applied any quality enforcement on the data passing
73-
through it &mdash; addresses can be invalid or malformed. To ensure the quality
74-
of data flowing through Bufstream, you can configure policies that require data
75-
to conform to a pre-determined schema, pass semantic validation, and even
76-
redact sensitive information on the fly.
71+
Thus far, Bufstream hasn't applied any quality enforcement on the data passing through it &mdash;
72+
addresses can be invalid or malformed. To ensure the quality of data flowing through Bufstream, you
73+
can configure policies that require data to conform to a pre-determined schema, pass semantic
74+
validation, and even redact sensitive information on the fly.
7775

78-
We'll demonstrte this functionality using the Buf Schema Registry, but
79-
Bufstream also works with any registry that implements Confluent's REST API.
76+
We'll demonstrate this functionality using the Buf Schema Registry, but Bufstream also works with
77+
any registry that implements Confluent's REST API.
8078

81-
1. Uncomment the `data_enforcement` block in [`configs/bufstream.yaml`](configs/bufstream.yaml).
79+
1. Uncomment the `data_enforcement` block in [`config/bufstream.yaml`](config/bufstream.yaml).
8280

83-
1. Uncomment the `--csr` command options under the `demo` service in [`docker-compose.yaml`](docker-compose.yaml).
81+
1. Uncomment the `--csr` command options under the `demo` service in
82+
[`docker-compose.yaml`](docker-compose.yaml).
8483

85-
1. To pick up the configuration changes, terminate the Docker apps via `ctrl+c` and restart them with `docker compose up`.
84+
1. To pick up the configuration changes, terminate the Docker apps via `ctrl+c` and restart them
85+
with `docker compose up`.
8686

87-
1. The app again logs both the attempted publishing and consumption of the
88-
three events. The first event will successfully reach the consumer. But
89-
with Bufstream's data quality enforcement enabled, the second and third
90-
messages are rejected.
87+
1. The app again logs both the attempted publishing and consumption of the three events. The first
88+
event will successfully reach the consumer. But with Bufstream's data quality enforcement
89+
enabled, the second and third messages are rejected.
9190

92-
1. For the message that reaches the consumer, notice the empty `old_address` in
93-
the logs. This field has been redacted by Bufstream as it's labeled with the
94-
`debug_redact` option.
91+
1. For the message that reaches the consumer, notice the empty `old_address` in the logs. This field
92+
has been redacted by Bufstream as it's labeled with the `debug_redact` option.
9593

96-
By configuring Bufstream with a few data quality and governance policies,
97-
you've ensured that consumers receive only well-formed, semantically valid
98-
data.
94+
By configuring Bufstream with a few data quality and governance policies, you've ensured that
95+
consumers receive only well-formed, semantically valid data.
9996

10097
### Cleanup
10198

102-
After completing the demo, you can stop Docker and remove credentials from your
103-
machine with the following commands:
99+
After completing the demo, you can stop Docker and remove credentials from your machine with the
100+
following commands:
104101

105102
```
106103
docker compose down --rmi all
107104
```
108105

109106
## Curious to see more?
110107

111-
To learn more about Bufstream, check out the [launch blog
112-
post](https://buf.build/blog/bufstream-kafka-lower-cost), dig into the
113-
[benchmark and cost analysis](https://buf.build/docs/bufstream/cost), or [join
114-
us in the Buf Slack](https://buf.build/links/slack)!
108+
To learn more about Bufstream, check out the
109+
[launch blog post](https://buf.build/blog/bufstream-kafka-lower-cost), dig into the
110+
[benchmark and cost analysis](https://buf.build/docs/bufstream/cost), or
111+
[join us in the Buf Slack](https://buf.build/links/slack)!

‎buf.gen.yaml

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,20 @@
1+
# buf.gen.yaml files define how to generate stubs using the buf generate command.
2+
#
3+
# See https://buf.build/docs/generate/tutorial for more details.
14
version: v2
25
managed:
36
enabled: true
4-
disable:
5-
- file_option: go_package
6-
module: buf.build/bufbuild/protovalidate
77
override:
88
- file_option: go_package_prefix
99
value: github.com/bufbuild/bufstream-demo/gen
1010
- file_option: go_package_prefix
11-
module: buf.build/bufbuild/confluent
11+
module: bufdemo.buf.dev/bufbuild/confluent
1212
value: buf.build/gen/go/bufbuild/confluent/protocolbuffers/go
13+
disable:
14+
- file_option: go_package
15+
module: bufdemo.buf.dev/bufbuild/protovalidate
1316
plugins:
14-
- remote: buf.build/protocolbuffers/go
17+
- remote: bufdemo.buf.dev/protocolbuffers/go
1518
out: gen
16-
opt: paths=source_relative
19+
opt: paths=source_relative
20+
clean: true

‎buf.lock

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
# Generated by buf. DO NOT EDIT.
22
version: v2
33
deps:
4-
- name: bufdemo.buf.dev/bufbuild/confluent
5-
commit: 046489383fc54dd9baa85e938c5dea77
4+
- name: demo.buf.dev/bufbuild/confluent
5+
commit: 12f52fdd833847bf85ec7b9c8cf41074
66
digest: b5:874a7d1caa457939d815d93fe322707402e436ff1aeac04716182836832b125adf6378b144e9beeb10ccdb196f627d72d1460825e98d3e9f171192e23c17c775
7-
- name: bufdemo.buf.dev/bufbuild/protovalidate
8-
commit: 5f5d731ea1c64221ba1ffc26e40c4749
9-
digest: b5:2076a950fdf4a8047064d55fd1d20ef21e6d745bf56e3edf557071abd4488ed48c9466d60831d8a03489dc1fcc8ceaa073d197411b59ecd873e28b1328034e0b
7+
- name: demo.buf.dev/bufbuild/protovalidate
8+
commit: fec0a55454344f4a9741e4527ec929bc
9+
digest: b5:0f2dc6c9453e9cc9e9f36807aaa5f94022e837d91fef4dcaeed79a35c0843cc64eba28ff077aab24da3b2cb12639ad256246f9f9a36c033b99d5754b19996b7e

‎buf.yaml

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
1+
# buf.yaml files define how to build the .proto files within your local directory
2+
#
3+
# See https://buf.build/docs/tutorials/getting-started-with-buf-cli for more details.
14
version: v2
25
modules:
6+
# Our .proto files live within the proto directory.
37
- path: proto
4-
name: "bufdemo.buf.dev/bufbuild/bufstream-demo"
8+
name: demo.buf.dev/bufbuild/bufstream-demo
59
deps:
6-
- bufdemo.buf.dev/bufbuild/protovalidate
7-
- bufdemo.buf.dev/bufbuild/confluent
8-
lint:
9-
use:
10-
- DEFAULT
11-
breaking:
12-
use:
13-
- FILE
10+
# We import "buf/confluent/v1/extensions.proto" within our example files, which
11+
# comes from the demo.buf.dev/bufbuild/protovalidate module.
12+
- demo.buf.dev/bufbuild/confluent
13+
# We import "buf/validate/validate.proto" within our example files, which
14+
# comes from the demo.buf.dev/bufbuild/protovalidate module.
15+
- demo.buf.dev/bufbuild/protovalidate

0 commit comments

Comments
 (0)