init

Dockerfile

@@ -0,0 +1,20 @@
+ARG STRIMZI_VERSION="0.32.0"
+ARG KAFKA_VERSION="3.3.1"
+
+FROM quay.io/strimzi/kafka:${STRIMZI_VERSION}-kafka-${KAFKA_VERSION}
+
+ARG AWS_MSK_IAM_AUTH_VERSION="1.1.6"
+ENV CLASSPATH=/opt/kafka/libs/aws-msk-iam-auth-${AWS_MSK_IAM_AUTH_VERSION}-all.jar
+
+USER root
+RUN curl -sSL -o /opt/kafka/libs/aws-msk-iam-auth-${AWS_MSK_IAM_AUTH_VERSION}-all.jar https://github.com/aws/aws-msk-iam-auth/releases/download/v${AWS_MSK_IAM_AUTH_VERSION}/aws-msk-iam-auth-${AWS_MSK_IAM_AUTH_VERSION}-all.jar
+
+COPY kafka_connect_config_generator.sh /opt/kafka/kafka_connect_config_generator.sh
+COPY kafka_mirror_maker_2_connector_config_generator.sh /opt/kafka/kafka_mirror_maker_2_connector_config_generator.sh
+COPY kafka_mirror_maker_consumer_config_generator.sh /opt/kafka/kafka_mirror_maker_consumer_config_generator.sh
+COPY kafka_mirror_maker_producer_config_generator.sh /opt/kafka/kafka_mirror_maker_producer_config_generator.sh
+
+RUN chmod +x /opt/kafka/kafka_connect_config_generator.sh \
+ && chmod +x /opt/kafka/kafka_mirror_maker_2_connector_config_generator.sh \
+ && chmod +x /opt/kafka/kafka_mirror_maker_consumer_config_generator.sh \
+ && chmod +x /opt/kafka/kafka_mirror_maker_producer_config_generator.sh

Makefile

@@ -0,0 +1,27 @@
+SHELL := /bin/bash
+NAME := strimzi-kafka-operator-aws-msk-iam-auth
+STRIMZI_VERSION := 0.32.0
+KAFKA_VERSION := 3.3.1
+AWS_MSK_IAM_AUTH_VERSION := 1.1.6
+TAG := ${STRIMZI_VERSION}-kafka-${KAFKA_VERSION}-${AWS_MSK_IAM_AUTH_VERSION}
+
+.PHONY: help all
+
+help: ## This help.
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-35s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+.DEFAULT_GOAL := help
+
+build: ## Build docker images
+	docker build . -t $(NAME) --build-arg STRIMZI_VERSION=$(STRIMZI_VERSION) --build-arg KAFKA_VERSION=$(KAFKA_VERSION) --build-arg AWS_MSK_IAM_AUTH_VERSION=$(AWS_MSK_IAM_AUTH_VERSION)  
+	docker tag $(NAME) aidanmelen/$(NAME):$(TAG)
+	docker tag $(NAME) aidanmelen/$(NAME):latest
+
+dev: ## dev docker images
+	docker run -it --rm --entrypoint /bin/bash $(NAME)
+
+release: build ## Push docker images
+	docker login aidanmelen
+
+	docker push aidanmelen/$(NAME):$(TAG)
+	docker push aidanmelen/$(NAME):latest

README.md

@@ -1,5 +1,3 @@
 # strimzi-kafka-operator-aws-msk-iam-auth
 
 A pattern for extending the `kafka-base` image from the strimzi-kafka-operator to support SASL/IAM authentication mechanism for AWS MSK.# strimzi-kafka-operator-aws-msk-iam-auth
-# strimzi-kafka-operator-aws-msk-iam-auth
-# strimzi-kafka-operator-aws-msk-iam-auth

examples/connect.yaml

@@ -0,0 +1,28 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaConnect
+metadata:
+  name: my-connect
+spec:
+  image: aidanmelen/strimzi-kafka-operator-aws-msk-iam-auth:0.32.0-kafka-3.3.1-1.1.6
+  replicas: 1
+  bootstrapServers: "${BOOTSTRAP_BROKERS_SASL_IAM}"
+  template:
+    config:
+      group.id: connect-cluster
+      offset.storage.topic: _connect-storage
+      config.storage.topic: _connect-offset
+      status.storage.topic: _connect-status
+
+    # Uncomment for EKS IRSA credentials
+    # serviceAccount:
+    #   metadata:
+    #     annotations:
+    #       "eks.amazonaws.com/role-arn": "${AWS_ROLE_ARN}"
+
+    # Uncomment for IAM User credentials
+    # connectContainer:
+    #   env:        
+    #     - name: AWS_ACCESS_KEY_ID
+    #       value: "${AWS_ACCESS_KEY_ID}"
+    #     - name: AWS_SECRET_ACCESS_KEY
+    #       value: "${AWS_SECRET_ACCESS_KEY}"

examples/mirrormaker2.yaml

@@ -0,0 +1,57 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaMirrorMaker2
+metadata:
+  name: my-mirror-maker-2
+spec:
+  image: aidanmelen/strimzi-kafka-operator-aws-msk-iam-auth:0.32.0-kafka-3.3.1-1.1.6
+  replicas: 1
+  bootstrapServers: "${BOOTSTRAP_BROKERS_SASL_IAM}"
+  template:
+    config:
+      group.id: connect-cluster
+      offset.storage.topic: _connect-storage
+      config.storage.topic: _connect-offset
+      status.storage.topic: _connect-status
+
+    # Uncomment for EKS IRSA credentials
+    # serviceAccount:
+    #   metadata:
+    #     annotations:
+    #       "eks.amazonaws.com/role-arn": "${AWS_ROLE_ARN}"
+
+    # Uncomment for IAM User credentials
+    # connectContainer:
+    #   env:        
+    #     - name: AWS_ACCESS_KEY_ID
+    #       value: "${AWS_ACCESS_KEY_ID}"
+    #     - name: AWS_SECRET_ACCESS_KEY
+    #       value: "${AWS_SECRET_ACCESS_KEY}"
+
+  connectCluster: "my-target-cluster"
+  clusters:
+    - alias: "my-source-cluster"
+      bootstrapServers: my-source-cluster-kafka-bootstrap:9098
+    - alias: "my-target-cluster"
+      bootstrapServers: my-target-cluster-kafka-bootstrap:9098
+      config:
+        # -1 means it will use the default replication factor configured in the broker
+        config.storage.replication.factor: -1
+        offset.storage.replication.factor: -1
+        status.storage.replication.factor: -1
+  mirrors:
+    - sourceCluster: "my-source-cluster"
+      targetCluster: "my-target-cluster"
+      sourceConnector:
+        config:
+          replication.factor: 1
+          offset-syncs.topic.replication.factor: 1
+          sync.topic.acls.enabled: "false"
+      heartbeatConnector:
+        config:
+          heartbeats.topic.replication.factor: 1
+      checkpointConnector:
+        config:
+          checkpoints.topic.replication.factor: 1
+          sync.group.offsets.enabled: "true"
+      topicsPattern: ".*"
+      groupsPattern: ".*"

kafka_connect_config_generator.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -e
+
+# Write the config file
+cat <<EOF
+# Bootstrap servers
+bootstrap.servers=${KAFKA_CONNECT_BOOTSTRAP_SERVERS}
+# REST Listeners
+rest.port=8083
+rest.advertised.host.name=$(hostname -I | awk '{ print $1 }')
+rest.advertised.port=8083
+# Plugins
+plugin.path=${KAFKA_CONNECT_PLUGIN_PATH}
+# Provided configuration
+${KAFKA_CONNECT_CONFIGURATION}
+
+# AWS MSK SASL/IAM
+security.protocol=SASL_SSL
+sasl.mechanism=AWS_MSK_IAM
+sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
+sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+admin.security.protocol=SASL_SSL
+admin.sasl.mechanism=AWS_MSK_IAM
+admin.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
+admin.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+producer.security.protocol=SASL_SSL
+producer.sasl.mechanism=AWS_MSK_IAM
+producer.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
+producer.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+consumer.security.protocol=SASL_SSL
+consumer.sasl.mechanism=AWS_MSK_IAM
+consumer.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
+consumer.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+
+# Additional configuration
+consumer.client.rack=${STRIMZI_RACK_ID}
+EOF

kafka_mirror_maker_2_connector_config_generator.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -e
+
+# Write the config file
+cat <<EOF
+security.protocol=SASL_SSL
+sasl.mechanism=AWS_MSK_IAM
+sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
+sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+admin.security.protocol=SASL_SSL
+admin.sasl.mechanism=AWS_MSK_IAM
+admin.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
+admin.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+producer.security.protocol=SASL_SSL
+producer.sasl.mechanism=AWS_MSK_IAM
+producer.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
+producer.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+consumer.security.protocol=SASL_SSL
+consumer.sasl.mechanism=AWS_MSK_IAM
+consumer.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
+consumer.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+EOF

kafka_mirror_maker_consumer_config_generator.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -e
+
+# Write the config file
+cat <<EOF
+# Bootstrap servers
+bootstrap.servers=${KAFKA_MIRRORMAKER_BOOTSTRAP_SERVERS_CONSUMER}
+# Consumer group
+group.id=${KAFKA_MIRRORMAKER_GROUPID_CONSUMER}
+# Provided configuration
+${KAFKA_MIRRORMAKER_CONFIGURATION_CONSUMER}
+
+security.protocol=SASL_SSL
+sasl.mechanism=AWS_MSK_IAM
+sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
+sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+admin.security.protocol=SASL_SSL
+admin.sasl.mechanism=AWS_MSK_IAM
+admin.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
+admin.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+producer.security.protocol=SASL_SSL
+producer.sasl.mechanism=AWS_MSK_IAM
+producer.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
+producer.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+consumer.security.protocol=SASL_SSL
+consumer.sasl.mechanism=AWS_MSK_IAM
+consumer.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
+consumer.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+EOF

kafka_mirror_maker_producer_config_generator.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -e
+
+# Write the config file
+cat <<EOF
+# Bootstrap servers
+bootstrap.servers=${KAFKA_MIRRORMAKER_BOOTSTRAP_SERVERS_PRODUCER}
+# Provided configuration
+${KAFKA_MIRRORMAKER_CONFIGURATION_PRODUCER}
+
+security.protocol=SASL_SSL
+sasl.mechanism=AWS_MSK_IAM
+sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
+sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+admin.security.protocol=SASL_SSL
+admin.sasl.mechanism=AWS_MSK_IAM
+admin.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
+admin.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+producer.security.protocol=SASL_SSL
+producer.sasl.mechanism=AWS_MSK_IAM
+producer.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
+producer.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+consumer.security.protocol=SASL_SSL
+consumer.sasl.mechanism=AWS_MSK_IAM
+consumer.sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required;
+consumer.sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+EOF