Merge pull request #31 from bmaidics/cdktf_byoc

Add MSK CDKTF BYOC

amazon-msk/cdktf/iot-ingest-and-control/main.ts

@@ -25,8 +25,18 @@ import { IamInstanceProfile } from "@cdktf/provider-aws/lib/iam-instance-profile
 import { IamRole } from "@cdktf/provider-aws/lib/iam-role";
 import { IamRolePolicy } from "@cdktf/provider-aws/lib/iam-role-policy";
 import { SecurityGroup } from "@cdktf/provider-aws/lib/security-group";
-
 import { UserVariables } from "./variables";
+import Mustache = require("mustache");
+import fs =  require("fs");
+
+interface TemplateData {
+  name: string;
+  cloudwatch?: object;
+  public?: object;
+  topics?: object;
+  kafka?: object;
+
+}
 
 export class ZillaPlusIotAndControlStack extends TerraformStack {
   constructor(scope: Construct, id: string) {
@@ -316,8 +326,9 @@ export class ZillaPlusIotAndControlStack extends TerraformStack {
       errorMessage: "must be a valid EC2 instance type.",
     });
 
-    let zillaTelemetryContent = "";
-    let bindingTelemetryContent = "";
+    const data: TemplateData = {
+      name: 'iot',
+    }
 
     if (!userVariables.cloudwatchDisabled) {
       const defaultLogGroupName = `${id}-group`;
@@ -339,41 +350,14 @@ export class ZillaPlusIotAndControlStack extends TerraformStack {
         name: cloudWatchLogsGroup.stringValue,
       });
 
-      const logsSection = `
-        logs:
-          group: ${cloudWatchLogsGroup.stringValue}
-          stream: events`;
-
-      const metricsSection = `
-        metrics:
-          namespace: ${cloudWatchMetricsNamespace.stringValue}`;
-
-      zillaTelemetryContent = `
-telemetry:
-  metrics:
-    - stream.active.received
-    - stream.active.sent
-    - stream.opens.received
-    - stream.opens.sent
-    - stream.data.received
-    - stream.data.sent
-    - stream.errors.received
-    - stream.errors.sent
-    - stream.closes.received
-    - stream.closes.sent
-  exporters:
-    stdout_logs_exporter:
-      type: stdout
-    aws0:
-      type: aws-cloudwatch
-      options:
-${logsSection}
-${metricsSection}`;
-
-      bindingTelemetryContent = `
-    telemetry:
-      metrics:
-        - stream.*`;
+      data.cloudwatch = {
+        logs: {
+          group: cloudWatchLogsGroup.stringValue
+        },
+        metrics: {
+          namespace: cloudWatchMetricsNamespace.stringValue
+        } 
+      };
     }
 
     const ami = new dataAwsAmi.DataAwsAmi(this, "LatestAmi", {
@@ -425,81 +409,25 @@ ${metricsSection}`;
 
     const kafkaBootstrapServers = `['${Fn.join(`','`, Fn.split(",", mskCluster.bootstrapBrokersSaslScram))}']`;
 
-    const zillaYamlContent = `
-name: public
-${zillaTelemetryContent}
-vaults:
-  secure:
-    type: aws
-bindings:
-  tcp_server:
-    type: tcp
-    kind: server
-${bindingTelemetryContent}
-    options:
-      host: 0.0.0.0
-      port: ${publicTcpPort}
-    exit: tls_server
-  tls_server:
-    type: tls
-    kind: server
-    vault: secure
-${bindingTelemetryContent}
-    options:
-      keys:
-      - ${publicTlsCertificateKey.stringValue}
-    exit: mqtt_server
-  mqtt_server:
-    type: mqtt
-    kind: server
-${bindingTelemetryContent}
-    exit: mqtt_kafka_mapping
-  mqtt_kafka_mapping:
-    type: mqtt-kafka
-    kind: proxy
-${bindingTelemetryContent}
-    options:
-      topics:
-        sessions: ${kafkaTopicMqttSessions}
-        messages: ${kafkaTopicMqttMessages}
-        retained: ${kafkaTopicMqttRetained}
-    exit: kafka_cache_client
-  kafka_cache_client:
-    type: kafka
-    kind: cache_client
-${bindingTelemetryContent}
-    exit: kafka_cache_server
-  kafka_cache_server:
-    type: kafka
-    kind: cache_server
-${bindingTelemetryContent}
-    options:
-      bootstrap:
-        - ${kafkaTopicMqttMessages}
-        - ${kafkaTopicMqttRetained}
-    exit: kafka_client
-  kafka_client:
-    type: kafka
-    kind: client
-    options:
-      servers: ${kafkaBootstrapServers}
-      sasl:
-        mechanism: scram-sha-512
-        username: '${kafkaSaslUsername}'
-        password: '${kafkaSaslPassword}'
-${bindingTelemetryContent}
-    exit: tls_client
-  tls_client:
-    type: tls
-    kind: client
-    vault: secure
-${bindingTelemetryContent}
-    exit: tcp_client
-  tcp_client:
-    type: tcp
-    kind: client
-${bindingTelemetryContent}
-`;
+    data.kafka = {
+      bootstrapServers: kafkaBootstrapServers,
+      sasl : {
+        username: kafkaSaslUsername,
+        password: kafkaSaslPassword
+      }
+    }
+    data.public = {
+      port: publicTcpPort.value,
+      tlsCertificateKey: publicTlsCertificateKey.stringValue
+    }
+    data.topics = {
+      sessions: kafkaTopicMqttSessions.stringValue,
+      messages: kafkaTopicMqttMessages.stringValue,
+      retained: kafkaTopicMqttRetained.stringValue
+    };
+
+    const yamlTemplate: string = fs.readFileSync('zilla.yaml.mustache', 'utf8');
+    const renderedYaml: string = Mustache.render(yamlTemplate, data);
 
     const cfnHupConfContent = `
 [main]
@@ -538,7 +466,7 @@ END_HELP
     const userData = `#!/bin/bash -xe
 yum update -y aws-cfn-bootstrap
 cat <<'END_HELP' > /etc/zilla/zilla.yaml
-${zillaYamlContent}
+${renderedYaml}
 END_HELP
 
 chown ec2-user:ec2-user /etc/zilla/zilla.yaml

amazon-msk/cdktf/iot-ingest-and-control/package.json

@@ -26,10 +26,12 @@
     "aws-sdk": "^2.1668.0",
     "cdktf": "^0.20.8",
     "constructs": "^10.3.0",
-    "kafkajs": "^2.2.4"
+    "kafkajs": "^2.2.4",
+    "mustache": "^4.2.0"
   },
   "devDependencies": {
     "@types/jest": "^29.5.12",
+    "@types/mustache": "^4.2.5",
     "@types/node": "^20.14.11",
     "cdktf-cli": "^0.20.8",
     "jest": "^29.7.0",

amazon-msk/cdktf/iot-ingest-and-control/zilla.yaml.mustache

@@ -0,0 +1,115 @@
+name: {{name}}
+vaults:
+  secure:
+    type: aws-secrets
+telemetry:
+  metrics:
+    - stream.active.received
+    - stream.active.sent
+    - stream.opens.received
+    - stream.opens.sent
+    - stream.data.received
+    - stream.data.sent
+    - stream.errors.received
+    - stream.errors.sent
+    - stream.closes.received
+    - stream.closes.sent
+  exporters:
+    stdout_logs_exporter:
+      type: stdout
+{{#cloudwatch}}
+    aws0:
+      type: aws-cloudwatch
+      options:
+        logs:
+          group: {{cloudwatch.logs.group}}
+          stream: events
+        metrics:
+          namespace: {{cloudwatch.metrics.namespace}}
+{{/cloudwatch}}
+bindings:
+  tcp_server:
+    type: tcp
+    kind: server
+    options:
+      host: 0.0.0.0
+      port: {{public.port}}
+    telemetry:
+      metrics:
+        - stream.*
+    exit: tls_server
+  tls_server:
+    type: tls
+    kind: server
+    vault: secure
+    options:
+      keys:
+        - {{public.tlsCertificateKey}}
+    telemetry:
+      metrics:
+        - stream.*
+    exit: mqtt_server
+  mqtt_server:
+    type: mqtt
+    kind: server
+    telemetry:
+      metrics:
+        - stream.*
+    exit: mqtt_kafka_mapping
+  mqtt_kafka_mapping:
+    type: mqtt-kafka
+    kind: proxy
+    options:
+      topics:
+        sessions: {{topics.sessions}}
+        messages: {{topics.messages}}
+        retained: {{topics.retained}}
+    telemetry:
+      metrics:
+        - stream.*
+    exit: kafka_cache_client
+  kafka_cache_client:
+    type: kafka
+    kind: cache_client
+    telemetry:
+      metrics:
+        - stream.*
+    exit: kafka_cache_server
+  kafka_cache_server:
+    type: kafka
+    kind: cache_server
+    options:
+      bootstrap:
+        - {{topics.messages}}
+        - {{topics.retained}}
+    telemetry:
+      metrics:
+        - stream.*
+    exit: kafka_client
+  kafka_client:
+    type: kafka
+    kind: client
+    options:
+      servers: {{{kafka.bootstrapServers}}}
+      sasl:
+        mechanism: scram-sha-512
+        username: '{{kafka.sasl.username}}'
+        password: '{{kafka.sasl.password}}'
+    telemetry:
+      metrics:
+        - stream.*
+    exit: tls_client
+  tls_client:
+    type: tls
+    kind: client
+    vault: secure
+    telemetry:
+      metrics:
+        - stream.*
+    exit: tcp_client
+  tcp_client:
+    type: tcp
+    kind: client
+    telemetry:
+      metrics:
+        - stream.*