From 0024b074e165f3b5c09792eff5492fee27c1a455 Mon Sep 17 00:00:00 2001 From: bmaidics Date: Tue, 8 Oct 2024 17:23:20 +0200 Subject: [PATCH 1/5] Add MSK CDKTF BYOC --- amazon-msk/cdktf/secure-public-access/main.ts | 154 ++++++------------ .../secure-public-access/package-lock.json | 16 ++ .../cdktf/secure-public-access/package.json | 2 + .../cdktf/secure-public-access/zilla.mustache | 109 +++++++++++++ 4 files changed, 176 insertions(+), 105 deletions(-) create mode 100644 amazon-msk/cdktf/secure-public-access/zilla.mustache diff --git a/amazon-msk/cdktf/secure-public-access/main.ts b/amazon-msk/cdktf/secure-public-access/main.ts index 7f5a6c1..3f33988 100644 --- a/amazon-msk/cdktf/secure-public-access/main.ts +++ b/amazon-msk/cdktf/secure-public-access/main.ts @@ -25,11 +25,31 @@ import { DataAwsAvailabilityZones } from "@cdktf/provider-aws/lib/data-aws-avail import { DataAwsSubnets } from "@cdktf/provider-aws/lib/data-aws-subnets"; import { IamInstanceProfile } from "@cdktf/provider-aws/lib/iam-instance-profile"; + import { UserVariables } from "./variables"; import { AwsProvider } from "@cdktf/provider-aws/lib/provider"; import { ec2EnclaveCertificateIamRoleAssociation } from "./.gen/providers/awscc" import { AwsccProvider } from "./.gen/providers/awscc/provider"; - +import Mustache = require("mustache"); +import fs = require("fs"); + +interface TemplateData { + name: string; + useAcm: boolean; + cloudwatchDisabled?: boolean; + cloudWatchLogsGroup?: string; + cloudWatchMetricsNamespace?: string; + publicPort?: number; + publicTlsCertificateKey?: string; + mTLS?: boolean; + publicCertificateAuthority?: string; + publicWildcardDNS?: string; + externalHost?: string; + internalHost?: string; + mskPort?: number; + mskCertificateAuthority?: string; + mskWildcardDNS?: string; +} export class ZillaPlusSecurePublicAccessStack extends TerraformStack { constructor(scope: Construct, id: string) { @@ -147,9 +167,17 @@ export class ZillaPlusSecurePublicAccessStack extends TerraformStack { const mskBootstrapCommonPart = Fn.join(".", Fn.slice(addressParts, 1, Fn.lengthOf(addressParts))); mskWildcardDNS = Fn.format("*.%s", [mskBootstrapCommonPart]); - let tlsTrust = ""; - let tlsClientSigners = ""; - if (mskClientAuthentication === "mTLS") { + const mTLSEnabled = mskClientAuthentication === "mTLS"; + const publicTlsCertificateViaAcm = userVariables.publicTlsCertificateViaAcm; + + const data: TemplateData = { + name: 'public', + useAcm: publicTlsCertificateViaAcm, + cloudwatchDisabled: userVariables.cloudwatchDisabled, + mTLS: mTLSEnabled + }; + + if (mTLSEnabled) { // Seems like we can't get this from the MSK Cluster const mskCertificateAuthorityVar = new TerraformVariable(this, "msk_certificate_authority_arn", { type: "string", @@ -176,11 +204,7 @@ export class ZillaPlusSecurePublicAccessStack extends TerraformStack { }); publicCertificateAuthority = publicCertificateAuthorityVar.stringValue; } - - tlsTrust = ` trust: - - ${publicCertificateAuthority}`; - tlsClientSigners = ` signers: -- ${mskCertificateAuthority}`; + data.publicCertificateAuthority = publicCertificateAuthority; } const publicTlsCertificateKey = new TerraformVariable(this, "public_tls_certificate_key", { @@ -188,8 +212,6 @@ export class ZillaPlusSecurePublicAccessStack extends TerraformStack { description: "TLS Certificate SecretsManager or CertificateManager ARN", }); - const publicTlsCertificateViaAcm = userVariables.publicTlsCertificateViaAcm; - let zillaPlusRole; if (!userVariables.createZillaPlusRole) { const zillaPlusRoleVar = new TerraformVariable(this, "zilla_plus_role_name", { @@ -379,8 +401,6 @@ export class ZillaPlusSecurePublicAccessStack extends TerraformStack { let acmYamlContent = ""; let enclavesAcmServiceStart = ""; - let zillaTelemetryContent = ""; - let bindingTelemetryContent = ""; if (publicTlsCertificateViaAcm) { acmYamlContent = ` @@ -425,41 +445,8 @@ systemctl start nitro-enclaves-acm.service name: cloudWatchLogsGroup.stringValue, }); - const logsSection = ` - logs: - group: ${cloudWatchLogsGroup.stringValue} - stream: events`; - - const metricsSection = ` - metrics: - namespace: ${cloudWatchMetricsNamespace.stringValue}`; - - zillaTelemetryContent = ` -telemetry: - metrics: - - stream.active.received - - stream.active.sent - - stream.opens.received - - stream.opens.sent - - stream.data.received - - stream.data.sent - - stream.errors.received - - stream.errors.sent - - stream.closes.received - - stream.closes.sent - exporters: - stdout_logs_exporter: - type: stdout - aws0: - type: aws-cloudwatch - options: -${logsSection} -${metricsSection}`; - - bindingTelemetryContent = ` - telemetry: - metrics: - - stream.*`; + data.cloudWatchLogsGroup = cloudWatchLogsGroup.stringValue; + data.cloudWatchMetricsNamespace = cloudWatchMetricsNamespace.stringValue; } const instanceType = new TerraformVariable(this, "zilla_plus_instance_type", { @@ -522,62 +509,19 @@ ${metricsSection}`; const internalHost = ["b-#.", Fn.element(Fn.split("*.", mskWildcardDNS), 1)].join(""); - const zillaYamlContent = ` -name: public -vaults: - secure: - type: ${publicTlsCertificateViaAcm ? "aws-acm" : "aws-secrets"} -${zillaTelemetryContent} -bindings: - tcp_server: - type: tcp - kind: server - options: - host: 0.0.0.0 - port: ${publicPort} -${bindingTelemetryContent} - exit: tls_server - tls_server: - type: tls - kind: server - vault: secure - options: - keys: - - ${publicTlsCertificateKey.stringValue} -${tlsTrust} - routes: - - exit: kafka_proxy - when: - - authority: '${publicWildcardDNS.stringValue}' - kafka_proxy: - type: kafka-proxy - kind: proxy - options: - external: - host: '${externalHost}' - port: ${publicPort} - internal: - host: '${internalHost}' - port: ${mskPort} - exit: tls_client - tls_client: - type: tls - kind: client - vault: secure - options: -${tlsClientSigners} - trustcacerts: true - exit: tcp_client - tcp_client: - type: tcp - kind: client - options: - host: '*' - port: ${mskPort} - routes: - - when: - - authority: '${mskWildcardDNS}' - `; + data.publicPort = publicPort.value; + data.publicTlsCertificateKey = publicTlsCertificateKey.stringValue; + data.publicWildcardDNS = publicWildcardDNS.stringValue; + data.externalHost = externalHost; + data.internalHost = internalHost; + data.mskPort = mskPort; + data.mskWildcardDNS = mskWildcardDNS; + const yamlTemplate: string = fs.readFileSync('zilla.mustache', 'utf8'); + const renderedYaml: string = Mustache.render(yamlTemplate, data); + + new TerraformOutput(this, "renderedYaml", { + value: renderedYaml, + }); const cfnHupConfContent = ` [main] @@ -596,7 +540,7 @@ runas=root const userData = `#!/bin/bash -xe yum update -y aws-cfn-bootstrap cat < /etc/zilla/zilla.yaml -${zillaYamlContent} +${renderedYaml} EOF cat < /etc/nitro_enclaves/acm.yaml diff --git a/amazon-msk/cdktf/secure-public-access/package-lock.json b/amazon-msk/cdktf/secure-public-access/package-lock.json index 8f9d180..eb4c64a 100644 --- a/amazon-msk/cdktf/secure-public-access/package-lock.json +++ b/amazon-msk/cdktf/secure-public-access/package-lock.json @@ -16,10 +16,12 @@ "aws-cdk-lib": "^2.158.0", "cdktf": "^0.20.8", "constructs": "^10.3.0", + "mustache": "^4.2.0", "netmask": "^2.0.2" }, "devDependencies": { "@types/jest": "^29.5.12", + "@types/mustache": "^4.2.5", "@types/netmask": "^2.0.5", "@types/node": "^20.14.10", "cdktf-cli": "^0.20.8", @@ -3888,6 +3890,12 @@ "pretty-format": "^29.0.0" } }, + "node_modules/@types/mustache": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/@types/mustache/-/mustache-4.2.5.tgz", + "integrity": "sha512-PLwiVvTBg59tGFL/8VpcGvqOu3L4OuveNvPi0EYbWchRdEVP++yRUXJPFl+CApKEq13017/4Nf7aQ5lTtHUNsA==", + "dev": true + }, "node_modules/@types/mute-stream": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.1.tgz", @@ -9218,6 +9226,14 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/mute-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", diff --git a/amazon-msk/cdktf/secure-public-access/package.json b/amazon-msk/cdktf/secure-public-access/package.json index 09fcc0b..91bc553 100644 --- a/amazon-msk/cdktf/secure-public-access/package.json +++ b/amazon-msk/cdktf/secure-public-access/package.json @@ -28,10 +28,12 @@ "aws-cdk-lib": "^2.158.0", "cdktf": "^0.20.8", "constructs": "^10.3.0", + "mustache": "^4.2.0", "netmask": "^2.0.2" }, "devDependencies": { "@types/jest": "^29.5.12", + "@types/mustache": "^4.2.5", "@types/netmask": "^2.0.5", "@types/node": "^20.14.10", "cdktf-cli": "^0.20.8", diff --git a/amazon-msk/cdktf/secure-public-access/zilla.mustache b/amazon-msk/cdktf/secure-public-access/zilla.mustache new file mode 100644 index 0000000..15f8f82 --- /dev/null +++ b/amazon-msk/cdktf/secure-public-access/zilla.mustache @@ -0,0 +1,109 @@ +name: {{name}} +vaults: + secure: + type: {{#useAcm}}aws-acm{{/useAcm}}{{^useAcm}}aws-secrets{{/useAcm}} +{{^cloudwatchDisabled}} +telemetry: + metrics: + - stream.active.received + - stream.active.sent + - stream.opens.received + - stream.opens.sent + - stream.data.received + - stream.data.sent + - stream.errors.received + - stream.errors.sent + - stream.closes.received + - stream.closes.sent + exporters: + stdout_logs_exporter: + type: stdout + aws0: + type: aws-cloudwatch + options: + logs: + group: {{cloudWatchLogsGroup}} + stream: events + metrics: + namespace: {{cloudWatchMetricsNamespace}} +{{/cloudwatchDisabled}} +bindings: + tcp_server: + type: tcp + kind: server + options: + host: 0.0.0.0 + port: {{publicPort}} +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: tls_server + tls_server: + type: tls + kind: server + vault: secure + options: + keys: + - {{publicTlsCertificateKey}} + {{#mTLS}} + trust: + - {{publicCertificateAuthority}} + {{/mTLS}} +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + routes: + - exit: kafka_proxy + when: + - authority: '{{publicWildcardDNS}}' + kafka_proxy: + type: kafka-proxy + kind: proxy + options: + external: + host: '{{externalHost}}' + port: {{publicPort}} + internal: + host: '{{internalHost}}' + port: {{mskPort}} +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: tls_client + tls_client: + type: tls + kind: client + vault: secure + options: + {{#mTLS}} + signers: + - {{mskCertificateAuthority}} + {{/mTLS}} + trustcacerts: true +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: tcp_client + tcp_client: + type: tcp + kind: client + options: + host: '*' + port: {{mskPort}} +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + routes: + - when: + - authority: '{{mskWildcardDNS}}' + From 43076dc438723fa5424b69956dcab3631603b3b3 Mon Sep 17 00:00:00 2001 From: bmaidics Date: Thu, 10 Oct 2024 14:48:02 +0200 Subject: [PATCH 2/5] Add IOT and Web byoc --- .../cdktf/iot-ingest-and-control/main.ts | 149 +++-------- .../iot-ingest-and-control/package-lock.json | 19 +- .../cdktf/iot-ingest-and-control/package.json | 4 +- .../terraform.tfvars.example | 6 +- .../iot-ingest-and-control/zilla.mustache | 133 ++++++++++ amazon-msk/cdktf/secure-public-access/main.ts | 4 - amazon-msk/cdktf/web-streaming/main.ts | 243 +++--------------- .../cdktf/web-streaming/package-lock.json | 18 +- amazon-msk/cdktf/web-streaming/package.json | 4 +- amazon-msk/cdktf/web-streaming/variables.ts | 2 +- amazon-msk/cdktf/web-streaming/zilla.mustache | 232 +++++++++++++++++ 11 files changed, 482 insertions(+), 332 deletions(-) create mode 100644 amazon-msk/cdktf/iot-ingest-and-control/zilla.mustache create mode 100644 amazon-msk/cdktf/web-streaming/zilla.mustache diff --git a/amazon-msk/cdktf/iot-ingest-and-control/main.ts b/amazon-msk/cdktf/iot-ingest-and-control/main.ts index 1cbc9be..c71234d 100644 --- a/amazon-msk/cdktf/iot-ingest-and-control/main.ts +++ b/amazon-msk/cdktf/iot-ingest-and-control/main.ts @@ -25,8 +25,24 @@ import { IamInstanceProfile } from "@cdktf/provider-aws/lib/iam-instance-profile import { IamRole } from "@cdktf/provider-aws/lib/iam-role"; import { IamRolePolicy } from "@cdktf/provider-aws/lib/iam-role-policy"; import { SecurityGroup } from "@cdktf/provider-aws/lib/security-group"; - import { UserVariables } from "./variables"; +import Mustache = require("mustache"); +import fs = require("fs"); + +interface TemplateData { + name: string; + cloudwatchDisabled?: boolean; + cloudWatchLogsGroup?: string; + cloudWatchMetricsNamespace?: string; + publicPort?: number; + publicTlsCertificateKey?: string; + kafkaTopicMqttSessions?: string; + kafkaTopicMqttMessages?: string; + kafkaTopicMqttRetained?: string; + kafkaBootstrapServers?: string; + kafkaSaslUsername?: string; + kafkaSaslPassword?: string; +} export class ZillaPlusIotAndControlStack extends TerraformStack { constructor(scope: Construct, id: string) { @@ -316,8 +332,10 @@ export class ZillaPlusIotAndControlStack extends TerraformStack { errorMessage: "must be a valid EC2 instance type.", }); - let zillaTelemetryContent = ""; - let bindingTelemetryContent = ""; + const data: TemplateData = { + name: 'iot', + cloudwatchDisabled: userVariables.cloudwatchDisabled + } if (!userVariables.cloudwatchDisabled) { const defaultLogGroupName = `${id}-group`; @@ -339,41 +357,8 @@ export class ZillaPlusIotAndControlStack extends TerraformStack { name: cloudWatchLogsGroup.stringValue, }); - const logsSection = ` - logs: - group: ${cloudWatchLogsGroup.stringValue} - stream: events`; - - const metricsSection = ` - metrics: - namespace: ${cloudWatchMetricsNamespace.stringValue}`; - - zillaTelemetryContent = ` -telemetry: - metrics: - - stream.active.received - - stream.active.sent - - stream.opens.received - - stream.opens.sent - - stream.data.received - - stream.data.sent - - stream.errors.received - - stream.errors.sent - - stream.closes.received - - stream.closes.sent - exporters: - stdout_logs_exporter: - type: stdout - aws0: - type: aws-cloudwatch - options: -${logsSection} -${metricsSection}`; - - bindingTelemetryContent = ` - telemetry: - metrics: - - stream.*`; + data.cloudWatchLogsGroup = cloudWatchLogsGroup.stringValue; + data.cloudWatchMetricsNamespace = cloudWatchMetricsNamespace.stringValue; } const ami = new dataAwsAmi.DataAwsAmi(this, "LatestAmi", { @@ -425,81 +410,17 @@ ${metricsSection}`; const kafkaBootstrapServers = `['${Fn.join(`','`, Fn.split(",", mskCluster.bootstrapBrokersSaslScram))}']`; - const zillaYamlContent = ` -name: public -${zillaTelemetryContent} -vaults: - secure: - type: aws -bindings: - tcp_server: - type: tcp - kind: server -${bindingTelemetryContent} - options: - host: 0.0.0.0 - port: ${publicTcpPort} - exit: tls_server - tls_server: - type: tls - kind: server - vault: secure -${bindingTelemetryContent} - options: - keys: - - ${publicTlsCertificateKey.stringValue} - exit: mqtt_server - mqtt_server: - type: mqtt - kind: server -${bindingTelemetryContent} - exit: mqtt_kafka_mapping - mqtt_kafka_mapping: - type: mqtt-kafka - kind: proxy -${bindingTelemetryContent} - options: - topics: - sessions: ${kafkaTopicMqttSessions} - messages: ${kafkaTopicMqttMessages} - retained: ${kafkaTopicMqttRetained} - exit: kafka_cache_client - kafka_cache_client: - type: kafka - kind: cache_client -${bindingTelemetryContent} - exit: kafka_cache_server - kafka_cache_server: - type: kafka - kind: cache_server -${bindingTelemetryContent} - options: - bootstrap: - - ${kafkaTopicMqttMessages} - - ${kafkaTopicMqttRetained} - exit: kafka_client - kafka_client: - type: kafka - kind: client - options: - servers: ${kafkaBootstrapServers} - sasl: - mechanism: scram-sha-512 - username: '${kafkaSaslUsername}' - password: '${kafkaSaslPassword}' -${bindingTelemetryContent} - exit: tls_client - tls_client: - type: tls - kind: client - vault: secure -${bindingTelemetryContent} - exit: tcp_client - tcp_client: - type: tcp - kind: client -${bindingTelemetryContent} -`; + data.kafkaBootstrapServers = kafkaBootstrapServers; + data.kafkaSaslUsername = kafkaSaslUsername; + data.kafkaSaslPassword = kafkaSaslPassword; + data.publicPort = publicTcpPort.value; + data.publicTlsCertificateKey = publicTlsCertificateKey.stringValue; + data.kafkaTopicMqttSessions = kafkaTopicMqttSessions.stringValue; + data.kafkaTopicMqttMessages = kafkaTopicMqttMessages.stringValue; + data.kafkaTopicMqttRetained = kafkaTopicMqttRetained.stringValue; + + const yamlTemplate: string = fs.readFileSync('zilla.mustache', 'utf8'); + const renderedYaml: string = Mustache.render(yamlTemplate, data); const cfnHupConfContent = ` [main] @@ -538,7 +459,7 @@ END_HELP const userData = `#!/bin/bash -xe yum update -y aws-cfn-bootstrap cat <<'END_HELP' > /etc/zilla/zilla.yaml -${zillaYamlContent} +${renderedYaml} END_HELP chown ec2-user:ec2-user /etc/zilla/zilla.yaml diff --git a/amazon-msk/cdktf/iot-ingest-and-control/package-lock.json b/amazon-msk/cdktf/iot-ingest-and-control/package-lock.json index 51182bc..58f1652 100644 --- a/amazon-msk/cdktf/iot-ingest-and-control/package-lock.json +++ b/amazon-msk/cdktf/iot-ingest-and-control/package-lock.json @@ -12,11 +12,14 @@ "@cdktf/provider-aws": "^19.26.0", "@dotenvx/dotenvx": "^1.6.4", "aws-sdk": "^2.1668.0", + "cdktf": "^0.20.8", "constructs": "^10.3.0", - "kafkajs": "^2.2.4" + "kafkajs": "^2.2.4", + "mustache": "^4.2.0" }, "devDependencies": { "@types/jest": "^29.5.12", + "@types/mustache": "^4.2.5", "@types/node": "^20.14.11", "cdktf-cli": "^0.20.8", "jest": "^29.7.0", @@ -2733,6 +2736,12 @@ "pretty-format": "^29.0.0" } }, + "node_modules/@types/mustache": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/@types/mustache/-/mustache-4.2.5.tgz", + "integrity": "sha512-PLwiVvTBg59tGFL/8VpcGvqOu3L4OuveNvPi0EYbWchRdEVP++yRUXJPFl+CApKEq13017/4Nf7aQ5lTtHUNsA==", + "dev": true + }, "node_modules/@types/mute-stream": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.1.tgz", @@ -7899,6 +7908,14 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/mute-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", diff --git a/amazon-msk/cdktf/iot-ingest-and-control/package.json b/amazon-msk/cdktf/iot-ingest-and-control/package.json index 496e601..bc61159 100644 --- a/amazon-msk/cdktf/iot-ingest-and-control/package.json +++ b/amazon-msk/cdktf/iot-ingest-and-control/package.json @@ -26,10 +26,12 @@ "aws-sdk": "^2.1668.0", "cdktf": "^0.20.8", "constructs": "^10.3.0", - "kafkajs": "^2.2.4" + "kafkajs": "^2.2.4", + "mustache": "^4.2.0" }, "devDependencies": { "@types/jest": "^29.5.12", + "@types/mustache": "^4.2.5", "@types/node": "^20.14.11", "cdktf-cli": "^0.20.8", "jest": "^29.7.0", diff --git a/amazon-msk/cdktf/iot-ingest-and-control/terraform.tfvars.example b/amazon-msk/cdktf/iot-ingest-and-control/terraform.tfvars.example index 54c8d13..3f5a951 100644 --- a/amazon-msk/cdktf/iot-ingest-and-control/terraform.tfvars.example +++ b/amazon-msk/cdktf/iot-ingest-and-control/terraform.tfvars.example @@ -1,6 +1,6 @@ -msk_cluster_name="" -msk_access_credentials_name="" -public_tls_certificate_key="" +msk_cluster_name="my-msk-cluster" +msk_access_credentials_name="AmazonMSK_alice" +public_tls_certificate_key="arn:aws:secretsmanager:us-east-1:445711703002:secret:wildcard.example.aklivity.io-8-kq3J3T" ## optional #kafka_topic_mqtt_sessions="" diff --git a/amazon-msk/cdktf/iot-ingest-and-control/zilla.mustache b/amazon-msk/cdktf/iot-ingest-and-control/zilla.mustache new file mode 100644 index 0000000..4b79917 --- /dev/null +++ b/amazon-msk/cdktf/iot-ingest-and-control/zilla.mustache @@ -0,0 +1,133 @@ +name: {{name}} +vaults: + secure: + type: aws-secrets +{{^cloudwatchDisabled}} +telemetry: + metrics: + - stream.active.received + - stream.active.sent + - stream.opens.received + - stream.opens.sent + - stream.data.received + - stream.data.sent + - stream.errors.received + - stream.errors.sent + - stream.closes.received + - stream.closes.sent + exporters: + stdout_logs_exporter: + type: stdout + aws0: + type: aws-cloudwatch + options: + logs: + group: {{cloudWatchLogsGroup}} + stream: events + metrics: + namespace: {{cloudWatchMetricsNamespace}} +{{/cloudwatchDisabled}} +bindings: + tcp_server: + type: tcp + kind: server + options: + host: 0.0.0.0 + port: {{publicPort}} +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: tls_server + tls_server: + type: tls + kind: server + vault: secure + options: + keys: + - {{publicTlsCertificateKey}} +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: mqtt_server + mqtt_server: + type: mqtt + kind: server +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: mqtt_kafka_mapping + mqtt_kafka_mapping: + type: mqtt-kafka + kind: proxy + options: + topics: + sessions: {{kafkaTopicMqttSessions}} + messages: {{kafkaTopicMqttMessages}} + retained: {{kafkaTopicMqttRetained}} +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: kafka_cache_client + kafka_cache_client: + type: kafka + kind: cache_client +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: kafka_cache_server + kafka_cache_server: + type: kafka + kind: cache_server + options: + bootstrap: + - {{kafkaTopicMqttMessages}} + - {{kafkaTopicMqttRetained}} +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: kafka_client + kafka_client: + type: kafka + kind: client + options: + servers: {{{kafkaBootstrapServers}}} + sasl: + mechanism: scram-sha-512 + username: '{{kafkaSaslUsername}}' + password: '{{kafkaSaslPassword}}' +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: tls_client + tls_client: + type: tls + kind: client + vault: secure +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: tcp_client + tcp_client: + type: tcp + kind: client +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} diff --git a/amazon-msk/cdktf/secure-public-access/main.ts b/amazon-msk/cdktf/secure-public-access/main.ts index 3f33988..18391de 100644 --- a/amazon-msk/cdktf/secure-public-access/main.ts +++ b/amazon-msk/cdktf/secure-public-access/main.ts @@ -519,10 +519,6 @@ systemctl start nitro-enclaves-acm.service const yamlTemplate: string = fs.readFileSync('zilla.mustache', 'utf8'); const renderedYaml: string = Mustache.render(yamlTemplate, data); - new TerraformOutput(this, "renderedYaml", { - value: renderedYaml, - }); - const cfnHupConfContent = ` [main] stack=${id} diff --git a/amazon-msk/cdktf/web-streaming/main.ts b/amazon-msk/cdktf/web-streaming/main.ts index 40fd2a0..5ad6e87 100644 --- a/amazon-msk/cdktf/web-streaming/main.ts +++ b/amazon-msk/cdktf/web-streaming/main.ts @@ -25,8 +25,25 @@ import { IamInstanceProfile } from "@cdktf/provider-aws/lib/iam-instance-profile import { IamRole } from "@cdktf/provider-aws/lib/iam-role"; import { IamRolePolicy } from "@cdktf/provider-aws/lib/iam-role-policy"; import { SecurityGroup } from "@cdktf/provider-aws/lib/security-group"; - import { UserVariables } from "./variables"; +import Mustache = require("mustache"); +import fs = require("fs"); + +interface TemplateData { + name: string; + glueEnabled: boolean; + glueRegistry?: string; + cloudwatchDisabled?: boolean; + cloudWatchLogsGroup?: string; + cloudWatchMetricsNamespace?: string; + publicPort?: number; + publicTlsCertificateKey?: string; + path?: string; + topic?: string; + kafkaBootstrapServers?: string; + kafkaSaslUsername?: string; + kafkaSaslPassword?: string; +} export class ZillaPlusWebStreamingStack extends TerraformStack { constructor(scope: Construct, id: string) { @@ -310,8 +327,11 @@ export class ZillaPlusWebStreamingStack extends TerraformStack { errorMessage: "must be a valid EC2 instance type.", }); - let zillaTelemetryContent = ""; - let bindingTelemetryContent = ""; + const data: TemplateData = { + name: 'web', + cloudwatchDisabled: userVars.cloudwatchDisabled, + glueEnabled: userVars.glueRegistryEnabled + } if (!userVars.cloudwatchDisabled) { const defaultLogGroupName = `${id}-group`; @@ -333,77 +353,17 @@ export class ZillaPlusWebStreamingStack extends TerraformStack { name: cloudWatchLogsGroup.stringValue, }); - const logsSection = ` - logs: - group: ${cloudWatchLogsGroup.stringValue} - stream: events`; - - const metricsSection = ` - metrics: - namespace: ${cloudWatchMetricsNamespace.stringValue}`; - - zillaTelemetryContent = `telemetry: - metrics: - - stream.active.received - - stream.active.sent - - stream.opens.received - - stream.opens.sent - - stream.data.received - - stream.data.sent - - stream.errors.received - - stream.errors.sent - - stream.closes.received - - stream.closes.sent - exporters: - stdout_logs_exporter: - type: stdout - aws0: - type: aws-cloudwatch - options: -${logsSection} -${metricsSection}`; - - bindingTelemetryContent = ` telemetry: - metrics: - - stream.*`; + data.cloudWatchLogsGroup = cloudWatchLogsGroup.stringValue; + data.cloudWatchMetricsNamespace = cloudWatchMetricsNamespace.stringValue; } - let glueContent = ""; - let kafkaCacheClientGlueContent = ""; - let kafkaCacheServerGlueContent = ""; - if (userVars.glueRegistryEnabled) { const glueRegistry = new TerraformVariable(this, "glue_registry", { type: "string", description: "The Glue Registry to fetch the schemas from", }); - glueContent = `catalogs: - glue_catalog: - type: aws-glue - options: - registry: ${glueRegistry.stringValue}`; - - kafkaCacheClientGlueContent = ` options: - topics: - - name: ${topic} - value: - model: avro - view: json - catalog: - glue_catalog: - - strategy: topic - version: latest`; - - kafkaCacheServerGlueContent = ` topics: - - name: ${topic} - value: - model: avro - view: json - catalog: - glue_catalog: - - strategy: topic - version: latest`; + data.glueRegistry = glueRegistry.stringValue; } const ami = new dataAwsAmi.DataAwsAmi(this, "LatestAmi", { @@ -455,145 +415,16 @@ ${metricsSection}`; const kafkaBootstrapServers = `['${Fn.join(`','`, Fn.split(",", mskCluster.bootstrapBrokersSaslScram))}']`; - const zillaYamlContent = ` -name: public -${glueContent} -${zillaTelemetryContent} -vaults: - secure: - type: aws -bindings: - tcp_server: - type: tcp - kind: server -${bindingTelemetryContent} - options: - host: 0.0.0.0 - port: ${publicTcpPort} - exit: tls_server - tls_server: - type: tls - kind: server - vault: secure -${bindingTelemetryContent} - options: - keys: - - ${publicTlsCertificateKey.stringValue} - exit: north_http_server - north_http_server: - type: http - kind: server -${bindingTelemetryContent} - routes: - - when: - - headers: - :path: /streams${path} - exit: north_sse_server - - when: - - headers: - :path: ${path}/* - - headers: - :path: ${path} - exit: north_http_kafka_mapping - north_sse_server: - type: sse - kind: server -${bindingTelemetryContent} - exit: north_sse_kafka_mapping - north_sse_kafka_mapping: - type: sse-kafka - kind: proxy -${bindingTelemetryContent} - routes: - - when: - - path: /streams${path} - exit: kafka_cache_client - with: - topic: ${topic} - north_http_kafka_mapping: - type: http-kafka - kind: proxy -${bindingTelemetryContent} - routes: - - when: - - method: POST - path: ${path} - exit: kafka_cache_client - with: - capability: produce - topic: ${topic.stringValue} - key: \${idempotencyKey} - - when: - - method: PUT - path: ${path}/{id} - exit: kafka_cache_client - with: - capability: produce - topic: ${topic.stringValue} - key: \${params.id} - - when: - - method: DELETE - path: ${path}/{id} - exit: kafka_cache_client - with: - capability: produce - topic: ${topic.stringValue} - key: \${params.id} - - when: - - method: GET - path: ${path} - exit: kafka_cache_client - with: - capability: fetch - topic: ${topic.stringValue} - merge: - content-type: application/json - - when: - - method: GET - path: ${path}/{id} - exit: kafka_cache_client - with: - capability: fetch - topic: ${topic.stringValue} - filters: - - key: \${params.id} - kafka_cache_client: - type: kafka - kind: cache_client -${kafkaCacheClientGlueContent} -${bindingTelemetryContent} - exit: kafka_cache_server - kafka_cache_server: - type: kafka - kind: cache_server -${bindingTelemetryContent} - options: - bootstrap: - - ${topic.stringValue} -${kafkaCacheServerGlueContent} - exit: kafka_client - kafka_client: - type: kafka - kind: client - options: - servers: ${kafkaBootstrapServers} - sasl: - mechanism: scram-sha-512 - username: '${kafkaSaslUsername}' - password: '${kafkaSaslPassword}' -${bindingTelemetryContent} - exit: tls_client - tls_client: - type: tls - kind: client - vault: secure -${bindingTelemetryContent} - exit: tcp_client - tcp_client: - type: tcp - kind: client -${bindingTelemetryContent} -`; + data.kafkaBootstrapServers = kafkaBootstrapServers; + data.kafkaSaslUsername = kafkaSaslUsername; + data.kafkaSaslPassword = kafkaSaslPassword; + data.publicPort = publicTcpPort.value; + data.publicTlsCertificateKey = publicTlsCertificateKey.stringValue; + data.path = path; + data.topic = topic.stringValue; + + const yamlTemplate: string = fs.readFileSync('zilla.mustache', 'utf8'); + const renderedYaml: string = Mustache.render(yamlTemplate, data); const cfnHupConfContent = ` [main] @@ -634,7 +465,7 @@ EOF const userData = `#!/bin/bash -xe yum update -y aws-cfn-bootstrap cat <<'END_HELP' > /etc/zilla/zilla.yaml -${zillaYamlContent} +${renderedYaml} END_HELP chown ec2-user:ec2-user /etc/zilla/zilla.yaml diff --git a/amazon-msk/cdktf/web-streaming/package-lock.json b/amazon-msk/cdktf/web-streaming/package-lock.json index 77950df..ca97a28 100644 --- a/amazon-msk/cdktf/web-streaming/package-lock.json +++ b/amazon-msk/cdktf/web-streaming/package-lock.json @@ -12,10 +12,12 @@ "@cdktf/provider-aws": "^19.27.0", "@dotenvx/dotenvx": "^1.6.4", "cdktf": "^0.20.8", - "constructs": "^10.3.0" + "constructs": "^10.3.0", + "mustache": "^4.2.0" }, "devDependencies": { "@types/jest": "^29.5.12", + "@types/mustache": "^4.2.5", "@types/node": "^20.14.11", "cdktf-cli": "^0.20.8", "jest": "^29.7.0", @@ -2722,6 +2724,12 @@ "pretty-format": "^29.0.0" } }, + "node_modules/@types/mustache": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/@types/mustache/-/mustache-4.2.5.tgz", + "integrity": "sha512-PLwiVvTBg59tGFL/8VpcGvqOu3L4OuveNvPi0EYbWchRdEVP++yRUXJPFl+CApKEq13017/4Nf7aQ5lTtHUNsA==", + "dev": true + }, "node_modules/@types/mute-stream": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.1.tgz", @@ -7835,6 +7843,14 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/mute-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", diff --git a/amazon-msk/cdktf/web-streaming/package.json b/amazon-msk/cdktf/web-streaming/package.json index 4a671c3..b25f509 100644 --- a/amazon-msk/cdktf/web-streaming/package.json +++ b/amazon-msk/cdktf/web-streaming/package.json @@ -24,10 +24,12 @@ "@cdktf/provider-aws": "^19.27.0", "@dotenvx/dotenvx": "^1.6.4", "cdktf": "^0.20.8", - "constructs": "^10.3.0" + "constructs": "^10.3.0", + "mustache": "^4.2.0" }, "devDependencies": { "@types/jest": "^29.5.12", + "@types/mustache": "^4.2.5", "@types/node": "^20.14.11", "cdktf-cli": "^0.20.8", "jest": "^29.7.0", diff --git a/amazon-msk/cdktf/web-streaming/variables.ts b/amazon-msk/cdktf/web-streaming/variables.ts index 66ac1c0..0618edf 100644 --- a/amazon-msk/cdktf/web-streaming/variables.ts +++ b/amazon-msk/cdktf/web-streaming/variables.ts @@ -8,7 +8,7 @@ export class UserVariables extends Construct { createZillaPlusSecurityGroup: boolean = false; sshKeyEnabled: boolean = false; cloudwatchDisabled: boolean = false; - glueRegistryEnabled: boolean = false; + glueRegistryEnabled: boolean = true; kafkaTopicCreationDisabled: boolean = false; constructor(scope: Construct, name: string) { diff --git a/amazon-msk/cdktf/web-streaming/zilla.mustache b/amazon-msk/cdktf/web-streaming/zilla.mustache new file mode 100644 index 0000000..983bdbe --- /dev/null +++ b/amazon-msk/cdktf/web-streaming/zilla.mustache @@ -0,0 +1,232 @@ +name: {{name}} +vaults: + secure: + type: aws-secrets +{{#glueEnabled}} +catalogs: + glue_catalog: + type: aws-glue + options: + registry: {{glueRegistry}} +{{/glueEnabled}} +{{^cloudwatchDisabled}} +telemetry: + metrics: + - stream.active.received + - stream.active.sent + - stream.opens.received + - stream.opens.sent + - stream.data.received + - stream.data.sent + - stream.errors.received + - stream.errors.sent + - stream.closes.received + - stream.closes.sent + exporters: + stdout_logs_exporter: + type: stdout + aws0: + type: aws-cloudwatch + options: + logs: + group: {{cloudWatchLogsGroup}} + stream: events + metrics: + namespace: {{cloudWatchMetricsNamespace}} +{{/cloudwatchDisabled}} +bindings: + tcp_server: + type: tcp + kind: server + options: + host: 0.0.0.0 + port: {{publicPort}} +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: tls_server + tls_server: + type: tls + kind: server + vault: secure + options: + keys: + - {{publicTlsCertificateKey}} +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: http_server + http_server: + type: http + kind: server +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + routes: + - when: + - headers: + :path: /streams{{{path}}} + exit: sse_server + - when: + - headers: + :path: {{{path}}}/* + - headers: + :path: {{{path}}} + exit: http_kafka_mapping + sse_server: + type: sse + kind: server +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: sse_kafka_mapping + sse_kafka_mapping: + type: sse-kafka + kind: proxy +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + routes: + - when: + - path: /streams{{{path}}} + exit: kafka_cache_client + with: + topic: {{topic}} + http_kafka_mapping: + type: http-kafka + kind: proxy +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + routes: + - when: + - method: POST + path: {{{path}}} + exit: kafka_cache_client + with: + capability: produce + topic: {{topic}} + key: ${idempotencyKey} + - when: + - method: PUT + path: {{{path}}}/{id} + exit: kafka_cache_client + with: + capability: produce + topic: {{topic}} + key: ${params.id} + - when: + - method: DELETE + path: {{{path}}}/{id} + exit: kafka_cache_client + with: + capability: produce + topic: {{topic}} + key: ${params.id} + - when: + - method: GET + path: {{{path}}} + exit: kafka_cache_client + with: + capability: fetch + topic: {{topic}} + merge: + content-type: application/json + - when: + - method: GET + path: {{{path}}}/{id} + exit: kafka_cache_client + with: + capability: fetch + topic: {{topic}} + filters: + - key: ${params.id} + kafka_cache_client: + type: kafka + kind: cache_client +{{#glueEnabled}} + options: + topics: + - name: {{topic}} + value: + model: avro + view: json + catalog: + glue_catalog: + - strategy: topic + version: latest +{{/glueEnabled}} +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: kafka_cache_server + kafka_cache_server: + type: kafka + kind: cache_server + options: + bootstrap: + - {{topic}} +{{#glueEnabled}} + topics: + - name: {{topic}} + value: + model: avro + view: json + catalog: + glue_catalog: + - strategy: topic + version: latest +{{/glueEnabled}} +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: kafka_client + kafka_client: + type: kafka + kind: client + options: + servers: {{{kafkaBootstrapServers}}} + sasl: + mechanism: scram-sha-512 + username: '{{kafkaSaslUsername}}' + password: '{{kafkaSaslPassword}}' +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: tls_client + tls_client: + type: tls + kind: client + vault: secure +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} + exit: tcp_client + tcp_client: + type: tcp + kind: client +{{^cloudwatchDisabled}} + telemetry: + metrics: + - stream.* +{{/cloudwatchDisabled}} From 96ad4dad4346da1599b1197869c11f451c8d2955 Mon Sep 17 00:00:00 2001 From: bmaidics Date: Thu, 10 Oct 2024 14:49:56 +0200 Subject: [PATCH 3/5] Remove values from example --- .../cdktf/iot-ingest-and-control/terraform.tfvars.example | 6 +++--- amazon-msk/cdktf/web-streaming/variables.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/amazon-msk/cdktf/iot-ingest-and-control/terraform.tfvars.example b/amazon-msk/cdktf/iot-ingest-and-control/terraform.tfvars.example index 3f5a951..54c8d13 100644 --- a/amazon-msk/cdktf/iot-ingest-and-control/terraform.tfvars.example +++ b/amazon-msk/cdktf/iot-ingest-and-control/terraform.tfvars.example @@ -1,6 +1,6 @@ -msk_cluster_name="my-msk-cluster" -msk_access_credentials_name="AmazonMSK_alice" -public_tls_certificate_key="arn:aws:secretsmanager:us-east-1:445711703002:secret:wildcard.example.aklivity.io-8-kq3J3T" +msk_cluster_name="" +msk_access_credentials_name="" +public_tls_certificate_key="" ## optional #kafka_topic_mqtt_sessions="" diff --git a/amazon-msk/cdktf/web-streaming/variables.ts b/amazon-msk/cdktf/web-streaming/variables.ts index 0618edf..66ac1c0 100644 --- a/amazon-msk/cdktf/web-streaming/variables.ts +++ b/amazon-msk/cdktf/web-streaming/variables.ts @@ -8,7 +8,7 @@ export class UserVariables extends Construct { createZillaPlusSecurityGroup: boolean = false; sshKeyEnabled: boolean = false; cloudwatchDisabled: boolean = false; - glueRegistryEnabled: boolean = true; + glueRegistryEnabled: boolean = false; kafkaTopicCreationDisabled: boolean = false; constructor(scope: Construct, name: string) { From 0fcc2f0eab44d4a1cd5f157cd636f756017c5c76 Mon Sep 17 00:00:00 2001 From: bmaidics Date: Fri, 18 Oct 2024 12:03:35 +0200 Subject: [PATCH 4/5] Review items --- .../cdktf/iot-ingest-and-control/main.ts | 17 ++++++----- .../{zilla.mustache => zilla.yaml.mustache} | 26 +++------------- amazon-msk/cdktf/secure-public-access/main.ts | 18 ++++++----- .../{zilla.mustache => zilla.yaml.mustache} | 18 +++-------- amazon-msk/cdktf/web-streaming/main.ts | 17 ++++++----- .../{zilla.mustache => zilla.yaml.mustache} | 30 +++---------------- 6 files changed, 43 insertions(+), 83 deletions(-) rename amazon-msk/cdktf/iot-ingest-and-control/{zilla.mustache => zilla.yaml.mustache} (79%) rename amazon-msk/cdktf/secure-public-access/{zilla.mustache => zilla.yaml.mustache} (83%) rename amazon-msk/cdktf/web-streaming/{zilla.mustache => zilla.yaml.mustache} (86%) diff --git a/amazon-msk/cdktf/iot-ingest-and-control/main.ts b/amazon-msk/cdktf/iot-ingest-and-control/main.ts index c71234d..8a31036 100644 --- a/amazon-msk/cdktf/iot-ingest-and-control/main.ts +++ b/amazon-msk/cdktf/iot-ingest-and-control/main.ts @@ -31,9 +31,7 @@ import fs = require("fs"); interface TemplateData { name: string; - cloudwatchDisabled?: boolean; - cloudWatchLogsGroup?: string; - cloudWatchMetricsNamespace?: string; + cloudwatch?: object; publicPort?: number; publicTlsCertificateKey?: string; kafkaTopicMqttSessions?: string; @@ -334,7 +332,6 @@ export class ZillaPlusIotAndControlStack extends TerraformStack { const data: TemplateData = { name: 'iot', - cloudwatchDisabled: userVariables.cloudwatchDisabled } if (!userVariables.cloudwatchDisabled) { @@ -357,8 +354,14 @@ export class ZillaPlusIotAndControlStack extends TerraformStack { name: cloudWatchLogsGroup.stringValue, }); - data.cloudWatchLogsGroup = cloudWatchLogsGroup.stringValue; - data.cloudWatchMetricsNamespace = cloudWatchMetricsNamespace.stringValue; + data.cloudwatch = { + logs: { + group: cloudWatchLogsGroup.stringValue + }, + metrics: { + namespace: cloudWatchMetricsNamespace.stringValue + } + }; } const ami = new dataAwsAmi.DataAwsAmi(this, "LatestAmi", { @@ -419,7 +422,7 @@ export class ZillaPlusIotAndControlStack extends TerraformStack { data.kafkaTopicMqttMessages = kafkaTopicMqttMessages.stringValue; data.kafkaTopicMqttRetained = kafkaTopicMqttRetained.stringValue; - const yamlTemplate: string = fs.readFileSync('zilla.mustache', 'utf8'); + const yamlTemplate: string = fs.readFileSync('zilla.yaml.mustache', 'utf8'); const renderedYaml: string = Mustache.render(yamlTemplate, data); const cfnHupConfContent = ` diff --git a/amazon-msk/cdktf/iot-ingest-and-control/zilla.mustache b/amazon-msk/cdktf/iot-ingest-and-control/zilla.yaml.mustache similarity index 79% rename from amazon-msk/cdktf/iot-ingest-and-control/zilla.mustache rename to amazon-msk/cdktf/iot-ingest-and-control/zilla.yaml.mustache index 4b79917..b7e74da 100644 --- a/amazon-msk/cdktf/iot-ingest-and-control/zilla.mustache +++ b/amazon-msk/cdktf/iot-ingest-and-control/zilla.yaml.mustache @@ -2,7 +2,6 @@ name: {{name}} vaults: secure: type: aws-secrets -{{^cloudwatchDisabled}} telemetry: metrics: - stream.active.received @@ -18,15 +17,16 @@ telemetry: exporters: stdout_logs_exporter: type: stdout +{{#cloudwatch}} aws0: type: aws-cloudwatch options: logs: - group: {{cloudWatchLogsGroup}} + group: {{cloudwatch.logs.group}} stream: events metrics: - namespace: {{cloudWatchMetricsNamespace}} -{{/cloudwatchDisabled}} + namespace: {{cloudwatch.metrics.namespace}} +{{/cloudwatch}} bindings: tcp_server: type: tcp @@ -34,11 +34,9 @@ bindings: options: host: 0.0.0.0 port: {{publicPort}} -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: tls_server tls_server: type: tls @@ -47,20 +45,16 @@ bindings: options: keys: - {{publicTlsCertificateKey}} -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: mqtt_server mqtt_server: type: mqtt kind: server -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: mqtt_kafka_mapping mqtt_kafka_mapping: type: mqtt-kafka @@ -70,20 +64,16 @@ bindings: sessions: {{kafkaTopicMqttSessions}} messages: {{kafkaTopicMqttMessages}} retained: {{kafkaTopicMqttRetained}} -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: kafka_cache_client kafka_cache_client: type: kafka kind: cache_client -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: kafka_cache_server kafka_cache_server: type: kafka @@ -92,11 +82,9 @@ bindings: bootstrap: - {{kafkaTopicMqttMessages}} - {{kafkaTopicMqttRetained}} -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: kafka_client kafka_client: type: kafka @@ -107,27 +95,21 @@ bindings: mechanism: scram-sha-512 username: '{{kafkaSaslUsername}}' password: '{{kafkaSaslPassword}}' -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: tls_client tls_client: type: tls kind: client vault: secure -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: tcp_client tcp_client: type: tcp kind: client -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} diff --git a/amazon-msk/cdktf/secure-public-access/main.ts b/amazon-msk/cdktf/secure-public-access/main.ts index 18391de..2588c61 100644 --- a/amazon-msk/cdktf/secure-public-access/main.ts +++ b/amazon-msk/cdktf/secure-public-access/main.ts @@ -36,9 +36,7 @@ import fs = require("fs"); interface TemplateData { name: string; useAcm: boolean; - cloudwatchDisabled?: boolean; - cloudWatchLogsGroup?: string; - cloudWatchMetricsNamespace?: string; + cloudwatch?: object; publicPort?: number; publicTlsCertificateKey?: string; mTLS?: boolean; @@ -173,7 +171,6 @@ export class ZillaPlusSecurePublicAccessStack extends TerraformStack { const data: TemplateData = { name: 'public', useAcm: publicTlsCertificateViaAcm, - cloudwatchDisabled: userVariables.cloudwatchDisabled, mTLS: mTLSEnabled }; @@ -445,8 +442,15 @@ systemctl start nitro-enclaves-acm.service name: cloudWatchLogsGroup.stringValue, }); - data.cloudWatchLogsGroup = cloudWatchLogsGroup.stringValue; - data.cloudWatchMetricsNamespace = cloudWatchMetricsNamespace.stringValue; + + data.cloudwatch = { + logs: { + group: cloudWatchLogsGroup.stringValue + }, + metrics: { + namespace: cloudWatchMetricsNamespace.stringValue + } + }; } const instanceType = new TerraformVariable(this, "zilla_plus_instance_type", { @@ -516,7 +520,7 @@ systemctl start nitro-enclaves-acm.service data.internalHost = internalHost; data.mskPort = mskPort; data.mskWildcardDNS = mskWildcardDNS; - const yamlTemplate: string = fs.readFileSync('zilla.mustache', 'utf8'); + const yamlTemplate: string = fs.readFileSync('zilla.yaml.mustache', 'utf8'); const renderedYaml: string = Mustache.render(yamlTemplate, data); const cfnHupConfContent = ` diff --git a/amazon-msk/cdktf/secure-public-access/zilla.mustache b/amazon-msk/cdktf/secure-public-access/zilla.yaml.mustache similarity index 83% rename from amazon-msk/cdktf/secure-public-access/zilla.mustache rename to amazon-msk/cdktf/secure-public-access/zilla.yaml.mustache index 15f8f82..4af2c96 100644 --- a/amazon-msk/cdktf/secure-public-access/zilla.mustache +++ b/amazon-msk/cdktf/secure-public-access/zilla.yaml.mustache @@ -2,7 +2,6 @@ name: {{name}} vaults: secure: type: {{#useAcm}}aws-acm{{/useAcm}}{{^useAcm}}aws-secrets{{/useAcm}} -{{^cloudwatchDisabled}} telemetry: metrics: - stream.active.received @@ -18,15 +17,16 @@ telemetry: exporters: stdout_logs_exporter: type: stdout +{{#cloudwatch}} aws0: type: aws-cloudwatch options: logs: - group: {{cloudWatchLogsGroup}} + group: {{cloudwatch.logs.group}} stream: events metrics: - namespace: {{cloudWatchMetricsNamespace}} -{{/cloudwatchDisabled}} + namespace: {{cloudwatch.metrics.namespace}} +{{/cloudwatch}} bindings: tcp_server: type: tcp @@ -34,11 +34,9 @@ bindings: options: host: 0.0.0.0 port: {{publicPort}} -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: tls_server tls_server: type: tls @@ -51,11 +49,9 @@ bindings: trust: - {{publicCertificateAuthority}} {{/mTLS}} -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} routes: - exit: kafka_proxy when: @@ -70,11 +66,9 @@ bindings: internal: host: '{{internalHost}}' port: {{mskPort}} -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: tls_client tls_client: type: tls @@ -86,11 +80,9 @@ bindings: - {{mskCertificateAuthority}} {{/mTLS}} trustcacerts: true -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: tcp_client tcp_client: type: tcp @@ -98,11 +90,9 @@ bindings: options: host: '*' port: {{mskPort}} -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} routes: - when: - authority: '{{mskWildcardDNS}}' diff --git a/amazon-msk/cdktf/web-streaming/main.ts b/amazon-msk/cdktf/web-streaming/main.ts index 5ad6e87..4cb89c5 100644 --- a/amazon-msk/cdktf/web-streaming/main.ts +++ b/amazon-msk/cdktf/web-streaming/main.ts @@ -33,9 +33,7 @@ interface TemplateData { name: string; glueEnabled: boolean; glueRegistry?: string; - cloudwatchDisabled?: boolean; - cloudWatchLogsGroup?: string; - cloudWatchMetricsNamespace?: string; + cloudwatch?: object; publicPort?: number; publicTlsCertificateKey?: string; path?: string; @@ -329,7 +327,6 @@ export class ZillaPlusWebStreamingStack extends TerraformStack { const data: TemplateData = { name: 'web', - cloudwatchDisabled: userVars.cloudwatchDisabled, glueEnabled: userVars.glueRegistryEnabled } @@ -353,8 +350,14 @@ export class ZillaPlusWebStreamingStack extends TerraformStack { name: cloudWatchLogsGroup.stringValue, }); - data.cloudWatchLogsGroup = cloudWatchLogsGroup.stringValue; - data.cloudWatchMetricsNamespace = cloudWatchMetricsNamespace.stringValue; + data.cloudwatch = { + logs: { + group: cloudWatchLogsGroup.stringValue + }, + metrics: { + namespace: cloudWatchMetricsNamespace.stringValue + } + }; } if (userVars.glueRegistryEnabled) { @@ -423,7 +426,7 @@ export class ZillaPlusWebStreamingStack extends TerraformStack { data.path = path; data.topic = topic.stringValue; - const yamlTemplate: string = fs.readFileSync('zilla.mustache', 'utf8'); + const yamlTemplate: string = fs.readFileSync('zilla.yaml.mustache', 'utf8'); const renderedYaml: string = Mustache.render(yamlTemplate, data); const cfnHupConfContent = ` diff --git a/amazon-msk/cdktf/web-streaming/zilla.mustache b/amazon-msk/cdktf/web-streaming/zilla.yaml.mustache similarity index 86% rename from amazon-msk/cdktf/web-streaming/zilla.mustache rename to amazon-msk/cdktf/web-streaming/zilla.yaml.mustache index 983bdbe..9794661 100644 --- a/amazon-msk/cdktf/web-streaming/zilla.mustache +++ b/amazon-msk/cdktf/web-streaming/zilla.yaml.mustache @@ -9,7 +9,6 @@ catalogs: options: registry: {{glueRegistry}} {{/glueEnabled}} -{{^cloudwatchDisabled}} telemetry: metrics: - stream.active.received @@ -25,15 +24,16 @@ telemetry: exporters: stdout_logs_exporter: type: stdout +{{#cloudwatch}} aws0: type: aws-cloudwatch options: logs: - group: {{cloudWatchLogsGroup}} + group: {{cloudwatch.logs.group}} stream: events metrics: - namespace: {{cloudWatchMetricsNamespace}} -{{/cloudwatchDisabled}} + namespace: {{cloudwatch.metrics.namespace}} +{{/cloudwatch}} bindings: tcp_server: type: tcp @@ -41,11 +41,9 @@ bindings: options: host: 0.0.0.0 port: {{publicPort}} -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: tls_server tls_server: type: tls @@ -54,20 +52,16 @@ bindings: options: keys: - {{publicTlsCertificateKey}} -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: http_server http_server: type: http kind: server -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} routes: - when: - headers: @@ -82,20 +76,16 @@ bindings: sse_server: type: sse kind: server -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: sse_kafka_mapping sse_kafka_mapping: type: sse-kafka kind: proxy -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} routes: - when: - path: /streams{{{path}}} @@ -105,11 +95,9 @@ bindings: http_kafka_mapping: type: http-kafka kind: proxy -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} routes: - when: - method: POST @@ -168,11 +156,9 @@ bindings: - strategy: topic version: latest {{/glueEnabled}} -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: kafka_cache_server kafka_cache_server: type: kafka @@ -191,11 +177,9 @@ bindings: - strategy: topic version: latest {{/glueEnabled}} -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: kafka_client kafka_client: type: kafka @@ -206,27 +190,21 @@ bindings: mechanism: scram-sha-512 username: '{{kafkaSaslUsername}}' password: '{{kafkaSaslPassword}}' -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: tls_client tls_client: type: tls kind: client vault: secure -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} exit: tcp_client tcp_client: type: tcp kind: client -{{^cloudwatchDisabled}} telemetry: metrics: - stream.* -{{/cloudwatchDisabled}} From 4f13fcf21e94e2a4addfc2e9856c4532b53de8da Mon Sep 17 00:00:00 2001 From: bmaidics Date: Tue, 22 Oct 2024 18:03:29 +0200 Subject: [PATCH 5/5] Review items --- .../cdktf/iot-ingest-and-control/main.ts | 36 ++++++++++--------- .../zilla.yaml.mustache | 20 +++++------ amazon-msk/cdktf/secure-public-access/main.ts | 31 ++++++++-------- .../secure-public-access/zilla.yaml.mustache | 18 +++++----- amazon-msk/cdktf/web-streaming/main.ts | 31 ++++++++-------- .../cdktf/web-streaming/zilla.yaml.mustache | 29 +++++++-------- 6 files changed, 88 insertions(+), 77 deletions(-) diff --git a/amazon-msk/cdktf/iot-ingest-and-control/main.ts b/amazon-msk/cdktf/iot-ingest-and-control/main.ts index 8a31036..a9f34fc 100644 --- a/amazon-msk/cdktf/iot-ingest-and-control/main.ts +++ b/amazon-msk/cdktf/iot-ingest-and-control/main.ts @@ -32,14 +32,10 @@ import fs = require("fs"); interface TemplateData { name: string; cloudwatch?: object; - publicPort?: number; - publicTlsCertificateKey?: string; - kafkaTopicMqttSessions?: string; - kafkaTopicMqttMessages?: string; - kafkaTopicMqttRetained?: string; - kafkaBootstrapServers?: string; - kafkaSaslUsername?: string; - kafkaSaslPassword?: string; + public?: object; + topics?: object; + kafka?: object; + } export class ZillaPlusIotAndControlStack extends TerraformStack { @@ -413,14 +409,22 @@ export class ZillaPlusIotAndControlStack extends TerraformStack { const kafkaBootstrapServers = `['${Fn.join(`','`, Fn.split(",", mskCluster.bootstrapBrokersSaslScram))}']`; - data.kafkaBootstrapServers = kafkaBootstrapServers; - data.kafkaSaslUsername = kafkaSaslUsername; - data.kafkaSaslPassword = kafkaSaslPassword; - data.publicPort = publicTcpPort.value; - data.publicTlsCertificateKey = publicTlsCertificateKey.stringValue; - data.kafkaTopicMqttSessions = kafkaTopicMqttSessions.stringValue; - data.kafkaTopicMqttMessages = kafkaTopicMqttMessages.stringValue; - data.kafkaTopicMqttRetained = kafkaTopicMqttRetained.stringValue; + data.kafka = { + bootstrapServers: kafkaBootstrapServers, + sasl : { + username: kafkaSaslUsername, + password: kafkaSaslPassword + } + } + data.public = { + port: publicTcpPort.value, + tlsCertificateKey: publicTlsCertificateKey.stringValue + } + data.topics = { + sessions: kafkaTopicMqttSessions.stringValue, + messages: kafkaTopicMqttMessages.stringValue, + retained: kafkaTopicMqttRetained.stringValue + }; const yamlTemplate: string = fs.readFileSync('zilla.yaml.mustache', 'utf8'); const renderedYaml: string = Mustache.render(yamlTemplate, data); diff --git a/amazon-msk/cdktf/iot-ingest-and-control/zilla.yaml.mustache b/amazon-msk/cdktf/iot-ingest-and-control/zilla.yaml.mustache index b7e74da..2399288 100644 --- a/amazon-msk/cdktf/iot-ingest-and-control/zilla.yaml.mustache +++ b/amazon-msk/cdktf/iot-ingest-and-control/zilla.yaml.mustache @@ -33,7 +33,7 @@ bindings: kind: server options: host: 0.0.0.0 - port: {{publicPort}} + port: {{public.port}} telemetry: metrics: - stream.* @@ -44,7 +44,7 @@ bindings: vault: secure options: keys: - - {{publicTlsCertificateKey}} + - {{public.tlsCertificateKey}} telemetry: metrics: - stream.* @@ -61,9 +61,9 @@ bindings: kind: proxy options: topics: - sessions: {{kafkaTopicMqttSessions}} - messages: {{kafkaTopicMqttMessages}} - retained: {{kafkaTopicMqttRetained}} + sessions: {{topics.sessions}} + messages: {{topics.messages}} + retained: {{topics.retained}} telemetry: metrics: - stream.* @@ -80,8 +80,8 @@ bindings: kind: cache_server options: bootstrap: - - {{kafkaTopicMqttMessages}} - - {{kafkaTopicMqttRetained}} + - {{topics.messages}} + - {{topics.retained}} telemetry: metrics: - stream.* @@ -90,11 +90,11 @@ bindings: type: kafka kind: client options: - servers: {{{kafkaBootstrapServers}}} + servers: {{{kafka.bootstrapServers}}} sasl: mechanism: scram-sha-512 - username: '{{kafkaSaslUsername}}' - password: '{{kafkaSaslPassword}}' + username: '{{kafka.sasl.username}}' + password: '{{kafka.sasl.password}}' telemetry: metrics: - stream.* diff --git a/amazon-msk/cdktf/secure-public-access/main.ts b/amazon-msk/cdktf/secure-public-access/main.ts index 2588c61..7fb34e9 100644 --- a/amazon-msk/cdktf/secure-public-access/main.ts +++ b/amazon-msk/cdktf/secure-public-access/main.ts @@ -37,16 +37,11 @@ interface TemplateData { name: string; useAcm: boolean; cloudwatch?: object; - publicPort?: number; - publicTlsCertificateKey?: string; + public?: object; mTLS?: boolean; - publicCertificateAuthority?: string; - publicWildcardDNS?: string; externalHost?: string; internalHost?: string; - mskPort?: number; - mskCertificateAuthority?: string; - mskWildcardDNS?: string; + msk?: object; } export class ZillaPlusSecurePublicAccessStack extends TerraformStack { @@ -171,7 +166,8 @@ export class ZillaPlusSecurePublicAccessStack extends TerraformStack { const data: TemplateData = { name: 'public', useAcm: publicTlsCertificateViaAcm, - mTLS: mTLSEnabled + mTLS: mTLSEnabled, + public: {} }; if (mTLSEnabled) { @@ -201,7 +197,9 @@ export class ZillaPlusSecurePublicAccessStack extends TerraformStack { }); publicCertificateAuthority = publicCertificateAuthorityVar.stringValue; } - data.publicCertificateAuthority = publicCertificateAuthority; + data.public = { + certificateAuthority: publicCertificateAuthority + } } const publicTlsCertificateKey = new TerraformVariable(this, "public_tls_certificate_key", { @@ -513,13 +511,18 @@ systemctl start nitro-enclaves-acm.service const internalHost = ["b-#.", Fn.element(Fn.split("*.", mskWildcardDNS), 1)].join(""); - data.publicPort = publicPort.value; - data.publicTlsCertificateKey = publicTlsCertificateKey.stringValue; - data.publicWildcardDNS = publicWildcardDNS.stringValue; + data.public = { + ...data.public, + port: publicPort.value, + tlsCertificateKey: publicTlsCertificateKey.stringValue, + wildcardDNS: publicWildcardDNS.stringValue + } data.externalHost = externalHost; data.internalHost = internalHost; - data.mskPort = mskPort; - data.mskWildcardDNS = mskWildcardDNS; + data.msk = { + port: mskPort, + wildcardDNS: mskWildcardDNS + } const yamlTemplate: string = fs.readFileSync('zilla.yaml.mustache', 'utf8'); const renderedYaml: string = Mustache.render(yamlTemplate, data); diff --git a/amazon-msk/cdktf/secure-public-access/zilla.yaml.mustache b/amazon-msk/cdktf/secure-public-access/zilla.yaml.mustache index 4af2c96..359f2f5 100644 --- a/amazon-msk/cdktf/secure-public-access/zilla.yaml.mustache +++ b/amazon-msk/cdktf/secure-public-access/zilla.yaml.mustache @@ -33,7 +33,7 @@ bindings: kind: server options: host: 0.0.0.0 - port: {{publicPort}} + port: {{public.port}} telemetry: metrics: - stream.* @@ -44,10 +44,10 @@ bindings: vault: secure options: keys: - - {{publicTlsCertificateKey}} + - {{public.tlsCertificateKey}} {{#mTLS}} trust: - - {{publicCertificateAuthority}} + - {{public.certificateAuthority}} {{/mTLS}} telemetry: metrics: @@ -55,17 +55,17 @@ bindings: routes: - exit: kafka_proxy when: - - authority: '{{publicWildcardDNS}}' + - authority: '{{public.wildcardDNS}}' kafka_proxy: type: kafka-proxy kind: proxy options: external: host: '{{externalHost}}' - port: {{publicPort}} + port: {{public.port}} internal: host: '{{internalHost}}' - port: {{mskPort}} + port: {{msk.port}} telemetry: metrics: - stream.* @@ -77,7 +77,7 @@ bindings: options: {{#mTLS}} signers: - - {{mskCertificateAuthority}} + - {{msk.certificateAuthority}} {{/mTLS}} trustcacerts: true telemetry: @@ -89,11 +89,11 @@ bindings: kind: client options: host: '*' - port: {{mskPort}} + port: {{msk.port}} telemetry: metrics: - stream.* routes: - when: - - authority: '{{mskWildcardDNS}}' + - authority: '{{msk.wildcardDNS}}' diff --git a/amazon-msk/cdktf/web-streaming/main.ts b/amazon-msk/cdktf/web-streaming/main.ts index 4cb89c5..1a02657 100644 --- a/amazon-msk/cdktf/web-streaming/main.ts +++ b/amazon-msk/cdktf/web-streaming/main.ts @@ -31,16 +31,12 @@ import fs = require("fs"); interface TemplateData { name: string; - glueEnabled: boolean; - glueRegistry?: string; + glue?: object; cloudwatch?: object; - publicPort?: number; - publicTlsCertificateKey?: string; path?: string; topic?: string; - kafkaBootstrapServers?: string; - kafkaSaslUsername?: string; - kafkaSaslPassword?: string; + public?: object; + kafka?: object; } export class ZillaPlusWebStreamingStack extends TerraformStack { @@ -327,7 +323,6 @@ export class ZillaPlusWebStreamingStack extends TerraformStack { const data: TemplateData = { name: 'web', - glueEnabled: userVars.glueRegistryEnabled } if (!userVars.cloudwatchDisabled) { @@ -366,7 +361,9 @@ export class ZillaPlusWebStreamingStack extends TerraformStack { description: "The Glue Registry to fetch the schemas from", }); - data.glueRegistry = glueRegistry.stringValue; + data.glue = { + registry: glueRegistry.stringValue + } } const ami = new dataAwsAmi.DataAwsAmi(this, "LatestAmi", { @@ -418,11 +415,17 @@ export class ZillaPlusWebStreamingStack extends TerraformStack { const kafkaBootstrapServers = `['${Fn.join(`','`, Fn.split(",", mskCluster.bootstrapBrokersSaslScram))}']`; - data.kafkaBootstrapServers = kafkaBootstrapServers; - data.kafkaSaslUsername = kafkaSaslUsername; - data.kafkaSaslPassword = kafkaSaslPassword; - data.publicPort = publicTcpPort.value; - data.publicTlsCertificateKey = publicTlsCertificateKey.stringValue; + data.kafka = { + bootstrapServers: kafkaBootstrapServers, + sasl : { + username: kafkaSaslUsername, + password: kafkaSaslPassword + } + } + data.public = { + port: publicTcpPort.value, + tlsCertificateKey: publicTlsCertificateKey.stringValue + } data.path = path; data.topic = topic.stringValue; diff --git a/amazon-msk/cdktf/web-streaming/zilla.yaml.mustache b/amazon-msk/cdktf/web-streaming/zilla.yaml.mustache index 9794661..1a643df 100644 --- a/amazon-msk/cdktf/web-streaming/zilla.yaml.mustache +++ b/amazon-msk/cdktf/web-streaming/zilla.yaml.mustache @@ -2,13 +2,13 @@ name: {{name}} vaults: secure: type: aws-secrets -{{#glueEnabled}} +{{#glue}} catalogs: glue_catalog: type: aws-glue options: - registry: {{glueRegistry}} -{{/glueEnabled}} + registry: {{glue.registry}} +{{/glue}} telemetry: metrics: - stream.active.received @@ -40,7 +40,7 @@ bindings: kind: server options: host: 0.0.0.0 - port: {{publicPort}} + port: {{public.port}} telemetry: metrics: - stream.* @@ -51,7 +51,7 @@ bindings: vault: secure options: keys: - - {{publicTlsCertificateKey}} + - {{public.tlsCertificateKey}} telemetry: metrics: - stream.* @@ -65,7 +65,8 @@ bindings: routes: - when: - headers: - :path: /streams{{{path}}} + :path: {{{path}}} + accept: text/event-stream exit: sse_server - when: - headers: @@ -88,7 +89,7 @@ bindings: - stream.* routes: - when: - - path: /streams{{{path}}} + - path: {{{path}}} exit: kafka_cache_client with: topic: {{topic}} @@ -144,7 +145,7 @@ bindings: kafka_cache_client: type: kafka kind: cache_client -{{#glueEnabled}} +{{#glue}} options: topics: - name: {{topic}} @@ -155,7 +156,7 @@ bindings: glue_catalog: - strategy: topic version: latest -{{/glueEnabled}} +{{/glue}} telemetry: metrics: - stream.* @@ -166,7 +167,7 @@ bindings: options: bootstrap: - {{topic}} -{{#glueEnabled}} +{{#glue}} topics: - name: {{topic}} value: @@ -176,7 +177,7 @@ bindings: glue_catalog: - strategy: topic version: latest -{{/glueEnabled}} +{{/glue}} telemetry: metrics: - stream.* @@ -185,11 +186,11 @@ bindings: type: kafka kind: client options: - servers: {{{kafkaBootstrapServers}}} + servers: {{{kafka.bootstrapServers}}} sasl: mechanism: scram-sha-512 - username: '{{kafkaSaslUsername}}' - password: '{{kafkaSaslPassword}}' + username: '{{kafka.sasl.username}}' + password: '{{kafka.sasl.password}}' telemetry: metrics: - stream.*