diff --git a/amazon-msk/cdktf/iot-ingest-and-control/main.ts b/amazon-msk/cdktf/iot-ingest-and-control/main.ts index 1cbc9be..a9f34fc 100644 --- a/amazon-msk/cdktf/iot-ingest-and-control/main.ts +++ b/amazon-msk/cdktf/iot-ingest-and-control/main.ts @@ -25,8 +25,18 @@ import { IamInstanceProfile } from "@cdktf/provider-aws/lib/iam-instance-profile import { IamRole } from "@cdktf/provider-aws/lib/iam-role"; import { IamRolePolicy } from "@cdktf/provider-aws/lib/iam-role-policy"; import { SecurityGroup } from "@cdktf/provider-aws/lib/security-group"; - import { UserVariables } from "./variables"; +import Mustache = require("mustache"); +import fs = require("fs"); + +interface TemplateData { + name: string; + cloudwatch?: object; + public?: object; + topics?: object; + kafka?: object; + +} export class ZillaPlusIotAndControlStack extends TerraformStack { constructor(scope: Construct, id: string) { @@ -316,8 +326,9 @@ export class ZillaPlusIotAndControlStack extends TerraformStack { errorMessage: "must be a valid EC2 instance type.", }); - let zillaTelemetryContent = ""; - let bindingTelemetryContent = ""; + const data: TemplateData = { + name: 'iot', + } if (!userVariables.cloudwatchDisabled) { const defaultLogGroupName = `${id}-group`; @@ -339,41 +350,14 @@ export class ZillaPlusIotAndControlStack extends TerraformStack { name: cloudWatchLogsGroup.stringValue, }); - const logsSection = ` - logs: - group: ${cloudWatchLogsGroup.stringValue} - stream: events`; - - const metricsSection = ` - metrics: - namespace: ${cloudWatchMetricsNamespace.stringValue}`; - - zillaTelemetryContent = ` -telemetry: - metrics: - - stream.active.received - - stream.active.sent - - stream.opens.received - - stream.opens.sent - - stream.data.received - - stream.data.sent - - stream.errors.received - - stream.errors.sent - - stream.closes.received - - stream.closes.sent - exporters: - stdout_logs_exporter: - type: stdout - aws0: - type: aws-cloudwatch - options: -${logsSection} -${metricsSection}`; - - bindingTelemetryContent = ` - telemetry: - metrics: - - stream.*`; + data.cloudwatch = { + logs: { + group: cloudWatchLogsGroup.stringValue + }, + metrics: { + namespace: cloudWatchMetricsNamespace.stringValue + } + }; } const ami = new dataAwsAmi.DataAwsAmi(this, "LatestAmi", { @@ -425,81 +409,25 @@ ${metricsSection}`; const kafkaBootstrapServers = `['${Fn.join(`','`, Fn.split(",", mskCluster.bootstrapBrokersSaslScram))}']`; - const zillaYamlContent = ` -name: public -${zillaTelemetryContent} -vaults: - secure: - type: aws -bindings: - tcp_server: - type: tcp - kind: server -${bindingTelemetryContent} - options: - host: 0.0.0.0 - port: ${publicTcpPort} - exit: tls_server - tls_server: - type: tls - kind: server - vault: secure -${bindingTelemetryContent} - options: - keys: - - ${publicTlsCertificateKey.stringValue} - exit: mqtt_server - mqtt_server: - type: mqtt - kind: server -${bindingTelemetryContent} - exit: mqtt_kafka_mapping - mqtt_kafka_mapping: - type: mqtt-kafka - kind: proxy -${bindingTelemetryContent} - options: - topics: - sessions: ${kafkaTopicMqttSessions} - messages: ${kafkaTopicMqttMessages} - retained: ${kafkaTopicMqttRetained} - exit: kafka_cache_client - kafka_cache_client: - type: kafka - kind: cache_client -${bindingTelemetryContent} - exit: kafka_cache_server - kafka_cache_server: - type: kafka - kind: cache_server -${bindingTelemetryContent} - options: - bootstrap: - - ${kafkaTopicMqttMessages} - - ${kafkaTopicMqttRetained} - exit: kafka_client - kafka_client: - type: kafka - kind: client - options: - servers: ${kafkaBootstrapServers} - sasl: - mechanism: scram-sha-512 - username: '${kafkaSaslUsername}' - password: '${kafkaSaslPassword}' -${bindingTelemetryContent} - exit: tls_client - tls_client: - type: tls - kind: client - vault: secure -${bindingTelemetryContent} - exit: tcp_client - tcp_client: - type: tcp - kind: client -${bindingTelemetryContent} -`; + data.kafka = { + bootstrapServers: kafkaBootstrapServers, + sasl : { + username: kafkaSaslUsername, + password: kafkaSaslPassword + } + } + data.public = { + port: publicTcpPort.value, + tlsCertificateKey: publicTlsCertificateKey.stringValue + } + data.topics = { + sessions: kafkaTopicMqttSessions.stringValue, + messages: kafkaTopicMqttMessages.stringValue, + retained: kafkaTopicMqttRetained.stringValue + }; + + const yamlTemplate: string = fs.readFileSync('zilla.yaml.mustache', 'utf8'); + const renderedYaml: string = Mustache.render(yamlTemplate, data); const cfnHupConfContent = ` [main] @@ -538,7 +466,7 @@ END_HELP const userData = `#!/bin/bash -xe yum update -y aws-cfn-bootstrap cat <<'END_HELP' > /etc/zilla/zilla.yaml -${zillaYamlContent} +${renderedYaml} END_HELP chown ec2-user:ec2-user /etc/zilla/zilla.yaml diff --git a/amazon-msk/cdktf/iot-ingest-and-control/package-lock.json b/amazon-msk/cdktf/iot-ingest-and-control/package-lock.json index 51182bc..58f1652 100644 --- a/amazon-msk/cdktf/iot-ingest-and-control/package-lock.json +++ b/amazon-msk/cdktf/iot-ingest-and-control/package-lock.json @@ -12,11 +12,14 @@ "@cdktf/provider-aws": "^19.26.0", "@dotenvx/dotenvx": "^1.6.4", "aws-sdk": "^2.1668.0", + "cdktf": "^0.20.8", "constructs": "^10.3.0", - "kafkajs": "^2.2.4" + "kafkajs": "^2.2.4", + "mustache": "^4.2.0" }, "devDependencies": { "@types/jest": "^29.5.12", + "@types/mustache": "^4.2.5", "@types/node": "^20.14.11", "cdktf-cli": "^0.20.8", "jest": "^29.7.0", @@ -2733,6 +2736,12 @@ "pretty-format": "^29.0.0" } }, + "node_modules/@types/mustache": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/@types/mustache/-/mustache-4.2.5.tgz", + "integrity": "sha512-PLwiVvTBg59tGFL/8VpcGvqOu3L4OuveNvPi0EYbWchRdEVP++yRUXJPFl+CApKEq13017/4Nf7aQ5lTtHUNsA==", + "dev": true + }, "node_modules/@types/mute-stream": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.1.tgz", @@ -7899,6 +7908,14 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/mute-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", diff --git a/amazon-msk/cdktf/iot-ingest-and-control/package.json b/amazon-msk/cdktf/iot-ingest-and-control/package.json index 496e601..bc61159 100644 --- a/amazon-msk/cdktf/iot-ingest-and-control/package.json +++ b/amazon-msk/cdktf/iot-ingest-and-control/package.json @@ -26,10 +26,12 @@ "aws-sdk": "^2.1668.0", "cdktf": "^0.20.8", "constructs": "^10.3.0", - "kafkajs": "^2.2.4" + "kafkajs": "^2.2.4", + "mustache": "^4.2.0" }, "devDependencies": { "@types/jest": "^29.5.12", + "@types/mustache": "^4.2.5", "@types/node": "^20.14.11", "cdktf-cli": "^0.20.8", "jest": "^29.7.0", diff --git a/amazon-msk/cdktf/iot-ingest-and-control/zilla.yaml.mustache b/amazon-msk/cdktf/iot-ingest-and-control/zilla.yaml.mustache new file mode 100644 index 0000000..2399288 --- /dev/null +++ b/amazon-msk/cdktf/iot-ingest-and-control/zilla.yaml.mustache @@ -0,0 +1,115 @@ +name: {{name}} +vaults: + secure: + type: aws-secrets +telemetry: + metrics: + - stream.active.received + - stream.active.sent + - stream.opens.received + - stream.opens.sent + - stream.data.received + - stream.data.sent + - stream.errors.received + - stream.errors.sent + - stream.closes.received + - stream.closes.sent + exporters: + stdout_logs_exporter: + type: stdout +{{#cloudwatch}} + aws0: + type: aws-cloudwatch + options: + logs: + group: {{cloudwatch.logs.group}} + stream: events + metrics: + namespace: {{cloudwatch.metrics.namespace}} +{{/cloudwatch}} +bindings: + tcp_server: + type: tcp + kind: server + options: + host: 0.0.0.0 + port: {{public.port}} + telemetry: + metrics: + - stream.* + exit: tls_server + tls_server: + type: tls + kind: server + vault: secure + options: + keys: + - {{public.tlsCertificateKey}} + telemetry: + metrics: + - stream.* + exit: mqtt_server + mqtt_server: + type: mqtt + kind: server + telemetry: + metrics: + - stream.* + exit: mqtt_kafka_mapping + mqtt_kafka_mapping: + type: mqtt-kafka + kind: proxy + options: + topics: + sessions: {{topics.sessions}} + messages: {{topics.messages}} + retained: {{topics.retained}} + telemetry: + metrics: + - stream.* + exit: kafka_cache_client + kafka_cache_client: + type: kafka + kind: cache_client + telemetry: + metrics: + - stream.* + exit: kafka_cache_server + kafka_cache_server: + type: kafka + kind: cache_server + options: + bootstrap: + - {{topics.messages}} + - {{topics.retained}} + telemetry: + metrics: + - stream.* + exit: kafka_client + kafka_client: + type: kafka + kind: client + options: + servers: {{{kafka.bootstrapServers}}} + sasl: + mechanism: scram-sha-512 + username: '{{kafka.sasl.username}}' + password: '{{kafka.sasl.password}}' + telemetry: + metrics: + - stream.* + exit: tls_client + tls_client: + type: tls + kind: client + vault: secure + telemetry: + metrics: + - stream.* + exit: tcp_client + tcp_client: + type: tcp + kind: client + telemetry: + metrics: + - stream.* diff --git a/amazon-msk/cdktf/secure-public-access/main.ts b/amazon-msk/cdktf/secure-public-access/main.ts index 7f5a6c1..7fb34e9 100644 --- a/amazon-msk/cdktf/secure-public-access/main.ts +++ b/amazon-msk/cdktf/secure-public-access/main.ts @@ -25,11 +25,24 @@ import { DataAwsAvailabilityZones } from "@cdktf/provider-aws/lib/data-aws-avail import { DataAwsSubnets } from "@cdktf/provider-aws/lib/data-aws-subnets"; import { IamInstanceProfile } from "@cdktf/provider-aws/lib/iam-instance-profile"; + import { UserVariables } from "./variables"; import { AwsProvider } from "@cdktf/provider-aws/lib/provider"; import { ec2EnclaveCertificateIamRoleAssociation } from "./.gen/providers/awscc" import { AwsccProvider } from "./.gen/providers/awscc/provider"; - +import Mustache = require("mustache"); +import fs = require("fs"); + +interface TemplateData { + name: string; + useAcm: boolean; + cloudwatch?: object; + public?: object; + mTLS?: boolean; + externalHost?: string; + internalHost?: string; + msk?: object; +} export class ZillaPlusSecurePublicAccessStack extends TerraformStack { constructor(scope: Construct, id: string) { @@ -147,9 +160,17 @@ export class ZillaPlusSecurePublicAccessStack extends TerraformStack { const mskBootstrapCommonPart = Fn.join(".", Fn.slice(addressParts, 1, Fn.lengthOf(addressParts))); mskWildcardDNS = Fn.format("*.%s", [mskBootstrapCommonPart]); - let tlsTrust = ""; - let tlsClientSigners = ""; - if (mskClientAuthentication === "mTLS") { + const mTLSEnabled = mskClientAuthentication === "mTLS"; + const publicTlsCertificateViaAcm = userVariables.publicTlsCertificateViaAcm; + + const data: TemplateData = { + name: 'public', + useAcm: publicTlsCertificateViaAcm, + mTLS: mTLSEnabled, + public: {} + }; + + if (mTLSEnabled) { // Seems like we can't get this from the MSK Cluster const mskCertificateAuthorityVar = new TerraformVariable(this, "msk_certificate_authority_arn", { type: "string", @@ -176,11 +197,9 @@ export class ZillaPlusSecurePublicAccessStack extends TerraformStack { }); publicCertificateAuthority = publicCertificateAuthorityVar.stringValue; } - - tlsTrust = ` trust: - - ${publicCertificateAuthority}`; - tlsClientSigners = ` signers: -- ${mskCertificateAuthority}`; + data.public = { + certificateAuthority: publicCertificateAuthority + } } const publicTlsCertificateKey = new TerraformVariable(this, "public_tls_certificate_key", { @@ -188,8 +207,6 @@ export class ZillaPlusSecurePublicAccessStack extends TerraformStack { description: "TLS Certificate SecretsManager or CertificateManager ARN", }); - const publicTlsCertificateViaAcm = userVariables.publicTlsCertificateViaAcm; - let zillaPlusRole; if (!userVariables.createZillaPlusRole) { const zillaPlusRoleVar = new TerraformVariable(this, "zilla_plus_role_name", { @@ -379,8 +396,6 @@ export class ZillaPlusSecurePublicAccessStack extends TerraformStack { let acmYamlContent = ""; let enclavesAcmServiceStart = ""; - let zillaTelemetryContent = ""; - let bindingTelemetryContent = ""; if (publicTlsCertificateViaAcm) { acmYamlContent = ` @@ -425,41 +440,15 @@ systemctl start nitro-enclaves-acm.service name: cloudWatchLogsGroup.stringValue, }); - const logsSection = ` - logs: - group: ${cloudWatchLogsGroup.stringValue} - stream: events`; - - const metricsSection = ` - metrics: - namespace: ${cloudWatchMetricsNamespace.stringValue}`; - - zillaTelemetryContent = ` -telemetry: - metrics: - - stream.active.received - - stream.active.sent - - stream.opens.received - - stream.opens.sent - - stream.data.received - - stream.data.sent - - stream.errors.received - - stream.errors.sent - - stream.closes.received - - stream.closes.sent - exporters: - stdout_logs_exporter: - type: stdout - aws0: - type: aws-cloudwatch - options: -${logsSection} -${metricsSection}`; - - bindingTelemetryContent = ` - telemetry: - metrics: - - stream.*`; + + data.cloudwatch = { + logs: { + group: cloudWatchLogsGroup.stringValue + }, + metrics: { + namespace: cloudWatchMetricsNamespace.stringValue + } + }; } const instanceType = new TerraformVariable(this, "zilla_plus_instance_type", { @@ -522,62 +511,20 @@ ${metricsSection}`; const internalHost = ["b-#.", Fn.element(Fn.split("*.", mskWildcardDNS), 1)].join(""); - const zillaYamlContent = ` -name: public -vaults: - secure: - type: ${publicTlsCertificateViaAcm ? "aws-acm" : "aws-secrets"} -${zillaTelemetryContent} -bindings: - tcp_server: - type: tcp - kind: server - options: - host: 0.0.0.0 - port: ${publicPort} -${bindingTelemetryContent} - exit: tls_server - tls_server: - type: tls - kind: server - vault: secure - options: - keys: - - ${publicTlsCertificateKey.stringValue} -${tlsTrust} - routes: - - exit: kafka_proxy - when: - - authority: '${publicWildcardDNS.stringValue}' - kafka_proxy: - type: kafka-proxy - kind: proxy - options: - external: - host: '${externalHost}' - port: ${publicPort} - internal: - host: '${internalHost}' - port: ${mskPort} - exit: tls_client - tls_client: - type: tls - kind: client - vault: secure - options: -${tlsClientSigners} - trustcacerts: true - exit: tcp_client - tcp_client: - type: tcp - kind: client - options: - host: '*' - port: ${mskPort} - routes: - - when: - - authority: '${mskWildcardDNS}' - `; + data.public = { + ...data.public, + port: publicPort.value, + tlsCertificateKey: publicTlsCertificateKey.stringValue, + wildcardDNS: publicWildcardDNS.stringValue + } + data.externalHost = externalHost; + data.internalHost = internalHost; + data.msk = { + port: mskPort, + wildcardDNS: mskWildcardDNS + } + const yamlTemplate: string = fs.readFileSync('zilla.yaml.mustache', 'utf8'); + const renderedYaml: string = Mustache.render(yamlTemplate, data); const cfnHupConfContent = ` [main] @@ -596,7 +543,7 @@ runas=root const userData = `#!/bin/bash -xe yum update -y aws-cfn-bootstrap cat < /etc/zilla/zilla.yaml -${zillaYamlContent} +${renderedYaml} EOF cat < /etc/nitro_enclaves/acm.yaml diff --git a/amazon-msk/cdktf/secure-public-access/package-lock.json b/amazon-msk/cdktf/secure-public-access/package-lock.json index 8f9d180..eb4c64a 100644 --- a/amazon-msk/cdktf/secure-public-access/package-lock.json +++ b/amazon-msk/cdktf/secure-public-access/package-lock.json @@ -16,10 +16,12 @@ "aws-cdk-lib": "^2.158.0", "cdktf": "^0.20.8", "constructs": "^10.3.0", + "mustache": "^4.2.0", "netmask": "^2.0.2" }, "devDependencies": { "@types/jest": "^29.5.12", + "@types/mustache": "^4.2.5", "@types/netmask": "^2.0.5", "@types/node": "^20.14.10", "cdktf-cli": "^0.20.8", @@ -3888,6 +3890,12 @@ "pretty-format": "^29.0.0" } }, + "node_modules/@types/mustache": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/@types/mustache/-/mustache-4.2.5.tgz", + "integrity": "sha512-PLwiVvTBg59tGFL/8VpcGvqOu3L4OuveNvPi0EYbWchRdEVP++yRUXJPFl+CApKEq13017/4Nf7aQ5lTtHUNsA==", + "dev": true + }, "node_modules/@types/mute-stream": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.1.tgz", @@ -9218,6 +9226,14 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/mute-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", diff --git a/amazon-msk/cdktf/secure-public-access/package.json b/amazon-msk/cdktf/secure-public-access/package.json index 09fcc0b..91bc553 100644 --- a/amazon-msk/cdktf/secure-public-access/package.json +++ b/amazon-msk/cdktf/secure-public-access/package.json @@ -28,10 +28,12 @@ "aws-cdk-lib": "^2.158.0", "cdktf": "^0.20.8", "constructs": "^10.3.0", + "mustache": "^4.2.0", "netmask": "^2.0.2" }, "devDependencies": { "@types/jest": "^29.5.12", + "@types/mustache": "^4.2.5", "@types/netmask": "^2.0.5", "@types/node": "^20.14.10", "cdktf-cli": "^0.20.8", diff --git a/amazon-msk/cdktf/secure-public-access/zilla.yaml.mustache b/amazon-msk/cdktf/secure-public-access/zilla.yaml.mustache new file mode 100644 index 0000000..359f2f5 --- /dev/null +++ b/amazon-msk/cdktf/secure-public-access/zilla.yaml.mustache @@ -0,0 +1,99 @@ +name: {{name}} +vaults: + secure: + type: {{#useAcm}}aws-acm{{/useAcm}}{{^useAcm}}aws-secrets{{/useAcm}} +telemetry: + metrics: + - stream.active.received + - stream.active.sent + - stream.opens.received + - stream.opens.sent + - stream.data.received + - stream.data.sent + - stream.errors.received + - stream.errors.sent + - stream.closes.received + - stream.closes.sent + exporters: + stdout_logs_exporter: + type: stdout +{{#cloudwatch}} + aws0: + type: aws-cloudwatch + options: + logs: + group: {{cloudwatch.logs.group}} + stream: events + metrics: + namespace: {{cloudwatch.metrics.namespace}} +{{/cloudwatch}} +bindings: + tcp_server: + type: tcp + kind: server + options: + host: 0.0.0.0 + port: {{public.port}} + telemetry: + metrics: + - stream.* + exit: tls_server + tls_server: + type: tls + kind: server + vault: secure + options: + keys: + - {{public.tlsCertificateKey}} + {{#mTLS}} + trust: + - {{public.certificateAuthority}} + {{/mTLS}} + telemetry: + metrics: + - stream.* + routes: + - exit: kafka_proxy + when: + - authority: '{{public.wildcardDNS}}' + kafka_proxy: + type: kafka-proxy + kind: proxy + options: + external: + host: '{{externalHost}}' + port: {{public.port}} + internal: + host: '{{internalHost}}' + port: {{msk.port}} + telemetry: + metrics: + - stream.* + exit: tls_client + tls_client: + type: tls + kind: client + vault: secure + options: + {{#mTLS}} + signers: + - {{msk.certificateAuthority}} + {{/mTLS}} + trustcacerts: true + telemetry: + metrics: + - stream.* + exit: tcp_client + tcp_client: + type: tcp + kind: client + options: + host: '*' + port: {{msk.port}} + telemetry: + metrics: + - stream.* + routes: + - when: + - authority: '{{msk.wildcardDNS}}' + diff --git a/amazon-msk/cdktf/web-streaming/main.ts b/amazon-msk/cdktf/web-streaming/main.ts index 40fd2a0..1a02657 100644 --- a/amazon-msk/cdktf/web-streaming/main.ts +++ b/amazon-msk/cdktf/web-streaming/main.ts @@ -25,8 +25,19 @@ import { IamInstanceProfile } from "@cdktf/provider-aws/lib/iam-instance-profile import { IamRole } from "@cdktf/provider-aws/lib/iam-role"; import { IamRolePolicy } from "@cdktf/provider-aws/lib/iam-role-policy"; import { SecurityGroup } from "@cdktf/provider-aws/lib/security-group"; - import { UserVariables } from "./variables"; +import Mustache = require("mustache"); +import fs = require("fs"); + +interface TemplateData { + name: string; + glue?: object; + cloudwatch?: object; + path?: string; + topic?: string; + public?: object; + kafka?: object; +} export class ZillaPlusWebStreamingStack extends TerraformStack { constructor(scope: Construct, id: string) { @@ -310,8 +321,9 @@ export class ZillaPlusWebStreamingStack extends TerraformStack { errorMessage: "must be a valid EC2 instance type.", }); - let zillaTelemetryContent = ""; - let bindingTelemetryContent = ""; + const data: TemplateData = { + name: 'web', + } if (!userVars.cloudwatchDisabled) { const defaultLogGroupName = `${id}-group`; @@ -333,77 +345,25 @@ export class ZillaPlusWebStreamingStack extends TerraformStack { name: cloudWatchLogsGroup.stringValue, }); - const logsSection = ` - logs: - group: ${cloudWatchLogsGroup.stringValue} - stream: events`; - - const metricsSection = ` - metrics: - namespace: ${cloudWatchMetricsNamespace.stringValue}`; - - zillaTelemetryContent = `telemetry: - metrics: - - stream.active.received - - stream.active.sent - - stream.opens.received - - stream.opens.sent - - stream.data.received - - stream.data.sent - - stream.errors.received - - stream.errors.sent - - stream.closes.received - - stream.closes.sent - exporters: - stdout_logs_exporter: - type: stdout - aws0: - type: aws-cloudwatch - options: -${logsSection} -${metricsSection}`; - - bindingTelemetryContent = ` telemetry: - metrics: - - stream.*`; + data.cloudwatch = { + logs: { + group: cloudWatchLogsGroup.stringValue + }, + metrics: { + namespace: cloudWatchMetricsNamespace.stringValue + } + }; } - let glueContent = ""; - let kafkaCacheClientGlueContent = ""; - let kafkaCacheServerGlueContent = ""; - if (userVars.glueRegistryEnabled) { const glueRegistry = new TerraformVariable(this, "glue_registry", { type: "string", description: "The Glue Registry to fetch the schemas from", }); - glueContent = `catalogs: - glue_catalog: - type: aws-glue - options: - registry: ${glueRegistry.stringValue}`; - - kafkaCacheClientGlueContent = ` options: - topics: - - name: ${topic} - value: - model: avro - view: json - catalog: - glue_catalog: - - strategy: topic - version: latest`; - - kafkaCacheServerGlueContent = ` topics: - - name: ${topic} - value: - model: avro - view: json - catalog: - glue_catalog: - - strategy: topic - version: latest`; + data.glue = { + registry: glueRegistry.stringValue + } } const ami = new dataAwsAmi.DataAwsAmi(this, "LatestAmi", { @@ -455,145 +415,22 @@ ${metricsSection}`; const kafkaBootstrapServers = `['${Fn.join(`','`, Fn.split(",", mskCluster.bootstrapBrokersSaslScram))}']`; - const zillaYamlContent = ` -name: public -${glueContent} -${zillaTelemetryContent} -vaults: - secure: - type: aws -bindings: - tcp_server: - type: tcp - kind: server -${bindingTelemetryContent} - options: - host: 0.0.0.0 - port: ${publicTcpPort} - exit: tls_server - tls_server: - type: tls - kind: server - vault: secure -${bindingTelemetryContent} - options: - keys: - - ${publicTlsCertificateKey.stringValue} - exit: north_http_server - north_http_server: - type: http - kind: server -${bindingTelemetryContent} - routes: - - when: - - headers: - :path: /streams${path} - exit: north_sse_server - - when: - - headers: - :path: ${path}/* - - headers: - :path: ${path} - exit: north_http_kafka_mapping - north_sse_server: - type: sse - kind: server -${bindingTelemetryContent} - exit: north_sse_kafka_mapping - north_sse_kafka_mapping: - type: sse-kafka - kind: proxy -${bindingTelemetryContent} - routes: - - when: - - path: /streams${path} - exit: kafka_cache_client - with: - topic: ${topic} - north_http_kafka_mapping: - type: http-kafka - kind: proxy -${bindingTelemetryContent} - routes: - - when: - - method: POST - path: ${path} - exit: kafka_cache_client - with: - capability: produce - topic: ${topic.stringValue} - key: \${idempotencyKey} - - when: - - method: PUT - path: ${path}/{id} - exit: kafka_cache_client - with: - capability: produce - topic: ${topic.stringValue} - key: \${params.id} - - when: - - method: DELETE - path: ${path}/{id} - exit: kafka_cache_client - with: - capability: produce - topic: ${topic.stringValue} - key: \${params.id} - - when: - - method: GET - path: ${path} - exit: kafka_cache_client - with: - capability: fetch - topic: ${topic.stringValue} - merge: - content-type: application/json - - when: - - method: GET - path: ${path}/{id} - exit: kafka_cache_client - with: - capability: fetch - topic: ${topic.stringValue} - filters: - - key: \${params.id} - kafka_cache_client: - type: kafka - kind: cache_client -${kafkaCacheClientGlueContent} -${bindingTelemetryContent} - exit: kafka_cache_server - kafka_cache_server: - type: kafka - kind: cache_server -${bindingTelemetryContent} - options: - bootstrap: - - ${topic.stringValue} -${kafkaCacheServerGlueContent} - exit: kafka_client - kafka_client: - type: kafka - kind: client - options: - servers: ${kafkaBootstrapServers} - sasl: - mechanism: scram-sha-512 - username: '${kafkaSaslUsername}' - password: '${kafkaSaslPassword}' -${bindingTelemetryContent} - exit: tls_client - tls_client: - type: tls - kind: client - vault: secure -${bindingTelemetryContent} - exit: tcp_client - tcp_client: - type: tcp - kind: client -${bindingTelemetryContent} -`; + data.kafka = { + bootstrapServers: kafkaBootstrapServers, + sasl : { + username: kafkaSaslUsername, + password: kafkaSaslPassword + } + } + data.public = { + port: publicTcpPort.value, + tlsCertificateKey: publicTlsCertificateKey.stringValue + } + data.path = path; + data.topic = topic.stringValue; + + const yamlTemplate: string = fs.readFileSync('zilla.yaml.mustache', 'utf8'); + const renderedYaml: string = Mustache.render(yamlTemplate, data); const cfnHupConfContent = ` [main] @@ -634,7 +471,7 @@ EOF const userData = `#!/bin/bash -xe yum update -y aws-cfn-bootstrap cat <<'END_HELP' > /etc/zilla/zilla.yaml -${zillaYamlContent} +${renderedYaml} END_HELP chown ec2-user:ec2-user /etc/zilla/zilla.yaml diff --git a/amazon-msk/cdktf/web-streaming/package-lock.json b/amazon-msk/cdktf/web-streaming/package-lock.json index 77950df..ca97a28 100644 --- a/amazon-msk/cdktf/web-streaming/package-lock.json +++ b/amazon-msk/cdktf/web-streaming/package-lock.json @@ -12,10 +12,12 @@ "@cdktf/provider-aws": "^19.27.0", "@dotenvx/dotenvx": "^1.6.4", "cdktf": "^0.20.8", - "constructs": "^10.3.0" + "constructs": "^10.3.0", + "mustache": "^4.2.0" }, "devDependencies": { "@types/jest": "^29.5.12", + "@types/mustache": "^4.2.5", "@types/node": "^20.14.11", "cdktf-cli": "^0.20.8", "jest": "^29.7.0", @@ -2722,6 +2724,12 @@ "pretty-format": "^29.0.0" } }, + "node_modules/@types/mustache": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/@types/mustache/-/mustache-4.2.5.tgz", + "integrity": "sha512-PLwiVvTBg59tGFL/8VpcGvqOu3L4OuveNvPi0EYbWchRdEVP++yRUXJPFl+CApKEq13017/4Nf7aQ5lTtHUNsA==", + "dev": true + }, "node_modules/@types/mute-stream": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.1.tgz", @@ -7835,6 +7843,14 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/mute-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", diff --git a/amazon-msk/cdktf/web-streaming/package.json b/amazon-msk/cdktf/web-streaming/package.json index 4a671c3..b25f509 100644 --- a/amazon-msk/cdktf/web-streaming/package.json +++ b/amazon-msk/cdktf/web-streaming/package.json @@ -24,10 +24,12 @@ "@cdktf/provider-aws": "^19.27.0", "@dotenvx/dotenvx": "^1.6.4", "cdktf": "^0.20.8", - "constructs": "^10.3.0" + "constructs": "^10.3.0", + "mustache": "^4.2.0" }, "devDependencies": { "@types/jest": "^29.5.12", + "@types/mustache": "^4.2.5", "@types/node": "^20.14.11", "cdktf-cli": "^0.20.8", "jest": "^29.7.0", diff --git a/amazon-msk/cdktf/web-streaming/zilla.yaml.mustache b/amazon-msk/cdktf/web-streaming/zilla.yaml.mustache new file mode 100644 index 0000000..1a643df --- /dev/null +++ b/amazon-msk/cdktf/web-streaming/zilla.yaml.mustache @@ -0,0 +1,211 @@ +name: {{name}} +vaults: + secure: + type: aws-secrets +{{#glue}} +catalogs: + glue_catalog: + type: aws-glue + options: + registry: {{glue.registry}} +{{/glue}} +telemetry: + metrics: + - stream.active.received + - stream.active.sent + - stream.opens.received + - stream.opens.sent + - stream.data.received + - stream.data.sent + - stream.errors.received + - stream.errors.sent + - stream.closes.received + - stream.closes.sent + exporters: + stdout_logs_exporter: + type: stdout +{{#cloudwatch}} + aws0: + type: aws-cloudwatch + options: + logs: + group: {{cloudwatch.logs.group}} + stream: events + metrics: + namespace: {{cloudwatch.metrics.namespace}} +{{/cloudwatch}} +bindings: + tcp_server: + type: tcp + kind: server + options: + host: 0.0.0.0 + port: {{public.port}} + telemetry: + metrics: + - stream.* + exit: tls_server + tls_server: + type: tls + kind: server + vault: secure + options: + keys: + - {{public.tlsCertificateKey}} + telemetry: + metrics: + - stream.* + exit: http_server + http_server: + type: http + kind: server + telemetry: + metrics: + - stream.* + routes: + - when: + - headers: + :path: {{{path}}} + accept: text/event-stream + exit: sse_server + - when: + - headers: + :path: {{{path}}}/* + - headers: + :path: {{{path}}} + exit: http_kafka_mapping + sse_server: + type: sse + kind: server + telemetry: + metrics: + - stream.* + exit: sse_kafka_mapping + sse_kafka_mapping: + type: sse-kafka + kind: proxy + telemetry: + metrics: + - stream.* + routes: + - when: + - path: {{{path}}} + exit: kafka_cache_client + with: + topic: {{topic}} + http_kafka_mapping: + type: http-kafka + kind: proxy + telemetry: + metrics: + - stream.* + routes: + - when: + - method: POST + path: {{{path}}} + exit: kafka_cache_client + with: + capability: produce + topic: {{topic}} + key: ${idempotencyKey} + - when: + - method: PUT + path: {{{path}}}/{id} + exit: kafka_cache_client + with: + capability: produce + topic: {{topic}} + key: ${params.id} + - when: + - method: DELETE + path: {{{path}}}/{id} + exit: kafka_cache_client + with: + capability: produce + topic: {{topic}} + key: ${params.id} + - when: + - method: GET + path: {{{path}}} + exit: kafka_cache_client + with: + capability: fetch + topic: {{topic}} + merge: + content-type: application/json + - when: + - method: GET + path: {{{path}}}/{id} + exit: kafka_cache_client + with: + capability: fetch + topic: {{topic}} + filters: + - key: ${params.id} + kafka_cache_client: + type: kafka + kind: cache_client +{{#glue}} + options: + topics: + - name: {{topic}} + value: + model: avro + view: json + catalog: + glue_catalog: + - strategy: topic + version: latest +{{/glue}} + telemetry: + metrics: + - stream.* + exit: kafka_cache_server + kafka_cache_server: + type: kafka + kind: cache_server + options: + bootstrap: + - {{topic}} +{{#glue}} + topics: + - name: {{topic}} + value: + model: avro + view: json + catalog: + glue_catalog: + - strategy: topic + version: latest +{{/glue}} + telemetry: + metrics: + - stream.* + exit: kafka_client + kafka_client: + type: kafka + kind: client + options: + servers: {{{kafka.bootstrapServers}}} + sasl: + mechanism: scram-sha-512 + username: '{{kafka.sasl.username}}' + password: '{{kafka.sasl.password}}' + telemetry: + metrics: + - stream.* + exit: tls_client + tls_client: + type: tls + kind: client + vault: secure + telemetry: + metrics: + - stream.* + exit: tcp_client + tcp_client: + type: tcp + kind: client + telemetry: + metrics: + - stream.*