Add make targets to generate manifests locally

Makefile

@@ -57,21 +57,45 @@ runtime_config_manifests_spec:
 	cd manifests/runtime-config &&\
 		bundle exec rspec
 
+.PHONY: cf_manifest
+cf_manifest: check-env
+	@./manifests/cf-manifest/scripts/generate-env-manifest.sh
+
 .PHONY: cf_manifest_spec
 cf_manifest_spec:
 	cd manifests/cf-manifest &&\
 		bundle exec rspec
 
+.PHONY: cf_manifest_stub
+cf_manifest_stub:
+	@./manifests/cf-manifest/scripts/generate-stub-manifest.sh
+
 .PHONY: prometheus_manifest_spec
 prometheus_manifest_spec:
 	cd manifests/prometheus &&\
 		bundle exec rspec
 
+.PHONY: prometheus_manifest
+prometheus_manifest: check-env
+	@./manifests/prometheus/scripts/generate-env-manifest.sh
+
+.PHONY: prometheus_manifest_stub
+prometheus_manifest_stub:
+	@./manifests/prometheus/scripts/generate-stub-manifest.sh
+
+.PHONY: app_autoscaler_manifest
+app_autoscaler_manifest: check-env
+	@./manifests/app-autoscaler/scripts/generate-env-manifest.sh
+
 .PHONY: app_autoscaler_manifest_spec
 app_autoscaler_manifest_spec:
 	cd manifests/app-autoscaler &&\
 		bundle exec rspec
 
+.PHONY: app_autoscaler_manifest_stub
+app_autoscaler_manifest_stub:
+	@./manifests/app-autoscaler/scripts/generate-stub-manifest.sh
+
 .PHONY: manifest_spec
 manifests_spec: cloud_config_manifests_spec runtime_config_manifests_spec cf_manifest_spec prometheus_manifest_spec app_autoscaler_manifest_spec
 

manifests/app-autoscaler/scripts/generate-env-manifest.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# work out the script directory
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PAAS_CF_DIR="$( cd "${SCRIPT_DIR}/../../.." && pwd )"
+export PAAS_CF_DIR
+
+# create temporary directory
+WORKDIR=$(mktemp -d)
+
+# clean up temporary directory on exit
+function cleanup {
+    # check WORKDIR does not equal /
+    if [[ "$WORKDIR" != "/" ]]; then
+        rm -rf "$WORKDIR"
+    fi
+}
+trap cleanup EXIT
+
+STATE_BUCKET="gds-paas-${DEPLOY_ENV}-state"
+
+cd "${WORKDIR}"
+
+mkdir -p "${WORKDIR}/terraform-tfstate"
+mkdir -p "${WORKDIR}/paas-trusted-people"
+mkdir -p "${WORKDIR}/terraform-outputs"
+aws s3 cp "s3://${STATE_BUCKET}/cf.tfstate" "$WORKDIR/terraform-tfstate/cf.tfstate" --quiet
+aws s3 cp "s3://${STATE_BUCKET}/paas-trusted-people/users.yml" "$WORKDIR/paas-trusted-people/users.yml" --quiet
+aws s3 cp "s3://${STATE_BUCKET}/bosh-CA.crt" "$WORKDIR/bosh-CA.crt" --quiet
+aws s3 cp "s3://${STATE_BUCKET}/bosh-secrets.yml" "$WORKDIR/bosh-secrets.yml" --quiet
+
+"${PAAS_CF_DIR}/concourse/scripts/extract_terraform_state_to_yaml.rb" \
+    < "${WORKDIR}/terraform-tfstate/cf.tfstate" \
+    > "${WORKDIR}/terraform-outputs/cf.yml"
+
+BOSH_URL="bosh.${SYSTEM_DNS_ZONE_NAME}"
+export BOSH_URL
+
+VCAP_PASSWORD=$(ruby -ryaml -e "puts YAML.load_file('$WORKDIR/bosh-secrets.yml')['secrets']['vcap_password']")
+export VCAP_PASSWORD
+
+BOSH_CA_CERT="$(cat "${WORKDIR}/bosh-CA.crt")"
+BOSH_CA_CERT="$(awk -v ORS='\\n' '1' <(printenv BOSH_CA_CERT | tr -d '\r'))"
+export BOSH_CA_CERT
+
+"${PAAS_CF_DIR}/manifests/app-autoscaler/scripts/generate-manifest.sh"

manifests/app-autoscaler/scripts/generate-stub-manifest.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# work out the script directory
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+PAAS_CF_DIR="$( cd "${SCRIPT_DIR}/../../.." && pwd )"
+export PAAS_CF_DIR
+WORKDIR=$PAAS_CF_DIR/manifests/shared/stubs
+export WORKDIR
+
+cd "${WORKDIR}" || exit 1
+
+AWS_ACCOUNT="dev"
+export AWS_ACCOUNT
+SYSTEM_DNS_ZONE_NAME="system.example.com"
+export SYSTEM_DNS_ZONE_NAME
+APPS_DNS_ZONE_NAME="apps.example.com"
+export APPS_DNS_ZONE_NAME
+DEPLOY_ENV="test"
+export DEPLOY_ENV
+BOSH_CA_CERT="bosh-CA.crt"
+export BOSH_CA_CERT
+VCAP_PASSWORD="vcap-password"
+export VCAP_PASSWORD
+
+"${PAAS_CF_DIR}/manifests/app-autoscaler/scripts/generate-manifest.sh"

manifests/cf-manifest/scripts/generate-env-manifest.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# work out the script directory
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PAAS_CF_DIR="$( cd "${SCRIPT_DIR}/../../.." && pwd )"
+export PAAS_CF_DIR
+
+# create temporary directory
+WORKDIR=$(mktemp -d)
+
+# clean up temporary directory on exit
+function cleanup {
+    # check WORKDIR does not equal /
+    if [[ "$WORKDIR" != "/" ]]; then
+        rm -rf "$WORKDIR"
+    fi
+}
+trap cleanup EXIT
+
+STATE_BUCKET="gds-paas-${DEPLOY_ENV}-state"
+
+cd "${WORKDIR}"
+mkdir -p "${WORKDIR}/terraform-tfstate"
+mkdir -p "${WORKDIR}/terraform-outputs"
+# download state files from s3 bucket using aws command line
+aws s3 cp "s3://${STATE_BUCKET}/vpc.tfstate" "$WORKDIR/terraform-tfstate/vpc.tfstate" --quiet
+aws s3 cp "s3://${STATE_BUCKET}/psn.tfstate" "$WORKDIR/terraform-tfstate/psn.tfstate" --quiet
+aws s3 cp "s3://${STATE_BUCKET}/bosh.tfstate" "$WORKDIR/terraform-tfstate/bosh.tfstate" --quiet
+aws s3 cp "s3://${STATE_BUCKET}/cf.tfstate" "$WORKDIR/terraform-tfstate/cf.tfstate" --quiet
+aws s3 cp "s3://${STATE_BUCKET}/azhc.tfstate" "$WORKDIR/terraform-tfstate/azhc.tfstate" --quiet
+aws s3 cp "s3://${STATE_BUCKET}/bosh-secrets.yml" "$WORKDIR/bosh-secrets.yml" --quiet
+
+for state in vpc bosh cf; do
+    "${PAAS_CF_DIR}/concourse/scripts/extract_terraform_state_to_yaml.rb" \
+        < "${WORKDIR}/terraform-tfstate/${state}.tfstate" \
+        > "${WORKDIR}/terraform-outputs/${state}.yml"
+    "${PAAS_CF_DIR}/concourse/scripts/extract_tf_vars_from_terraform_state.rb" \
+        < "${WORKDIR}/terraform-tfstate/${state}.tfstate" \
+        > "${WORKDIR}/terraform-outputs/${state}.tfvars.sh"
+done
+
+terraform output \
+    -raw \
+    "-state=${WORKDIR}/terraform-tfstate/psn.tfstate" \
+    psn_security_group_seed_json > "${WORKDIR}/terraform-outputs/psn-peering.json"
+
+mkdir -p "${WORKDIR}/vpc-peering-opsfile"
+ruby "${PAAS_CF_DIR}/terraform/scripts/generate_vpc_peering_opsfile.rb" "${PAAS_CF_DIR}/terraform/${DEPLOY_ENV}.vpc_peering.json" \
+    > "${WORKDIR}/vpc-peering-opsfile/vpc-peers.yml"
+
+mkdir -p "${WORKDIR}/tenant-uaa-clients-opsfile"
+ruby "${PAAS_CF_DIR}/manifests/cf-manifest/scripts/generate-tenant-uaa-client-ops-file.rb" \
+    "${PAAS_CF_DIR}/manifests/cf-manifest/data/100-tenant-uaa-client-config.yml" \
+    "${MAKEFILE_ENV_TARGET}" \
+    > "${WORKDIR}/tenant-uaa-clients-opsfile/tenant-uaa-opsfile.yml"
+
+mkdir -p "${WORKDIR}/ms-oauth-endpoints"
+DISCOVERY_DOC=$(curl -s "https://login.microsoftonline.com/common/v2.0/.well-known/openid-configuration")
+
+echo "${DISCOVERY_DOC}" | jq '.authorization_endpoint' --raw-output \
+    > "${WORKDIR}/ms-oauth-endpoints/authorization_endpoint"
+
+echo "${DISCOVERY_DOC}" | jq '.token_endpoint' --raw-output \
+    > "${WORKDIR}/ms-oauth-endpoints/token_endpoint"
+
+echo "${DISCOVERY_DOC}" | jq '.jwks_uri' --raw-output \
+    > "${WORKDIR}/ms-oauth-endpoints/token_key_endpoint"
+
+echo "${DISCOVERY_DOC}" | jq '.issuer' --raw-output \
+    > "${WORKDIR}/ms-oauth-endpoints/issuer"
+
+mkdir -p "${WORKDIR}/psn-peering-opsfile"
+ruby "${PAAS_CF_DIR}/terraform/scripts/generate_vpc_peering_opsfile.rb" "${WORKDIR}/terraform-outputs/psn-peering.json" \
+    > "${WORKDIR}/psn-peering-opsfile/psn-peers.yml"
+
+mkdir -p "${WORKDIR}/paas-cf-cloud-config"
+"${PAAS_CF_DIR}/manifests/cloud-config/scripts/generate-cloud-config.sh" > "${WORKDIR}/paas-cf-cloud-config/paas-cf-cloud-config.yml"
+
+mkdir -p "${WORKDIR}/paas-cf-runtime-config"
+"${PAAS_CF_DIR}/manifests/runtime-config/scripts/generate-runtime-config.sh" > "${WORKDIR}/paas-cf-runtime-config/paas-cf-runtime-config.yml"
+
+ENV_SPECIFIC_BOSH_VARS_FILE="${PAAS_CF_DIR}/manifests/cf-manifest/env-specific/${ENV_SPECIFIC_BOSH_VARS_FILE}"
+export ENV_SPECIFIC_BOSH_VARS_FILE
+ENV_SPECIFIC_ISOLATION_SEGMENTS_DIR="${PAAS_CF_DIR}/manifests/cf-manifest/isolation-segments/${ENV_SPECIFIC_ISOLATION_SEGMENTS_DIR}"
+export ENV_SPECIFIC_ISOLATION_SEGMENTS_DIR
+
+VCAP_PASSWORD=$(ruby -ryaml -e "puts YAML.load_file('$WORKDIR/bosh-secrets.yml')['secrets']['vcap_password']")
+export VCAP_PASSWORD
+
+cat <<EOF > environment-variables.yml
+---
+system_domain: ${SYSTEM_DNS_ZONE_NAME}
+app_domain: ${APPS_DNS_ZONE_NAME}
+environment: ${DEPLOY_ENV}
+deployment_name: ${DEPLOY_ENV}
+aws_account: ${AWS_ACCOUNT}
+microsoft_oauth_auth_url: $(cat ms-oauth-endpoints/authorization_endpoint)
+microsoft_oauth_token_url: $(cat ms-oauth-endpoints/token_endpoint)
+microsoft_oauth_token_key_url: $(cat ms-oauth-endpoints/token_key_endpoint)
+microsoft_oauth_issuer: $(cat ms-oauth-endpoints/issuer)
+vcap_password: $VCAP_PASSWORD
+EOF
+
+"${PAAS_CF_DIR}/manifests/cf-manifest/scripts/generate-manifest.sh"

manifests/cf-manifest/scripts/generate-stub-manifest.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# work out the script directory
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+PAAS_CF_DIR="$( cd "${SCRIPT_DIR}/../../.." && pwd )"
+export PAAS_CF_DIR
+WORKDIR="${PAAS_CF_DIR}/manifests/shared/stubs"
+export WORKDIR
+
+cd "${WORKDIR}" || exit 1
+
+ENV_SPECIFIC_BOSH_VARS_FILE=$PAAS_CF_DIR/manifests/cf-manifest/env-specific/default.yml
+export ENV_SPECIFIC_BOSH_VARS_FILE
+ENV_SPECIFIC_ISOLATION_SEGMENTS_DIR=$PAAS_CF_DIR/manifests/cf-manifest/isolation-segments/default
+export ENV_SPECIFIC_ISOLATION_SEGMENTS_DIR
+
+"${PAAS_CF_DIR}/manifests/cf-manifest/scripts/generate-manifest.sh"
+
+

manifests/prometheus/scripts/generate-env-manifest.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# work out the script directory
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PAAS_CF_DIR="$( cd "${SCRIPT_DIR}/../../.." && pwd )"
+export PAAS_CF_DIR
+
+# create temporary directory
+WORKDIR=$(mktemp -d)
+
+# clean up temporary directory on exit
+function cleanup {
+    if [[ "$WORKDIR" != "/" ]]; then
+        rm -r "$WORKDIR"
+    fi
+}
+trap cleanup EXIT
+
+STATE_BUCKET="gds-paas-${DEPLOY_ENV}-state"
+
+cd "${WORKDIR}"
+
+credhub find -n /concourse/main/bosh-exporter-password > /dev/null 2>&1 || (echo "You need to connect to credhub." && exit 1)
+
+mkdir -p "${WORKDIR}/terraform-tfstate"
+mkdir -p "${WORKDIR}/paas-trusted-people"
+mkdir -p "${WORKDIR}/terraform-outputs"
+aws s3 cp "s3://${STATE_BUCKET}/cf.tfstate" "${WORKDIR}/terraform-tfstate/cf.tfstate" --quiet
+aws s3 cp "s3://${STATE_BUCKET}/paas-trusted-people/users.yml" "${WORKDIR}/paas-trusted-people/users.yml" --quiet
+aws s3 cp "s3://${STATE_BUCKET}/bosh-CA.crt" "${WORKDIR}/bosh-CA.crt" --quiet
+aws s3 cp "s3://${STATE_BUCKET}/bosh-secrets.yml" "${WORKDIR}/bosh-secrets.yml" --quiet
+
+"${PAAS_CF_DIR}/concourse/scripts/extract_terraform_state_to_yaml.rb" \
+    < "${WORKDIR}/terraform-tfstate/cf.tfstate" \
+    > "${WORKDIR}/terraform-outputs/cf.yml"
+
+BOSH_URL="bosh.${SYSTEM_DNS_ZONE_NAME}"
+export BOSH_URL
+
+VCAP_PASSWORD=$(ruby -ryaml -e "puts YAML.load_file('$WORKDIR/bosh-secrets.yml')['secrets']['vcap_password']")
+export VCAP_PASSWORD
+
+# All these variables are scoped into the wrong part of credhub. Why? who knows. This means
+# this script needs access to credhub to extract them. Wouldn't it be lovely if they were
+# scoped right, then we could just refer to them as ((variable)) in the manifest.
+
+BOSH_EXPORTER_PASSWORD="$(credhub get -q -n /concourse/main/bosh-exporter-password)"
+export BOSH_EXPORTER_PASSWORD
+GRAFANA_AUTH_GOOGLE_CLIENT_ID="$(credhub get -q -n /concourse/main/create-cloudfoundry/grafana_auth_google_client_id)"
+export GRAFANA_AUTH_GOOGLE_CLIENT_ID
+GRAFANA_AUTH_GOOGLE_CLIENT_SECRET="$(credhub get -q -n /concourse/main/create-cloudfoundry/grafana_auth_google_client_secret)"
+export GRAFANA_AUTH_GOOGLE_CLIENT_SECRET
+UAA_CLIENTS_CF_EXPORTER_SECRET="$(credhub get -q -n /concourse/main/create-cloudfoundry/uaa_clients_cf_exporter_secret)"
+export UAA_CLIENTS_CF_EXPORTER_SECRET
+UAA_CLIENTS_FIREHOSE_EXPORTER_SECRET="$(credhub get -q -n /concourse/main/create-cloudfoundry/uaa_clients_firehose_exporter_secret)"
+export UAA_CLIENTS_FIREHOSE_EXPORTER_SECRET
+
+BOSH_CA_CERT="$(cat "${WORKDIR}/bosh-CA.crt")"
+BOSH_CA_CERT="$(awk -v ORS='\\n' '1' <(printenv BOSH_CA_CERT | tr -d '\r'))"
+export BOSH_CA_CERT
+
+"${PAAS_CF_DIR}/manifests/prometheus/scripts/generate-manifest.sh"

manifests/prometheus/scripts/generate-stub-manifest.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# work out the script directory
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+PAAS_CF_DIR="$( cd "${SCRIPT_DIR}/../../.." && pwd )"
+export PAAS_CF_DIR
+WORKDIR=$PAAS_CF_DIR/manifests/shared/stubs
+export WORKDIR
+
+cd "${WORKDIR}" || exit 1
+
+export DEPLOY_ENV="dev"
+export BOSH_URL="https://bosh.local"
+export AWS_ACCOUNT="dev"
+export AWS_REGION="fake-1"
+export SYSTEM_DNS_ZONE_NAME="system.example.com"
+export APPS_DNS_ZONE_NAME="apps.example.com"
+export DEPLOY_ENV="test"
+export BOSH_URL="https://bosh.example.com:25555"
+export GRAFANA_AUTH_GOOGLE_CLIENT_ID="google-client-id"
+export GRAFANA_AUTH_GOOGLE_CLIENT_SECRET="google-client-secret"
+export UAA_CLIENTS_CF_EXPORTER_SECRET="uaa_clients_cf_exporter_secret"
+export UAA_CLIENTS_FIREHOSE_EXPORTER_SECRET="uaa_clients_firehose_exporter_secret"
+export BOSH_CA_CERT="bosh-CA.crt"
+export BOSH_EXPORTER_PASSWORD="bosh-exporter-password"
+export VCAP_PASSWORD="vcap-password"
+
+export VARS_STORE="${WORKDIR}/cf-vars-store.yml"
+export ENV_SPECIFIC_BOSH_VARS_FILE="default.yml"
+
+"${PAAS_CF_DIR}/manifests/prometheus/scripts/generate-manifest.sh"

manifests/shared/stubs/bosh-CA.crt

@@ -0,0 +1 @@
+../spec/fixtures/bosh-CA.crt

manifests/shared/stubs/bosh-secrets.yml

@@ -0,0 +1 @@
+../spec/fixtures/bosh-secrets.yml

manifests/shared/stubs/bosh-vars-store.yml

@@ -0,0 +1 @@
+../spec/fixtures/bosh-vars-store.yml

manifests/shared/stubs/cf-vars-store.yml

@@ -0,0 +1 @@
+../spec/fixtures/cf-vars-store.yml

manifests/shared/stubs/environment-variables.yml

@@ -0,0 +1 @@
+../spec/fixtures/environment-variables.yml

manifests/shared/stubs/pagerduty-secrets.yml

@@ -0,0 +1 @@
+../spec/fixtures/pagerduty-secrets.yml

manifests/shared/stubs/psn-peering-opsfile/psn-peers.yml

@@ -0,0 +1 @@
+---

manifests/shared/stubs/sample_key

@@ -0,0 +1 @@
+../spec/fixtures/sample_key

manifests/shared/stubs/sample_key.pub

@@ -0,0 +1 @@
+../spec/fixtures/sample_key.pub

manifests/shared/stubs/terraform-outputs/bosh.yml

@@ -0,0 +1 @@
+../../spec/fixtures/terraform/bosh.yml

manifests/shared/stubs/terraform-outputs/cf.yml

@@ -0,0 +1 @@
+../../spec/fixtures/terraform/cf.yml

manifests/shared/stubs/terraform-outputs/vpc.yml

@@ -0,0 +1 @@
+../../spec/fixtures/terraform/vpc.yml

manifests/shared/stubs/vpc-peering-opsfile/vpc-peers.yml

@@ -0,0 +1 @@
+---