diff --git a/.travis.yml b/.travis.yml index f7a9f6ae..7915ad0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,6 +58,14 @@ deploy: all_branches: true condition: $TRAVIS_BRANCH == sandbox-* && $TRAVIS_EVENT_TYPE == push +- provider: script + script: "$TRAVIS_BUILD_DIR/deploy_bbe_config.sh mlab-sandbox LINODE_PRIVATE_KEY_ipv6_monitoring" + skip_cleanup: true + on: + repo: m-lab/prometheus-support + branch: sandbox-* + condition: "$TRAVIS_EVENT_TYPE == push" + ######################################### ## Staging - provider: script @@ -84,6 +92,14 @@ deploy: all_branches: true condition: $TRAVIS_BRANCH == master && $TRAVIS_EVENT_TYPE == push +- provider: script + script: "$TRAVIS_BUILD_DIR/deploy_bbe_config.sh mlab-staging LINODE_PRIVATE_KEY_ipv6_monitoring" + skip_cleanup: true + on: + repo: m-lab/prometheus-support + branch: master + condition: "$TRAVIS_EVENT_TYPE == push" + ######################################### ## Production - provider: script @@ -110,6 +126,13 @@ deploy: repo: m-lab/prometheus-support tags: true +- provider: script + script: "$TRAVIS_BUILD_DIR/deploy_bbe_config.sh mlab-oti LINODE_PRIVATE_KEY_ipv6_monitoring" + skip_cleanup: true + on: + repo: m-lab/prometheus-support + tags: true + before_install: - travis/decrypt.sh "$encrypted_1c8dd3704323_key" "$encrypted_1c8dd3704323_iv" diff --git a/config/federation/blackbox/config.yml b/config/federation/blackbox/config.yml index 01ba0800..7bbf3b0c 100644 --- a/config/federation/blackbox/config.yml +++ b/config/federation/blackbox/config.yml @@ -26,14 +26,21 @@ # See https://github.com/prometheus/blackbox_exporter for additional examples. modules: - # target= + # IPv4: target= tcp_v4_online: prober: tcp timeout: 9s tcp: preferred_ip_protocol: "ip4" - # target= + # IPv6: target= + tcp_v6_online: + prober: tcp + timeout: 9s + tcp: + preferred_ip_protocol: "ip6" + + # IPv4 target= ssh_v4_online: prober: tcp timeout: 9s @@ -42,7 +49,16 @@ modules: query_response: - expect: "SSH-2.0-OpenSSH_.+" - # target= + # IPv6 target= + ssh_v6_online: + prober: tcp + timeout: 9s + tcp: + preferred_ip_protocol: "ip6" + query_response: + - expect: "SSH-2.0-OpenSSH_.+" + + # IPv4: target= tcp_v4_tls_online: prober: tcp timeout: 9s @@ -50,8 +66,16 @@ modules: preferred_ip_protocol: "ip4" tls: true - # target=:9773/sapi/state - neubot_online: + # IPv6: target= + tcp_v6_tls_online: + prober: tcp + timeout: 9s + tcp: + preferred_ip_protocol: "ip6" + tls: true + + # IPv4: target=:9773/sapi/state + neubot_online_v4: prober: http timeout: 9s http: @@ -59,6 +83,15 @@ modules: fail_if_not_matches_regexp: - "queue_len_cur" + # IPv6: target=:9773/sapi/state + neubot_online_v6: + prober: http + timeout: 9s + http: + preferred_ip_protocol: "ip6" + fail_if_not_matches_regexp: + - "queue_len_cur" + # target= rsyncd_online: prober: tcp diff --git a/config/federation/prometheus/prometheus.yml.template b/config/federation/prometheus/prometheus.yml.template index 9ca60520..afb63c66 100644 --- a/config/federation/prometheus/prometheus.yml.template +++ b/config/federation/prometheus/prometheus.yml.template @@ -381,6 +381,57 @@ scrape_configs: replacement: blackbox-public-service.default.svc.cluster.local:9115 + # Blackbox configurations for IPv6 probes. + # + # Each blackbox configuration uses a different probe (tcp, icmp, http, etc). + - job_name: 'blackbox-targets-ipv6' + metrics_path: /probe + + file_sd_configs: + - files: + - /blackbox-targets-ipv6/*.json + # Attempt to re-read files every five minutes. + refresh_interval: 5m + + # This relabel config is necessary. The relabel config redefines the address + # to scrape and sets the correct parameters to pass to the scrape target. + # + # While not as direct as other configs, this approach allows us to specify a + # dynamic list of targets for a static blackbox exporter. This is also the + # supported configuration: https://github.com/prometheus/blackbox_exporter + relabel_configs: + + # The default __address__ value is a target host from the config file. + # Here, we set (i.e. "replace") a request parameter "target" equal to the + # host value. + - source_labels: [__address__] + regex: (.*) + target_label: __param_target + replacement: ${1} + + # Use the "module" label defined in the input file as the module name for + # the blackbox exporter request. + - source_labels: [module] + regex: (.*) + target_label: __param_module + replacement: ${1} + + # Use the target parameter defined above and use it to define the + # "instance" label. + - source_labels: [__param_target] + regex: (.*) + target_label: instance + replacement: ${1} + + # Since __address__ is the target that prometheus will contact, + # unconditionally reset the __address__ label to the blackbox exporter + # address. + - source_labels: [__blackbox_port] + regex: (.*) + target_label: __address__ + replacement: blackbox-exporter-ipv6.{{PROJECT}}.measurementlab.net:${1} + + # Scrape config for the snmp_exporter. There is one snmp_exporter service # running in a Docker container on a GCE VM in each GCP project. # diff --git a/deploy_bbe_config.sh b/deploy_bbe_config.sh new file mode 100755 index 00000000..d6a03b48 --- /dev/null +++ b/deploy_bbe_config.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Deploys the blackbox_exporter config to an external (e.g., Linode) VM which +# will perform IPv6 probes, since GCP doesn't currently support IPv6. +# +# Example usage: +# ./deploy_bbe_config.sh mlab-sandbox LINODE_PRIVATE_KEY_ipv6_monitoring + +set -e +set -u +set -x + +BASE_DIR=$( dirname ${BASH_SOURCE[0]} ) +USAGE="Usage: $0 " +PROJECT=${1:?Please provide project name: $USAGE} +KEYNAME=${2:?Please provide an authentication key name: $USAGE} +BBE_CONFIG="${BASE_DIR}/config/federation/blackbox/config.yml" +LINODE_DOMAIN="blackbox-exporter-ipv6.${PROJECT}.measurementlab.net" +LINODE_USER="mlab" +LOCAL_KEY_FILE="id_rsa_linode" +SSH_OPTS="-i $LOCAL_KEY_FILE -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" + +# Extract the SSH key from the configured Travis environment variable. The key +# is base64 encoded to avoid the need for shell escaping and newlines. Set the +# mode of the file appropriately, as SSH will refuse to use it if the +# permissions are not strict enough. +echo "${!KEYNAME}" | base64 -d > $LOCAL_KEY_FILE +chmod 600 $LOCAL_KEY_FILE + +# Copy blackbox_exporter config file to the Linode VM. +scp $SSH_OPTS $BBE_CONFIG $LINODE_USER@$LINODE_DOMAIN:blackbox-exporter-config-$PROJECT.yml + +# HUP the blackbox_exporter so it reads the new config. +ssh $SSH_OPTS $LINODE_USER@$LINODE_DOMAIN "docker exec ${PROJECT} kill -HUP 1" diff --git a/k8s/prometheus-federation/deployments/prometheus.yml b/k8s/prometheus-federation/deployments/prometheus.yml index b9c0dd81..68f5c121 100644 --- a/k8s/prometheus-federation/deployments/prometheus.yml +++ b/k8s/prometheus-federation/deployments/prometheus.yml @@ -92,6 +92,10 @@ spec: - mountPath: /blackbox-targets name: prometheus-storage subPath: blackbox-targets + # /blackbox-targets-ipv6 should contain blackbox IPv6 target files. + - mountPath: /blackbox-targets-ipv6 + name: prometheus-storage + subPath: blackbox-targets-ipv6 # /aeflex-targets should contain AppEngine target config files. - mountPath: /aeflex-targets name: prometheus-storage @@ -143,6 +147,8 @@ spec: "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/legacy-targets/nodeexporter.json", "--http-target=/targets/blackbox-targets/ssh806.json", "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/blackbox-targets/ssh806.json", + "--http-target=/targets/blackbox-targets-ipv6/ssh806_ipv6.json", + "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/blackbox-targets-ipv6/ssh806_ipv6.json", "--http-target=/targets/blackbox-targets/rsyncd.json", "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/blackbox-targets/rsyncd.json", "--http-target=/targets/snmp-targets/snmpexporter.json", @@ -159,8 +165,12 @@ spec: "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/legacy-targets/ndt_inotify.json", "--http-target=/targets/blackbox-targets/mobiperf.json", "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/blackbox-targets/mobiperf.json", + "--http-target=/targets/blackbox-targets-ipv6/mobiperf_ipv6.json", + "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/blackbox-targets-ipv6/mobiperf_ipv6.json", "--http-target=/targets/blackbox-targets/neubot.json", "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/blackbox-targets/neubot.json", + "--http-target=/targets/blackbox-targets-ipv6/neubot_ipv6.json", + "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/blackbox-targets-ipv6/neubot_ipv6.json", "--project={{GCLOUD_PROJECT}}"] resources: requests: