Merge pull request #200 from m-lab/sandbox-kinkade

Adds script to deploy blackbox_exporter config to Linode VM, adds IPv6 BBE modules
m-lab · Mar 23, 2018 · 146d998 · 146d998
2 parents bd7f6b0 + d98ba0d
commit 146d998
Show file tree

Hide file tree

Showing 5 changed files with 156 additions and 5 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -58,6 +58,14 @@ deploy:
     all_branches: true
     condition: $TRAVIS_BRANCH == sandbox-* && $TRAVIS_EVENT_TYPE == push
 
+- provider: script
+  script: "$TRAVIS_BUILD_DIR/deploy_bbe_config.sh mlab-sandbox LINODE_PRIVATE_KEY_ipv6_monitoring"
+  skip_cleanup: true
+  on:
+    repo: m-lab/prometheus-support
+    branch: sandbox-*
+    condition: "$TRAVIS_EVENT_TYPE == push"
+
 #########################################
 ## Staging
 - provider: script
@@ -84,6 +92,14 @@ deploy:
     all_branches: true
     condition: $TRAVIS_BRANCH == master && $TRAVIS_EVENT_TYPE == push
 
+- provider: script
+  script: "$TRAVIS_BUILD_DIR/deploy_bbe_config.sh mlab-staging LINODE_PRIVATE_KEY_ipv6_monitoring"
+  skip_cleanup: true
+  on:
+    repo: m-lab/prometheus-support
+    branch: master
+    condition: "$TRAVIS_EVENT_TYPE == push"
+
 #########################################
 ## Production
 - provider: script
@@ -110,6 +126,13 @@ deploy:
     repo: m-lab/prometheus-support
     tags: true
 
+- provider: script
+  script: "$TRAVIS_BUILD_DIR/deploy_bbe_config.sh mlab-oti LINODE_PRIVATE_KEY_ipv6_monitoring"
+  skip_cleanup: true
+  on:
+    repo: m-lab/prometheus-support
+    tags: true
+
 
 before_install:
 - travis/decrypt.sh "$encrypted_1c8dd3704323_key" "$encrypted_1c8dd3704323_iv"

diff --git a/config/federation/blackbox/config.yml b/config/federation/blackbox/config.yml
@@ -26,14 +26,21 @@
 # See https://github.com/prometheus/blackbox_exporter for additional examples.
 
 modules:
-  # target=<hostname:port>
+  # IPv4: target=<hostname:port>
   tcp_v4_online:
     prober: tcp
     timeout: 9s
     tcp:
       preferred_ip_protocol: "ip4"
 
-  # target=<hostname:port>
+  # IPv6: target=<hostname:port>
+  tcp_v6_online:
+    prober: tcp
+    timeout: 9s
+    tcp:
+      preferred_ip_protocol: "ip6"
+
+  # IPv4 target=<hostname:port>
   ssh_v4_online:
     prober: tcp
     timeout: 9s
@@ -42,23 +49,49 @@ modules:
       query_response:
         - expect: "SSH-2.0-OpenSSH_.+"
 
-  # target=<hostname:port>
+  # IPv6 target=<hostname:port>
+  ssh_v6_online:
+    prober: tcp
+    timeout: 9s
+    tcp:
+      preferred_ip_protocol: "ip6"
+      query_response:
+        - expect: "SSH-2.0-OpenSSH_.+"
+
+  # IPv4: target=<hostname:port>
   tcp_v4_tls_online:
     prober: tcp
     timeout: 9s
     tcp:
       preferred_ip_protocol: "ip4"
       tls: true
 
-  # target=<hostname>:9773/sapi/state
-  neubot_online:
+  # IPv6: target=<hostname:port>
+  tcp_v6_tls_online:
+    prober: tcp
+    timeout: 9s
+    tcp:
+      preferred_ip_protocol: "ip6"
+      tls: true
+
+  # IPv4: target=<hostname>:9773/sapi/state
+  neubot_online_v4:
     prober: http
     timeout: 9s
     http:
       preferred_ip_protocol: "ip4"
       fail_if_not_matches_regexp:
         - "queue_len_cur"
 
+  # IPv6: target=<hostname>:9773/sapi/state
+  neubot_online_v6:
+    prober: http
+    timeout: 9s
+    http:
+      preferred_ip_protocol: "ip6"
+      fail_if_not_matches_regexp:
+        - "queue_len_cur"
+
   # target=<hostname:port>
   rsyncd_online:
     prober: tcp

diff --git a/config/federation/prometheus/prometheus.yml.template b/config/federation/prometheus/prometheus.yml.template
@@ -381,6 +381,57 @@ scrape_configs:
         replacement: blackbox-public-service.default.svc.cluster.local:9115
 
 
+  # Blackbox configurations for IPv6 probes.
+  #
+  # Each blackbox configuration uses a different probe (tcp, icmp, http, etc).
+  - job_name: 'blackbox-targets-ipv6'
+    metrics_path: /probe
+
+    file_sd_configs:
+      - files:
+          - /blackbox-targets-ipv6/*.json
+        # Attempt to re-read files every five minutes.
+        refresh_interval: 5m
+
+    # This relabel config is necessary. The relabel config redefines the address
+    # to scrape and sets the correct parameters to pass to the scrape target.
+    #
+    # While not as direct as other configs, this approach allows us to specify a
+    # dynamic list of targets for a static blackbox exporter. This is also the
+    # supported configuration: https://github.com/prometheus/blackbox_exporter
+    relabel_configs:
+
+      # The default __address__ value is a target host from the config file.
+      # Here, we set (i.e. "replace") a request parameter "target" equal to the
+      # host value.
+      - source_labels: [__address__]
+        regex: (.*)
+        target_label: __param_target
+        replacement: ${1}
+
+      # Use the "module" label defined in the input file as the module name for
+      # the blackbox exporter request.
+      - source_labels: [module]
+        regex: (.*)
+        target_label: __param_module
+        replacement: ${1}
+
+      # Use the target parameter defined above and use it to define the
+      # "instance" label.
+      - source_labels: [__param_target]
+        regex: (.*)
+        target_label: instance
+        replacement: ${1}
+
+      # Since __address__ is the target that prometheus will contact,
+      # unconditionally reset the __address__ label to the blackbox exporter
+      # address.
+      - source_labels: [__blackbox_port]
+        regex: (.*)
+        target_label: __address__
+        replacement: blackbox-exporter-ipv6.{{PROJECT}}.measurementlab.net:${1}
+
+
   # Scrape config for the snmp_exporter. There is one snmp_exporter service
   # running in a Docker container on a GCE VM in each GCP project.
   #

diff --git a/deploy_bbe_config.sh b/deploy_bbe_config.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Deploys the blackbox_exporter config to an external (e.g., Linode) VM which
+# will perform IPv6 probes, since GCP doesn't currently support IPv6.
+#
+#  Example usage:
+#    ./deploy_bbe_config.sh mlab-sandbox LINODE_PRIVATE_KEY_ipv6_monitoring
+
+set -e
+set -u
+set -x
+
+BASE_DIR=$( dirname ${BASH_SOURCE[0]} )
+USAGE="Usage: $0 <project> <keyname>"
+PROJECT=${1:?Please provide project name: $USAGE}
+KEYNAME=${2:?Please provide an authentication key name: $USAGE}
+BBE_CONFIG="${BASE_DIR}/config/federation/blackbox/config.yml"
+LINODE_DOMAIN="blackbox-exporter-ipv6.${PROJECT}.measurementlab.net"
+LINODE_USER="mlab"
+LOCAL_KEY_FILE="id_rsa_linode"
+SSH_OPTS="-i $LOCAL_KEY_FILE -o IdentitiesOnly=yes -o StrictHostKeyChecking=no"
+
+# Extract the SSH key from the configured Travis environment variable. The key
+# is base64 encoded to avoid the need for shell escaping and newlines. Set the
+# mode of the file appropriately, as SSH will refuse to use it if the
+# permissions are not strict enough.
+echo "${!KEYNAME}" | base64 -d > $LOCAL_KEY_FILE
+chmod 600 $LOCAL_KEY_FILE
+
+# Copy blackbox_exporter config file to the Linode VM.
+scp $SSH_OPTS $BBE_CONFIG $LINODE_USER@$LINODE_DOMAIN:blackbox-exporter-config-$PROJECT.yml
+
+# HUP the blackbox_exporter so it reads the new config.
+ssh $SSH_OPTS $LINODE_USER@$LINODE_DOMAIN "docker exec ${PROJECT} kill -HUP 1"
diff --git a/k8s/prometheus-federation/deployments/prometheus.yml b/k8s/prometheus-federation/deployments/prometheus.yml
@@ -92,6 +92,10 @@ spec:
         - mountPath: /blackbox-targets
           name: prometheus-storage
           subPath: blackbox-targets
+        # /blackbox-targets-ipv6 should contain blackbox IPv6 target files.
+        - mountPath: /blackbox-targets-ipv6
+          name: prometheus-storage
+          subPath: blackbox-targets-ipv6
         # /aeflex-targets should contain AppEngine target config files.
         - mountPath: /aeflex-targets
           name: prometheus-storage
@@ -143,6 +147,8 @@ spec:
                 "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/legacy-targets/nodeexporter.json",
                 "--http-target=/targets/blackbox-targets/ssh806.json",
                 "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/blackbox-targets/ssh806.json",
+                "--http-target=/targets/blackbox-targets-ipv6/ssh806_ipv6.json",
+                "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/blackbox-targets-ipv6/ssh806_ipv6.json",
                 "--http-target=/targets/blackbox-targets/rsyncd.json",
                 "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/blackbox-targets/rsyncd.json",
                 "--http-target=/targets/snmp-targets/snmpexporter.json",
@@ -159,8 +165,12 @@ spec:
                 "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/legacy-targets/ndt_inotify.json",
                 "--http-target=/targets/blackbox-targets/mobiperf.json",
                 "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/blackbox-targets/mobiperf.json",
+                "--http-target=/targets/blackbox-targets-ipv6/mobiperf_ipv6.json",
+                "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/blackbox-targets-ipv6/mobiperf_ipv6.json",
                 "--http-target=/targets/blackbox-targets/neubot.json",
                 "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/blackbox-targets/neubot.json",
+                "--http-target=/targets/blackbox-targets-ipv6/neubot_ipv6.json",
+                "--http-source=https://storage.googleapis.com/operator-{{GCLOUD_PROJECT}}/prometheus/blackbox-targets-ipv6/neubot_ipv6.json",
                 "--project={{GCLOUD_PROJECT}}"]
         resources:
           requests: