Add k8s cluster name to all telemetry

collector/config-k8s/.gitignore

@@ -0,0 +1 @@
+mid.yaml

collector/config-k8s/values-cluster.yaml

@@ -11,6 +11,12 @@ extraEnvs:
         name: servicenow-cloudobs-token
         key: token
         optional: true
+  - name: K8S_CLUSTER_NAME
+    valueFrom:
+      configMapKeyRef:
+        name: cluster-info
+        key: name
+        optional: true
   - name: SERVICENOW_EVENTS_URL
     valueFrom:
       configMapKeyRef:
@@ -41,6 +47,9 @@ extraEnvs:
         name: servicenow-events
         key: .password
         optional: true
+  - name: OTEL_RESOURCE_ATTRIBUTES
+    value: "k8s.cluster.name=$(K8S_CLUSTER_NAME)"
+
 # We only want one of these collectors - any more and we'd produce duplicate data
 replicaCount: 1
 
@@ -145,6 +154,10 @@ config:
       detectors: [env, aks]
       timeout: 2s
       override: false
+    resourcedetection/env:
+      detectors: [env]
+      timeout: 2s
+      override: false
     batch:
       send_batch_size: 1000
       send_batch_max_size: 1500
@@ -220,13 +233,13 @@ config:
       metrics/collector-monitoring:
         receivers: [prometheus]
         processors: [k8sattributes, concurrentbatch]
-        exporters: [debug]
+        exporters: [debug, otlp/cloudobs]
       metrics:
-        processors: [k8sattributes, concurrentbatch]
-        exporters: [debug, servicenow/metrics]
+        processors: [k8sattributes, resourcedetection/env, concurrentbatch]
+        exporters: [debug, servicenow/metrics, otlp/cloudobs]
       logs:
-        processors: [k8sattributes, transform/events, concurrentbatch]
-        exporters: [debug, servicenow/events]
+        processors: [k8sattributes, resourcedetection/env, transform/events, concurrentbatch]
+        exporters: [debug, servicenow/events, otlp/cloudobs]
 
 ports:
   jaeger-compact:

collector/config-k8s/values-node.yaml

@@ -22,6 +22,12 @@ extraEnvs:
       secretKeyRef:
         name: servicenow-cloudobs-token
         key: token
+  - name: K8S_CLUSTER_NAME
+    valueFrom:
+      configMapKeyRef:
+        name: cluster-info
+        key: name
+        optional: true
   - name: K8S_NODE_NAME
     valueFrom:
       fieldRef:
@@ -46,7 +52,7 @@ extraEnvs:
         apiVersion: v1
         fieldPath: metadata.uid
   - name: OTEL_RESOURCE_ATTRIBUTES
-    value: "k8s.node.name=$(K8S_NODE_NAME)"
+    value: "k8s.node.name=$(K8S_NODE_NAME),k8s.cluster.name=$(K8S_CLUSTER_NAME)"
 
 presets:
   # enables the k8sattributesprocessor and adds it to the traces, metrics, and logs pipelines
@@ -218,16 +224,16 @@ config:
     pipelines:
       traces:
         receivers: [otlp]
-        processors: [k8sattributes, concurrentbatch]
+        processors: [k8sattributes, resourcedetection/env, concurrentbatch]
         exporters: [debug, otlp/cloudobs]
       metrics:
         receivers: [otlp, prometheus, hostmetrics, kubeletstats]
-        processors: [k8sattributes, concurrentbatch]
+        processors: [k8sattributes, resourcedetection/env, concurrentbatch]
         exporters: [debug, otlp/cloudobs]
       logs:
         receivers: [otlp, filelog]
-        processors: [k8sattributes, concurrentbatch]
-        exporters: [debug, otelarrow/cloudobs]
+        processors: [k8sattributes, resourcedetection/env, concurrentbatch]
+        exporters: [debug, otlp/cloudobs]
 
 ports:
   jaeger-compact:

docs/monitor-kubernetes.md

@@ -90,12 +90,16 @@ The MID server should appear on your instance after a few minutes. After it does
 
 #### 5. Deploy ServiceNow Collector for Cluster Monitoring and CNO for Visibility
 
+ServiceNow CMDB generally requires a Kubernetes cluster name to be set. Since this varies depending on the type of cluster, set the name manually in a configuration map:
+
+```sh
+    kubectl create configmap cluster-info -n servicenow --from-literal=name=YOUR_CLUSTER_NAME
+```
+
 You're now ready to deploy a collector to your cluster to collect cluster-level metrics and events. To preview the generated manifest before deploying, add the `--dry-run` option to the below command:
 
 ```sh
-helm upgrade otel-collector-cluster open-telemetry/opentelemetry-collector \ 
-    --install --namespace servicenow \
-    --values https://raw.githubusercontent.com/lightstep/sn-collector/main/collector/config-k8s/values-cluster.yaml
+helm upgrade otel-collector-cluster open-telemetry/opentelemetry-collector --install --namespace servicenow --values https://raw.githubusercontent.com/lightstep/sn-collector/main/collector/config-k8s/values-cluster.yaml
 ```
 
 Next, install CNO for visibility. Additional install instructions for CNO are on the ServiceNow documentation [portal](https://docs.servicenow.com/bundle/washingtondc-it-operations-management/page/product/cloud-native-operations-visibility/task/cnov-deploy-install.html). By sending `Y` you accept the terms and conditions of ServiceNow CNO.
@@ -117,10 +121,7 @@ kubectl get pods -n servicenow
 Next, deploy collectors to each Kubernetes host to get workload metrics (via Kubelet). To preview the generated manifest before deploying, add the `--dry-run` option to the below command:
 
 ```sh
-helm upgrade otel-collector \
-    open-telemetry/opentelemetry-collector \
-    --install --namespace servicenow \
-    --values https://raw.githubusercontent.com/lightstep/sn-collector/main/collector/config-k8s/values-node.yaml
+helm upgrade otel-collector open-telemetry/opentelemetry-collector --install --namespace servicenow --values https://raw.githubusercontent.com/lightstep/sn-collector/main/collector/config-k8s/values-node.yaml
 ```
 
 #### 6. See events in ServiceNow