deploy datadog to gke clusters

infra/gcp/terraform/kubernetes-public/iam.tf

@@ -23,6 +23,9 @@ module "iam" {
   mode = "authoritative"
 
   bindings = {
+    "roles/container.admin" = [
+      "serviceAccount:argocd@k8s-infra-prow.iam.gserviceaccount.com",
+    ]
     "roles/secretmanager.secretAccessor" = [
       "serviceAccount:kubernetes-external-secrets@kubernetes-public.iam.gserviceaccount.com",
       "principal://iam.googleapis.com/projects/16065310909/locations/global/workloadIdentityPools/k8s-infra-prow.svc.id.goog/subject/ns/external-secrets/sa/external-secrets",

kubernetes/eks-prow-kops/datadog/kustomization.yaml

@@ -6,8 +6,8 @@ helmCharts:
   - name: datadog
     repo: https://helm.datadoghq.com
     releaseName: datadog
-    version: 3.118.0
-    kubeVersion: "1.29"
+    version: 3.135.4
+    kubeVersion: "1.30"
     valuesFile: values.yaml
 
 resources:

kubernetes/eks-prow-kops/datadog/values.yaml

@@ -24,6 +24,9 @@ datadog:
       uncompressedLayersSupport: true
     host:
       enabled: true
+  apm:
+    instrumentation:
+      skipKPITelemetry: true # https://github.com/DataDog/helm-charts/issues/1395
 clusterAgent:
   tokenExistingSecret: datadog-secret
 agents:

kubernetes/gke-aaa/datadog/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: datadog
+
+helmCharts:
+  - name: datadog
+    repo: https://helm.datadoghq.com
+    releaseName: datadog
+    version: 3.118.0
+    valuesFile: values.yaml
+
+resources:
+  - secrets.yaml

kubernetes/gke-aaa/datadog/secrets.yaml

@@ -0,0 +1,11 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: datadog-secret
+spec:
+  dataFrom:
+    - extract:
+        key: datadog-secrets
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: k8s-infra-prow-build

kubernetes/gke-aaa/datadog/values.yaml

@@ -0,0 +1,40 @@
+registry: gcr.io/datadoghq
+datadog:
+  apiKeyExistingSecret: datadog-secret
+  appKeyExistingSecret: datadog-secret
+  site: us5.datadoghq.com
+  clusterName: aaa
+  logs:
+    enabled: true
+    containerCollectAll: true
+  prometheusScrape:
+    enabled: true
+    serviceEndpoints: true
+  # COS specific https://docs.datadoghq.com/containers/kubernetes/distributions?tab=helm#GKE
+  systemProbe:
+    enableDefaultKernelHeadersPaths: false
+  kubeStateMetricsCore:
+    enabled: true
+  networkMonitoring:
+    enabled: true
+  processAgent:
+    enabled: true
+    processCollection: true
+  sbom:
+    enabled: true
+    containerImage:
+      enabled: true
+      uncompressedLayersSupport: true
+    host:
+      enabled: true
+  apm:
+    instrumentation:
+      skipKPITelemetry: true # https://github.com/DataDog/helm-charts/issues/1395
+clusterAgent:
+  tokenExistingSecret: datadog-secret
+agents:
+  tolerations: # datadog supports arm64
+    - key: kubernetes.io/arch
+      operator: Equal
+      value: arm64
+      effect: NoSchedule

kubernetes/gke-aaa/helm/external-secrets.yaml

@@ -0,0 +1,17 @@
+extraObjects:
+  - apiVersion: external-secrets.io/v1beta1
+    kind: ClusterSecretStore
+    metadata:
+      name: kubernetes-public
+    spec:
+      provider:
+        gcpsm:
+          projectID: kubernetes-public
+  - apiVersion: external-secrets.io/v1beta1
+    kind: ClusterSecretStore
+    metadata:
+      name: k8s-infra-prow-build
+    spec:
+      provider:
+        gcpsm:
+          projectID: k8s-infra-prow-build

kubernetes/gke-prow-build-trusted/datadog/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: datadog
+
+helmCharts:
+  - name: datadog
+    repo: https://helm.datadoghq.com
+    releaseName: datadog
+    version: 3.135.4
+    valuesFile: values.yaml
+
+resources:
+  - secrets.yaml

kubernetes/gke-prow-build-trusted/datadog/secrets.yaml

@@ -0,0 +1,11 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: datadog-secret
+spec:
+  dataFrom:
+    - extract:
+        key: datadog-secrets
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: k8s-infra-prow-build

kubernetes/gke-prow-build-trusted/datadog/values.yaml

@@ -0,0 +1,37 @@
+registry: gcr.io/datadoghq
+datadog:
+  apiKeyExistingSecret: datadog-secret
+  appKeyExistingSecret: datadog-secret
+  site: us5.datadoghq.com
+  clusterName: k8s-infra-prow-build-trusted
+  logs:
+    enabled: true
+    containerCollectAll: true
+  prometheusScrape:
+    enabled: true
+    serviceEndpoints: true
+  kubeStateMetricsCore:
+    enabled: true
+  networkMonitoring:
+    enabled: true
+  processAgent:
+    enabled: true
+    processCollection: true
+  sbom:
+    enabled: true
+    containerImage:
+      enabled: true
+      uncompressedLayersSupport: true
+    host:
+      enabled: true
+  apm:
+    instrumentation:
+      skipKPITelemetry: true # https://github.com/DataDog/helm-charts/issues/1395
+clusterAgent:
+  tokenExistingSecret: datadog-secret
+agents:
+  tolerations: # datadog supports arm64
+    - key: kubernetes.io/arch
+      operator: Equal
+      value: arm64
+      effect: NoSchedule

kubernetes/gke-prow-build-trusted/helm/external-secrets.yaml

@@ -7,6 +7,14 @@ extraObjects:
       provider:
         gcpsm:
           projectID: k8s-infra-prow-build-trusted
+  - apiVersion: external-secrets.io/v1beta1
+    kind: ClusterSecretStore
+    metadata:
+      name: k8s-infra-prow-build
+    spec:
+      provider:
+        gcpsm:
+          projectID: k8s-infra-prow-build
   - apiVersion: monitoring.googleapis.com/v1
     kind: PodMonitoring
     metadata:

kubernetes/gke-prow-build/datadog/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: datadog
+
+helmCharts:
+  - name: datadog
+    repo: https://helm.datadoghq.com
+    releaseName: datadog
+    version: 3.135.4
+    valuesFile: values.yaml
+
+resources:
+  - secrets.yaml

kubernetes/gke-prow-build/datadog/secrets.yaml

@@ -0,0 +1,11 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: datadog-secret
+spec:
+  dataFrom:
+    - extract:
+        key: datadog-secrets
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: k8s-infra-prow-build

kubernetes/gke-prow-build/datadog/values.yaml

@@ -0,0 +1,40 @@
+registry: gcr.io/datadoghq
+datadog:
+  apiKeyExistingSecret: datadog-secret
+  appKeyExistingSecret: datadog-secret
+  site: us5.datadoghq.com
+  clusterName: k8s-infra-prow-build
+  logs:
+    enabled: true
+    containerCollectAll: true
+  prometheusScrape:
+    enabled: true
+    serviceEndpoints: true
+  # COS specific https://docs.datadoghq.com/containers/kubernetes/distributions?tab=helm#GKE
+  systemProbe:
+    enableDefaultKernelHeadersPaths: false
+  kubeStateMetricsCore:
+    enabled: true
+  networkMonitoring:
+    enabled: true
+  processAgent:
+    enabled: true
+    processCollection: true
+  sbom:
+    enabled: true
+    containerImage:
+      enabled: true
+      uncompressedLayersSupport: true
+    host:
+      enabled: true
+  apm:
+    instrumentation:
+      skipKPITelemetry: true # https://github.com/DataDog/helm-charts/issues/1395
+clusterAgent:
+  tokenExistingSecret: datadog-secret
+agents:
+  tolerations: # datadog supports arm64
+    - key: kubernetes.io/arch
+      operator: Equal
+      value: arm64
+      effect: NoSchedule

kubernetes/gke-prow/datadog/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: datadog
+
+helmCharts:
+  - name: datadog
+    repo: https://helm.datadoghq.com
+    releaseName: datadog
+    version: 3.135.4
+    valuesFile: values.yaml
+
+resources:
+  - secrets.yaml

kubernetes/gke-prow/datadog/secrets.yaml

@@ -0,0 +1,11 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: datadog-secret
+spec:
+  dataFrom:
+    - extract:
+        key: datadog-secrets
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: k8s-infra-prow-build

kubernetes/gke-prow/datadog/values.yaml

@@ -0,0 +1,40 @@
+registry: gcr.io/datadoghq
+datadog:
+  apiKeyExistingSecret: datadog-secret
+  appKeyExistingSecret: datadog-secret
+  site: us5.datadoghq.com
+  clusterName: k8s-infra-prow
+  logs:
+    enabled: true
+    containerCollectAll: true
+  prometheusScrape:
+    enabled: true
+    serviceEndpoints: true
+  # COS specific https://docs.datadoghq.com/containers/kubernetes/distributions?tab=helm#GKE
+  systemProbe:
+    enableDefaultKernelHeadersPaths: false
+  kubeStateMetricsCore:
+    enabled: true
+  networkMonitoring:
+    enabled: true
+  processAgent:
+    enabled: true
+    processCollection: true
+  sbom:
+    enabled: true
+    containerImage:
+      enabled: true
+      uncompressedLayersSupport: true
+    host:
+      enabled: true
+  apm:
+    instrumentation:
+      skipKPITelemetry: true # https://github.com/DataDog/helm-charts/issues/1395
+clusterAgent:
+  tokenExistingSecret: datadog-secret
+agents:
+  tolerations: # datadog supports arm64
+    - key: kubernetes.io/arch
+      operator: Equal
+      value: arm64
+      effect: NoSchedule

kubernetes/gke-prow/helm/external-secrets.yaml

@@ -1,29 +1,37 @@
 extraObjects:
- -  apiVersion: external-secrets.io/v1beta1
+  - apiVersion: external-secrets.io/v1beta1
     kind: ClusterSecretStore
     metadata:
       name: k8s-infra-prow
     spec:
       provider:
         gcpsm:
           projectID: k8s-infra-prow
- -  apiVersion: external-secrets.io/v1beta1
+  - apiVersion: external-secrets.io/v1beta1
+    kind: ClusterSecretStore
+    metadata:
+      name: k8s-infra-prow-build
+    spec:
+      provider:
+        gcpsm:
+          projectID: k8s-infra-prow-build
+  - apiVersion: external-secrets.io/v1beta1
     kind: ClusterSecretStore
     metadata:
       name: kubernetes-public
     spec:
       provider:
         gcpsm:
           projectID: kubernetes-public
- -  apiVersion: external-secrets.io/v1beta1
+  - apiVersion: external-secrets.io/v1beta1
     kind: ClusterSecretStore
     metadata:
       name: k8s-infra-prow-build-trusted
     spec:
       provider:
         gcpsm:
           projectID: k8s-infra-prow-build-trusted
- -  apiVersion: monitoring.googleapis.com/v1
+  - apiVersion: monitoring.googleapis.com/v1
     kind: PodMonitoring
     metadata:
       labels:
@@ -34,5 +42,5 @@ extraObjects:
         matchLabels:
           app.kubernetes.io/name: external-secrets
       endpoints:
-      - port: metrics
-        interval: 30s
+        - port: metrics
+          interval: 30s