feat(k8s): allow addition of response header "X-Robots-Tag: noindex, …

…nofollow" (#2657)

* feat(k8s): allow addition of response header "X-Robots-Tag: noindex, nofollow"

Adding custom response headers is a convenient way to tell search engines what to index

X-Robots-Tag is supported wherever robots.txt is: https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#xrobotstag

By default, indexing is now _off_, unless .Values.noindex=false is set.

For relevant traefik docs, see https://doc.traefik.io/traefik/middlewares/http/headers/#adding-and-removing-headers

This PR allows pathoplexus/pathoplexus#48 to easily set noindex on all previews.

We probably _never_ want to index authentication pages, but this can be left for another PR.

* By default, don't index

* fix inconsistency typo

* Rename to `robotsNoIndexHeader`, make false by default

kubernetes/loculus/templates/ingressroute.yaml

@@ -23,6 +23,17 @@ spec:
   basicAuth:
     secret: basicauth
 {{ end  }}
+{{ if $.Values.robotsNoindexHeader }}
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: Middleware
+metadata:
+  name: noindex-robots-header
+spec:
+  headers:
+    customResponseHeaders:
+      X-Robots-Tag: "noindex, nofollow"
+{{ end }}
 ---
 apiVersion: traefik.containo.us/v1alpha1
 kind: Middleware
@@ -41,6 +52,9 @@ spec:
 {{- if $.Values.enforceHTTPS }}
 {{- $middlewareList = append $middlewareList (printf "%s-redirect-middleware@kubernetescrd" $.Release.Namespace) }}
 {{- end }}
+{{ if $.Values.robotsNoindexHeader }}
+{{ $middlewareList = append $middlewareList (printf "%s-noindex-robots-header@kubernetescrd" $.Release.Namespace) }}
+{{ end }}
 
 {{ $middlewareListForWebsite := $middlewareList }}
 

kubernetes/loculus/values.yaml

@@ -1,4 +1,5 @@
 environment: server
+robotsNoindexHeader: false
 crossRef:
   DOIPrefix: "placeholder"
   endpoint: "https://test.crossref.org"

kubernetes/loculus/values_preview_server.yaml

@@ -30,3 +30,4 @@ secrets:
       github-pat: AgBa5wjYBw3GWXihWdoedx3oEbR5z808WoeIGyk5aAZPQ1MTMbWm4ZMdvJUZVPlkRacQXSt+A2pniYMjzl0bqHkc5guVQOoHu6jZgtyfaOyqT4Ergz1+L9VUzyn6E1NwqV2BtdKQTN7A1VvtLw0BLVG0E01SUcHeoU9MWlF+9BWFtXWWYxWL/sTLKoz0tnj8yu6LLHMMYeCnABAoZwZbJ3kf1UVp3FwcJbuSqsrUTPaeS6z5g90jL/zwJ150VUh9kInHs+kwV1imZp06ZUm3gpdY1f/EQZnxHTPUoaYLOo91cEhG7bSD9KQz1B6m07VzjIlnrwduPUCFQs1gk1hU7gO+6MYlMqCIb8NUQbuX5d5cHiENJm9DXAJdd8oNW02JQ371gulVYvXagAdrYgYv75Mr3YjejHZeinZrJ/ZxeZI0+fL4SLkh+77RyuN1R6ab1vGBoUG+LgHsUEBT7mw/wXtrzRmNBwxNBXSfTrrKM+EzM9KUsrSe490sK3Rf4TExjynL/9biS9ZFw601tdBh8luO3xFhCr8Bn4q7sWnKNmvSu+gMZklyhR65Hf0LHTxUdDcK8WZKFyds6foHTAv9kX4+NK7upVfAdjx9Tpf0ONad3nSboi/+1vYoGtAZcL6BZoNL+1Xu2WAlVa+WollJSV2k4KeTEswE1vmPAhoapyxpry9Mz+8CGkohvSPjHGdfXSNSW7K9zKCCubqzHV2+EsnaDWJ9+cwghAQFOpKdbJuHLwalMpAkTztRjr5wHfD6KSo1apCHZl84S0B2u8floEzU1XRiW8zq10m8cb3/xmdteNL3FNMlZlEieD2nsiU=
 reduceResourceRequest: true
 previewDocs: true
+robotsNoindexHeader: true