How to make data persistent

[pmona]

I have been looking for a way to make my grafana and other data persistent across restarts. I deployed using the example:

kubectl apply --server-side -f manifests/setup
until kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done
kubectl apply -f manifests/

Everything is working as expected, but I would like the data to be persistent.
Can someone explain to me how I can implement such changes? From my searches, I have not found anything -- or at least nothing I understand.

[pmona]

I finally got this working. Assuming you have a NFS share working already, next it requires you have a storage class setup. In my deployment, I use the k8s-mgmt namespace instead of the default monitoring.
My storage class name is kube-prometheus-storage.

For setting up the storage class I basically followed this tutorial:
https://www.youtube.com/watch?v=DF3v2P8ENEg&t=1359s

My final config:

---
kind: ServiceAccount
apiVersion: v1
metadata:
  name: kube-prometheus-storage
  namespace: k8s-mgmt
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: kube-prometheus-storage-runner
  namespace: k8s-mgmt
rules:
  - apiGroups: [""]
	resources: ["persistentvolumes"]
	verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
	resources: ["persistentvolumeclaims"]
	verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
	resources: ["storageclasses"]
	verbs: ["get", "list", "watch"]
  - apiGroups: [""]
	resources: ["events"]
	verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-kube-prometheus-storage
  namespace: k8s-mgmt
subjects:
  - kind: ServiceAccount
	name: kube-prometheus-storage
	namespace: k8s-mgmt
roleRef:
  kind: ClusterRole
  name: kube-prometheus-storage-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-kube-prometheus-storage
  namespace: k8s-mgmt
rules:
  - apiGroups: [""]
	resources: ["endpoints"]
	verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-kube-prometheus-storage
  namespace: k8s-mgmt
subjects:
  - kind: ServiceAccount
	name: kube-prometheus-storage
	# replace with namespace where provisioner is deployed
	namespace: k8s-mgmt
roleRef:
  kind: Role
  name: leader-locking-kube-prometheus-storage
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: kube-prometheus-storage
  namespace: k8s-mgmt
provisioner: example.com/nfs
parameters:
  reclaimPolicy: Retain
---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: kube-prometheus-storage
  namespace: k8s-mgmt
spec:
  replicas: 1
  strategy:
	type: Recreate
  selector:
	matchLabels:
	  app: kube-prometheus-storage
  template:
	metadata:
	  labels:
		app: kube-prometheus-storage
	spec:
	  serviceAccountName: kube-prometheus-storage
	  containers:
		- name: kube-prometheus-storage
		  image: gcr.io/k8s-staging-sig-storage/nfs-subdir-external-provisioner:v4.0.1
		  volumeMounts:
			- name: kube-prometheus-storage
			  mountPath: /persistentvolumes
		  env:
			- name: PROVISIONER_NAME
			  value: example.com/nfs
			- name: NFS_SERVER
			  value: synology01.prdsnvlab.mydomain.net
			- name: NFS_PATH
			  value: /volume1/DOPS/kube-prometheus-storage
	  volumes:
		- name: kube-prometheus-storage
		  nfs:
			server: synology01.prdsnvlab.mydomain.net
			path: /volume1/DOPS/kube-prometheus-storage

My jsonnet file for building the manifest:

local kp =
  (import 'kube-prometheus/main.libsonnet') +
  // Uncomment the following imports to enable its patches
  // (import 'kube-prometheus/addons/anti-affinity.libsonnet') +
  // (import 'kube-prometheus/addons/managed-cluster.libsonnet') +
  (import 'kube-prometheus/addons/node-ports.libsonnet') +
  // (import 'kube-prometheus/addons/static-etcd.libsonnet') +
  // (import 'kube-prometheus/addons/custom-metrics.libsonnet') +
  // (import 'kube-prometheus/addons/external-metrics.libsonnet') +
  // (import 'kube-prometheus/addons/pyrra.libsonnet') +
  {
	values+:: {
	  common+: {
		namespace: 'k8s-mgmt',
	  },
	},
	
	   prometheus+:: {
	  prometheus+: {
		spec+: {  // https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#prometheusspec
		  // If a value isn't specified for 'retention', then by default the '--storage.tsdb.retention=24h' arg will be passed to prometheus by prometheus-operator.
		  // The possible values for a prometheus <duration> are:
		  //  * https://github.com/prometheus/common/blob/c7de230/model/time.go#L178 specifies "^([0-9]+)(y|w|d|h|m|s|ms)$" (years weeks days hours minutes seconds milliseconds)
		  retention: '365d',

		  // Reference info: https://github.com/coreos/prometheus-operator/blob/master/Documentation/user-guides/storage.md
		  // By default (if the following 'storage.volumeClaimTemplate' isn't created), prometheus will be created with an EmptyDir for the 'prometheus-k8s-db' volume (for the prom tsdb).
		  // This 'storage.volumeClaimTemplate' causes the following to be automatically created (via dynamic provisioning) for each prometheus pod:
		  //  * PersistentVolumeClaim (and a corresponding PersistentVolume)
		  //  * the actual volume (per the StorageClassName specified below)
		  storage: {  // https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#storagespec
			volumeClaimTemplate: {  // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#persistentvolumeclaim-v1-core (defines variable named 'spec' of type 'PersistentVolumeClaimSpec')
			  apiVersion: 'v1',
			  kind: 'PersistentVolumeClaim',
			  spec: {
				accessModes: ['ReadWriteOnce'],
				// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#resourcerequirements-v1-core (defines 'requests'),
				// and https://kubernetes.io/docs/concepts/policy/resource-quotas/#storage-resource-quota (defines 'requests.storage')
				resources: { requests: { storage: '10Gi' } },
				// A StorageClass of the following name (which can be seen via `kubectl get storageclass` from a node in the given K8s cluster) must exist prior to kube-prometheus being deployed.
				storageClassName: 'kube-prometheus-storage',
				// The following 'selector' is only needed if you're using manual storage provisioning (https://github.com/coreos/prometheus-operator/blob/master/Documentation/user-guides/storage.md#manual-storage-provisioning).
				// And note that this is not supported/allowed by AWS - uncommenting the following 'selector' line (when deploying kube-prometheus to a K8s cluster in AWS) will cause the pvc to be stuck in the Pending status and have the following error:
				//  * 'Failed to provision volume with StorageClass "ssd": claim.Spec.Selector is not supported for dynamic provisioning on AWS'
				// selector: { matchLabels: {} },
			  },
			},
		  },  // storage
		},  // spec
	  },  // prometheus
	},  // prometheus
	
  };

{ 'setup/0namespace-namespace': kp.kubePrometheus.namespace } +
{
  ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name]
  for name in std.filter((function(name) name != 'serviceMonitor' && name != 'prometheusRule'), std.objectFields(kp.prometheusOperator))
} +
// { 'setup/pyrra-slo-CustomResourceDefinition': kp.pyrra.crd } +
// serviceMonitor and prometheusRule are separated so that they can be created after the CRDs are ready
{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } +
{ 'prometheus-operator-prometheusRule': kp.prometheusOperator.prometheusRule } +
{ 'kube-prometheus-prometheusRule': kp.kubePrometheus.prometheusRule } +
{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
{ ['blackbox-exporter-' + name]: kp.blackboxExporter[name] for name in std.objectFields(kp.blackboxExporter) } +
{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } +
// { ['pyrra-' + name]: kp.pyrra[name] for name in std.objectFields(kp.pyrra) if name != 'crd' } +
{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
{ ['kubernetes-' + name]: kp.kubernetesControlPlane[name] for name in std.objectFields(kp.kubernetesControlPlane) }
{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) }