From 4d907e4223e028b1beb1d9b4902de6febe4f99ed Mon Sep 17 00:00:00 2001 From: Lucas Lopes Date: Wed, 8 Nov 2023 15:26:44 -0300 Subject: [PATCH 1/5] Emptying release.yaml after 10th batch release of 2.8 --- release.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/release.yaml b/release.yaml index 815e787b52..8b13789179 100644 --- a/release.yaml +++ b/release.yaml @@ -1,6 +1 @@ -fleet: - - 103.1.0+up0.9.0 -fleet-agent: - - 103.1.0+up0.9.0 -fleet-crd: - - 103.1.0+up0.9.0 + From 2c3bdc7bb9602244f27b2af44f29e31085aa9547 Mon Sep 17 00:00:00 2001 From: James Lu Date: Fri, 3 Nov 2023 14:19:19 +0800 Subject: [PATCH 2/5] make prepare/patch: release longhorn v1.4.4 into Rancher 2.8 Longhorn 6894 Signed-off-by: James Lu --- .../longhorn-1.4/charts/Chart.yaml | 11 + .../longhorn-1.4/charts/README.md | 2 + .../charts/templates/_helpers.tpl | 66 + .../longhorn-1.4/charts/templates/crds.yaml | 3469 +++++++++++++++++ .../generated-changes/exclude/questions.yaml | 24 +- .../generated-changes/exclude/values.yaml | 14 +- .../generated-changes/patch/Chart.yaml.patch | 4 +- .../longhorn-crd/longhorn-1.4/package.yaml | 6 +- .../longhorn/longhorn-1.4/charts/.helmignore | 21 + .../longhorn/longhorn-1.4/charts/Chart.yaml | 40 + .../longhorn/longhorn-1.4/charts/README.md | 49 + .../longhorn-1.4/charts/app-readme.md | 27 + .../longhorn-1.4/charts/questions.yaml | 850 ++++ .../longhorn-1.4/charts/templates/NOTES.txt | 5 + .../charts/templates/_helpers.tpl | 66 + .../charts/templates/clusterrole.yaml | 60 + .../charts/templates/clusterrolebinding.yaml | 27 + .../charts/templates/daemonset-sa.yaml | 147 + .../charts/templates/default-setting.yaml | 80 + .../charts/templates/deployment-driver.yaml | 118 + .../deployment-recovery-backend.yaml | 83 + .../charts/templates/deployment-ui.yaml | 114 + .../charts/templates/deployment-webhook.yaml | 166 + .../charts/templates/ingress.yaml | 48 + .../charts/templates/postupgrade-job.yaml | 58 + .../longhorn-1.4/charts/templates/psp.yaml | 66 + .../charts/templates/registry-secret.yaml | 13 + .../charts/templates/serviceaccount.yaml | 21 + .../charts/templates/services.yaml | 74 + .../charts/templates/storageclass.yaml | 44 + .../charts/templates/tls-secrets.yaml | 16 + .../charts/templates/uninstall-job.yaml | 59 + .../charts/templates/userroles.yaml | 50 + .../templates/validate-install-crd.yaml | 33 + .../templates/validate-psp-install.yaml | 7 + .../longhorn/longhorn-1.4/charts/values.yaml | 333 ++ .../generated-changes/patch/Chart.yaml.patch | 6 +- .../generated-changes/patch/values.yaml.patch | 14 +- packages/longhorn/longhorn-1.4/package.yaml | 6 +- 39 files changed, 6260 insertions(+), 37 deletions(-) create mode 100755 packages/longhorn-crd/longhorn-1.4/charts/Chart.yaml create mode 100755 packages/longhorn-crd/longhorn-1.4/charts/README.md create mode 100755 packages/longhorn-crd/longhorn-1.4/charts/templates/_helpers.tpl create mode 100755 packages/longhorn-crd/longhorn-1.4/charts/templates/crds.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/.helmignore create mode 100755 packages/longhorn/longhorn-1.4/charts/Chart.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/README.md create mode 100755 packages/longhorn/longhorn-1.4/charts/app-readme.md create mode 100755 packages/longhorn/longhorn-1.4/charts/questions.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/NOTES.txt create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/_helpers.tpl create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/clusterrole.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/clusterrolebinding.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/daemonset-sa.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/default-setting.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/deployment-driver.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/deployment-recovery-backend.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/deployment-ui.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/deployment-webhook.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/ingress.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/postupgrade-job.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/psp.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/registry-secret.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/serviceaccount.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/services.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/storageclass.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/tls-secrets.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/uninstall-job.yaml create mode 100644 packages/longhorn/longhorn-1.4/charts/templates/userroles.yaml create mode 100644 packages/longhorn/longhorn-1.4/charts/templates/validate-install-crd.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/templates/validate-psp-install.yaml create mode 100755 packages/longhorn/longhorn-1.4/charts/values.yaml diff --git a/packages/longhorn-crd/longhorn-1.4/charts/Chart.yaml b/packages/longhorn-crd/longhorn-1.4/charts/Chart.yaml new file mode 100755 index 0000000000..aef6266777 --- /dev/null +++ b/packages/longhorn-crd/longhorn-1.4/charts/Chart.yaml @@ -0,0 +1,11 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: longhorn-system + catalog.cattle.io/release-name: longhorn-crd +apiVersion: v1 +appVersion: v1.4.4 +description: Installs the CRDs for longhorn. +name: longhorn-crd +type: application +version: 1.4.4 diff --git a/packages/longhorn-crd/longhorn-1.4/charts/README.md b/packages/longhorn-crd/longhorn-1.4/charts/README.md new file mode 100755 index 0000000000..d9f7f14b33 --- /dev/null +++ b/packages/longhorn-crd/longhorn-1.4/charts/README.md @@ -0,0 +1,2 @@ +# longhorn-crd +A Rancher chart that installs the CRDs used by longhorn. diff --git a/packages/longhorn-crd/longhorn-1.4/charts/templates/_helpers.tpl b/packages/longhorn-crd/longhorn-1.4/charts/templates/_helpers.tpl new file mode 100755 index 0000000000..3fbc2ac02f --- /dev/null +++ b/packages/longhorn-crd/longhorn-1.4/charts/templates/_helpers.tpl @@ -0,0 +1,66 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "longhorn.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "longhorn.fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + + +{{- define "longhorn.managerIP" -}} +{{- $fullname := (include "longhorn.fullname" .) -}} +{{- printf "http://%s-backend:9500" $fullname | trunc 63 | trimSuffix "-" -}} +{{- end -}} + + +{{- define "secret" }} +{{- printf "{\"auths\": {\"%s\": {\"auth\": \"%s\"}}}" .Values.privateRegistry.registryUrl (printf "%s:%s" .Values.privateRegistry.registryUser .Values.privateRegistry.registryPasswd | b64enc) | b64enc }} +{{- end }} + +{{- /* +longhorn.labels generates the standard Helm labels. +*/ -}} +{{- define "longhorn.labels" -}} +app.kubernetes.io/name: {{ template "longhorn.name" . }} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion }} +{{- end -}} + + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{- define "registry_url" -}} +{{- if .Values.privateRegistry.registryUrl -}} +{{- printf "%s/" .Values.privateRegistry.registryUrl -}} +{{- else -}} +{{ include "system_default_registry" . }} +{{- end -}} +{{- end -}} + +{{- /* + define the longhorn release namespace +*/ -}} +{{- define "release_namespace" -}} +{{- if .Values.namespaceOverride -}} +{{- .Values.namespaceOverride -}} +{{- else -}} +{{- .Release.Namespace -}} +{{- end -}} +{{- end -}} diff --git a/packages/longhorn-crd/longhorn-1.4/charts/templates/crds.yaml b/packages/longhorn-crd/longhorn-1.4/charts/templates/crds.yaml new file mode 100755 index 0000000000..3338c6095d --- /dev/null +++ b/packages/longhorn-crd/longhorn-1.4/charts/templates/crds.yaml @@ -0,0 +1,3469 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: backingimagedatasources.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageDataSource + listKind: BackingImageDataSourceList + plural: backingimagedatasources + shortNames: + - lhbids + singular: backingimagedatasource + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the pod used to provision the backing image file from source + jsonPath: .status.currentState + name: State + type: string + - description: The data source type + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The node the backing image file will be prepared on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the backing image file will be prepared on + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImageDataSource is where Longhorn stores backing image data source object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The system generated UUID of the provisioned backing image file + jsonPath: .spec.uuid + name: UUID + type: string + - description: The current state of the pod used to provision the backing image file from source + jsonPath: .status.currentState + name: State + type: string + - description: The data source type + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size + jsonPath: .status.size + name: Size + type: string + - description: The node the backing image file will be prepared on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the backing image file will be prepared on + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageDataSource is where Longhorn stores backing image data source object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageDataSourceSpec defines the desired state of the Longhorn backing image data source + properties: + checksum: + type: string + diskPath: + type: string + diskUUID: + type: string + fileTransferred: + type: boolean + nodeID: + type: string + parameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + type: string + uuid: + type: string + type: object + status: + description: BackingImageDataSourceStatus defines the observed state of the Longhorn backing image data source + properties: + checksum: + type: string + currentState: + type: string + ip: + type: string + message: + type: string + ownerID: + type: string + progress: + type: integer + runningParameters: + additionalProperties: + type: string + nullable: true + type: object + size: + format: int64 + type: integer + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: backingimagemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: BackingImageManager + listKind: BackingImageManagerList + plural: backingimagemanagers + shortNames: + - lhbim + singular: backingimagemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the manager + jsonPath: .status.currentState + name: State + type: string + - description: The image the manager pod will use + jsonPath: .spec.image + name: Image + type: string + - description: The node the manager is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the manager is responsible for + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - description: The disk path the manager is using + jsonPath: .spec.diskPath + name: DiskPath + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImageManager is where Longhorn stores backing image manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The current state of the manager + jsonPath: .status.currentState + name: State + type: string + - description: The image the manager pod will use + jsonPath: .spec.image + name: Image + type: string + - description: The node the manager is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk the manager is responsible for + jsonPath: .spec.diskUUID + name: DiskUUID + type: string + - description: The disk path the manager is using + jsonPath: .spec.diskPath + name: DiskPath + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImageManager is where Longhorn stores backing image manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageManagerSpec defines the desired state of the Longhorn backing image manager + properties: + backingImages: + additionalProperties: + type: string + type: object + diskPath: + type: string + diskUUID: + type: string + image: + type: string + nodeID: + type: string + type: object + status: + description: BackingImageManagerStatus defines the observed state of the Longhorn backing image manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + backingImageFileMap: + additionalProperties: + properties: + currentChecksum: + type: string + directory: + description: 'Deprecated: This field is useless.' + type: string + downloadProgress: + description: 'Deprecated: This field is renamed to `Progress`.' + type: integer + message: + type: string + name: + type: string + progress: + type: integer + senderManagerAddress: + type: string + sendingReference: + type: integer + size: + format: int64 + type: integer + state: + type: string + url: + description: 'Deprecated: This field is useless now. The manager of backing image files doesn''t care if a file is downloaded and how.' + type: string + uuid: + type: string + type: object + nullable: true + type: object + currentState: + type: string + ip: + type: string + ownerID: + type: string + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: backingimages.longhorn.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: {{ include "release_namespace" . }} + path: /v1/webhook/conversion + port: 9443 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: BackingImage + listKind: BackingImageList + plural: backingimages + shortNames: + - lhbi + singular: backingimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backing image name + jsonPath: .spec.image + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: BackingImage is where Longhorn stores backing image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The system generated UUID + jsonPath: .status.uuid + name: UUID + type: string + - description: The source of the backing image file data + jsonPath: .spec.sourceType + name: SourceType + type: string + - description: The backing image file size in each disk + jsonPath: .status.size + name: Size + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: BackingImage is where Longhorn stores backing image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackingImageSpec defines the desired state of the Longhorn backing image + properties: + checksum: + type: string + disks: + additionalProperties: + type: string + type: object + imageURL: + description: 'Deprecated: This kind of info will be included in the related BackingImageDataSource.' + type: string + sourceParameters: + additionalProperties: + type: string + type: object + sourceType: + enum: + - download + - upload + - export-from-volume + type: string + type: object + status: + description: BackingImageStatus defines the observed state of the Longhorn backing image status + properties: + checksum: + type: string + diskDownloadProgressMap: + additionalProperties: + type: integer + description: 'Deprecated: Replaced by field `Progress` in `DiskFileStatusMap`.' + nullable: true + type: object + diskDownloadStateMap: + additionalProperties: + description: BackingImageDownloadState is replaced by BackingImageState. + type: string + description: 'Deprecated: Replaced by field `State` in `DiskFileStatusMap`.' + nullable: true + type: object + diskFileStatusMap: + additionalProperties: + properties: + lastStateTransitionTime: + type: string + message: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + diskLastRefAtMap: + additionalProperties: + type: string + nullable: true + type: object + ownerID: + type: string + size: + format: int64 + type: integer + uuid: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: backups.longhorn.io +spec: + group: longhorn.io + names: + kind: Backup + listKind: BackupList + plural: backups + shortNames: + - lhb + singular: backup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The snapshot name + jsonPath: .status.snapshotName + name: SnapshotName + type: string + - description: The snapshot size + jsonPath: .status.size + name: SnapshotSize + type: string + - description: The snapshot creation time + jsonPath: .status.snapshotCreatedAt + name: SnapshotCreatedAt + type: string + - description: The backup state + jsonPath: .status.state + name: State + type: string + - description: The backup last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: Backup is where Longhorn stores backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The snapshot name + jsonPath: .status.snapshotName + name: SnapshotName + type: string + - description: The snapshot size + jsonPath: .status.size + name: SnapshotSize + type: string + - description: The snapshot creation time + jsonPath: .status.snapshotCreatedAt + name: SnapshotCreatedAt + type: string + - description: The backup state + jsonPath: .status.state + name: State + type: string + - description: The backup last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Backup is where Longhorn stores backup object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupSpec defines the desired state of the Longhorn backup + properties: + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + type: object + snapshotName: + description: The snapshot name. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupStatus defines the observed state of the Longhorn backup + properties: + backupCreatedAt: + description: The snapshot backup upload finished time. + type: string + error: + description: The error message when taking the snapshot backup. + type: string + labels: + additionalProperties: + type: string + description: The labels of snapshot backup. + nullable: true + type: object + lastSyncedAt: + description: The last time that the backup was synced with the remote backup target. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when calling longhorn engine on listing or inspecting backups. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup CR. + type: string + progress: + description: The snapshot backup progress. + type: integer + replicaAddress: + description: The address of the replica that runs snapshot backup. + type: string + size: + description: The snapshot size. + type: string + snapshotCreatedAt: + description: The snapshot creation time. + type: string + snapshotName: + description: The snapshot name. + type: string + state: + description: The backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown". + type: string + url: + description: The snapshot backup URL. + type: string + volumeBackingImageName: + description: The volume's backing image name. + type: string + volumeCreated: + description: The volume creation time. + type: string + volumeName: + description: The volume name. + type: string + volumeSize: + description: The volume size. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: backuptargets.longhorn.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: {{ include "release_namespace" . }} + path: /v1/webhook/conversion + port: 9443 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: BackupTarget + listKind: BackupTargetList + plural: backuptargets + shortNames: + - lhbt + singular: backuptarget + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup target URL + jsonPath: .spec.backupTargetURL + name: URL + type: string + - description: The backup target credential secret + jsonPath: .spec.credentialSecret + name: Credential + type: string + - description: The backup target poll interval + jsonPath: .spec.pollInterval + name: LastBackupAt + type: string + - description: Indicate whether the backup target is available or not + jsonPath: .status.available + name: Available + type: boolean + - description: The backup target last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BackupTarget is where Longhorn stores backup target object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The backup target URL + jsonPath: .spec.backupTargetURL + name: URL + type: string + - description: The backup target credential secret + jsonPath: .spec.credentialSecret + name: Credential + type: string + - description: The backup target poll interval + jsonPath: .spec.pollInterval + name: LastBackupAt + type: string + - description: Indicate whether the backup target is available or not + jsonPath: .status.available + name: Available + type: boolean + - description: The backup target last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupTarget is where Longhorn stores backup target object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupTargetSpec defines the desired state of the Longhorn backup target + properties: + backupTargetURL: + description: The backup target URL. + type: string + credentialSecret: + description: The backup target credential secret. + type: string + pollInterval: + description: The interval that the cluster needs to run sync with the backup target. + type: string + syncRequestedAt: + description: The time to request run sync the remote backup target. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupTargetStatus defines the observed state of the Longhorn backup target + properties: + available: + description: Available indicates if the remote backup target is available or not. + type: boolean + conditions: + description: Records the reason on why the backup target is unavailable. + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + lastSyncedAt: + description: The last time that the controller synced with the remote backup target. + format: date-time + nullable: true + type: string + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup target CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: backupvolumes.longhorn.io +spec: + group: longhorn.io + names: + kind: BackupVolume + listKind: BackupVolumeList + plural: backupvolumes + shortNames: + - lhbv + singular: backupvolume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The backup volume creation time + jsonPath: .status.createdAt + name: CreatedAt + type: string + - description: The backup volume last backup name + jsonPath: .status.lastBackupName + name: LastBackupName + type: string + - description: The backup volume last backup time + jsonPath: .status.lastBackupAt + name: LastBackupAt + type: string + - description: The backup volume last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BackupVolume is where Longhorn stores backup volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The backup volume creation time + jsonPath: .status.createdAt + name: CreatedAt + type: string + - description: The backup volume last backup name + jsonPath: .status.lastBackupName + name: LastBackupName + type: string + - description: The backup volume last backup time + jsonPath: .status.lastBackupAt + name: LastBackupAt + type: string + - description: The backup volume last synced time + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BackupVolume is where Longhorn stores backup volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupVolumeSpec defines the desired state of the Longhorn backup volume + properties: + syncRequestedAt: + description: The time to request run sync the remote backup volume. + format: date-time + nullable: true + type: string + type: object + status: + description: BackupVolumeStatus defines the observed state of the Longhorn backup volume + properties: + backingImageChecksum: + description: the backing image checksum. + type: string + backingImageName: + description: The backing image name. + type: string + createdAt: + description: The backup volume creation time. + type: string + dataStored: + description: The backup volume block count. + type: string + labels: + additionalProperties: + type: string + description: The backup volume labels. + nullable: true + type: object + lastBackupAt: + description: The latest volume backup time. + type: string + lastBackupName: + description: The latest volume backup name. + type: string + lastModificationTime: + description: The backup volume config last modification time. + format: date-time + nullable: true + type: string + lastSyncedAt: + description: The last time that the backup volume was synced into the cluster. + format: date-time + nullable: true + type: string + messages: + additionalProperties: + type: string + description: The error messages when call longhorn engine on list or inspect backup volumes. + nullable: true + type: object + ownerID: + description: The node ID on which the controller is responsible to reconcile this backup volume CR. + type: string + size: + description: The backup volume size. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: engineimages.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: {{ include "release_namespace" . }} + path: /v1/webhook/conversion + port: 9443 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: EngineImage + listKind: EngineImageList + plural: engineimages + shortNames: + - lhei + singular: engineimage + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: State of the engine image + jsonPath: .status.state + name: State + type: string + - description: The Longhorn engine image + jsonPath: .spec.image + name: Image + type: string + - description: Number of resources using the engine image + jsonPath: .status.refCount + name: RefCount + type: integer + - description: The build date of the engine image + jsonPath: .status.buildDate + name: BuildDate + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: EngineImage is where Longhorn stores engine image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: State of the engine image + jsonPath: .status.state + name: State + type: string + - description: The Longhorn engine image + jsonPath: .spec.image + name: Image + type: string + - description: Number of resources using the engine image + jsonPath: .status.refCount + name: RefCount + type: integer + - description: The build date of the engine image + jsonPath: .status.buildDate + name: BuildDate + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: EngineImage is where Longhorn stores engine image object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EngineImageSpec defines the desired state of the Longhorn engine image + properties: + image: + minLength: 1 + type: string + required: + - image + type: object + status: + description: EngineImageStatus defines the observed state of the Longhorn engine image + properties: + buildDate: + type: string + cliAPIMinVersion: + type: integer + cliAPIVersion: + type: integer + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + controllerAPIMinVersion: + type: integer + controllerAPIVersion: + type: integer + dataFormatMinVersion: + type: integer + dataFormatVersion: + type: integer + gitCommit: + type: string + noRefSince: + type: string + nodeDeploymentMap: + additionalProperties: + type: boolean + nullable: true + type: object + ownerID: + type: string + refCount: + type: integer + state: + type: string + version: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: engines.longhorn.io +spec: + group: longhorn.io + names: + kind: Engine + listKind: EngineList + plural: engines + shortNames: + - lhe + singular: engine + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the engine + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the engine + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Engine is where Longhorn stores engine object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The current state of the engine + jsonPath: .status.currentState + name: State + type: string + - description: The node that the engine is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The instance manager of the engine + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the engine + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Engine is where Longhorn stores engine object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EngineSpec defines the desired state of the Longhorn engine + properties: + active: + type: boolean + backupVolume: + type: string + desireState: + type: string + disableFrontend: + type: boolean + engineImage: + type: string + frontend: + enum: + - blockdev + - iscsi + - "" + type: string + logRequested: + type: boolean + nodeID: + type: string + replicaAddressMap: + additionalProperties: + type: string + type: object + requestedBackupRestore: + type: string + requestedDataSource: + type: string + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + unmapMarkSnapChainRemovedEnabled: + type: boolean + upgradedReplicaAddressMap: + additionalProperties: + type: string + type: object + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: EngineStatus defines the observed state of the Longhorn engine + properties: + backupStatus: + additionalProperties: + properties: + backupURL: + type: string + error: + type: string + progress: + type: integer + replicaAddress: + type: string + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + cloneStatus: + additionalProperties: + properties: + error: + type: string + fromReplicaAddress: + type: string + isCloning: + type: boolean + progress: + type: integer + snapshotName: + type: string + state: + type: string + type: object + nullable: true + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentReplicaAddressMap: + additionalProperties: + type: string + nullable: true + type: object + currentSize: + format: int64 + type: string + currentState: + type: string + endpoint: + type: string + instanceManagerName: + type: string + ip: + type: string + isExpanding: + type: boolean + lastExpansionError: + type: string + lastExpansionFailedAt: + type: string + lastRestoredBackup: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + purgeStatus: + additionalProperties: + properties: + error: + type: string + isPurging: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + rebuildStatus: + additionalProperties: + properties: + error: + type: string + fromReplicaAddress: + type: string + isRebuilding: + type: boolean + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + replicaModeMap: + additionalProperties: + type: string + nullable: true + type: object + restoreStatus: + additionalProperties: + properties: + backupURL: + type: string + currentRestoringBackup: + type: string + error: + type: string + filename: + type: string + isRestoring: + type: boolean + lastRestored: + type: string + progress: + type: integer + state: + type: string + type: object + nullable: true + type: object + salvageExecuted: + type: boolean + snapshots: + additionalProperties: + properties: + children: + additionalProperties: + type: boolean + nullable: true + type: object + created: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + name: + type: string + parent: + type: string + removed: + type: boolean + size: + type: string + usercreated: + type: boolean + type: object + nullable: true + type: object + snapshotsError: + type: string + started: + type: boolean + storageIP: + type: string + unmapMarkSnapChainRemovedEnabled: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: instancemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: InstanceManager + listKind: InstanceManagerList + plural: instancemanagers + shortNames: + - lhim + singular: instancemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the instance manager + jsonPath: .status.currentState + name: State + type: string + - description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + name: Type + type: string + - description: The node that the instance manager is running on + jsonPath: .spec.nodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: InstanceManager is where Longhorn stores instance manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The state of the instance manager + jsonPath: .status.currentState + name: State + type: string + - description: The type of the instance manager (engine or replica) + jsonPath: .spec.type + name: Type + type: string + - description: The node that the instance manager is running on + jsonPath: .spec.nodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: InstanceManager is where Longhorn stores instance manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: InstanceManagerSpec defines the desired state of the Longhorn instancer manager + properties: + engineImage: + description: 'Deprecated: This field is useless.' + type: string + image: + type: string + nodeID: + type: string + type: + enum: + - engine + - replica + type: string + type: object + status: + description: InstanceManagerStatus defines the observed state of the Longhorn instance manager + properties: + apiMinVersion: + type: integer + apiVersion: + type: integer + proxyApiMinVersion: + type: integer + proxyApiVersion: + type: integer + currentState: + type: string + instances: + additionalProperties: + properties: + spec: + properties: + name: + type: string + type: object + status: + properties: + endpoint: + type: string + errorMsg: + type: string + listen: + type: string + portEnd: + format: int32 + type: integer + portStart: + format: int32 + type: integer + resourceVersion: + format: int64 + type: integer + state: + type: string + type: + type: string + type: object + type: object + nullable: true + type: object + ip: + type: string + ownerID: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: nodes.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: {{ include "release_namespace" . }} + path: /v1/webhook/conversion + port: 9443 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: Node + listKind: NodeList + plural: nodes + shortNames: + - lhn + singular: node + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicate whether the node is ready + jsonPath: .status.conditions['Ready']['status'] + name: Ready + type: string + - description: Indicate whether the user disabled/enabled replica scheduling for the node + jsonPath: .spec.allowScheduling + name: AllowScheduling + type: boolean + - description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions['Schedulable']['status'] + name: Schedulable + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Node is where Longhorn stores Longhorn node object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicate whether the node is ready + jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - description: Indicate whether the user disabled/enabled replica scheduling for the node + jsonPath: .spec.allowScheduling + name: AllowScheduling + type: boolean + - description: Indicate whether Longhorn can schedule replicas on the node + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Schedulable + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Node is where Longhorn stores Longhorn node object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeSpec defines the desired state of the Longhorn node + properties: + allowScheduling: + type: boolean + disks: + additionalProperties: + properties: + allowScheduling: + type: boolean + evictionRequested: + type: boolean + path: + type: string + storageReserved: + format: int64 + type: integer + tags: + items: + type: string + type: array + type: object + type: object + engineManagerCPURequest: + type: integer + evictionRequested: + type: boolean + name: + type: string + replicaManagerCPURequest: + type: integer + tags: + items: + type: string + type: array + type: object + status: + description: NodeStatus defines the observed state of the Longhorn node + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskStatus: + additionalProperties: + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + diskUUID: + type: string + scheduledReplica: + additionalProperties: + format: int64 + type: integer + nullable: true + type: object + storageAvailable: + format: int64 + type: integer + storageMaximum: + format: int64 + type: integer + storageScheduled: + format: int64 + type: integer + type: object + nullable: true + type: object + region: + type: string + snapshotCheckStatus: + properties: + lastPeriodicCheckedAt: + format: date-time + type: string + snapshotCheckState: + type: string + type: object + zone: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: orphans.longhorn.io +spec: + group: longhorn.io + names: + kind: Orphan + listKind: OrphanList + plural: orphans + shortNames: + - lho + singular: orphan + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The type of the orphan + jsonPath: .spec.orphanType + name: Type + type: string + - description: The node that the orphan is on + jsonPath: .spec.nodeID + name: Node + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: Orphan is where Longhorn stores orphan object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OrphanSpec defines the desired state of the Longhorn orphaned data + properties: + nodeID: + description: The node ID on which the controller is responsible to reconcile this orphan CR. + type: string + orphanType: + description: The type of the orphaned data. Can be "replica". + type: string + parameters: + additionalProperties: + type: string + description: The parameters of the orphaned data + type: object + type: object + status: + description: OrphanStatus defines the observed state of the Longhorn orphaned data + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + longhorn-manager: "" + name: recurringjobs.longhorn.io +spec: + group: longhorn.io + names: + kind: RecurringJob + listKind: RecurringJobList + plural: recurringjobs + shortNames: + - lhrj + singular: recurringjob + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume + jsonPath: .spec.groups + name: Groups + type: string + - description: Should be one of "backup" or "snapshot" + jsonPath: .spec.task + name: Task + type: string + - description: The cron expression represents recurring job scheduling + jsonPath: .spec.cron + name: Cron + type: string + - description: The number of snapshots/backups to keep for the volume + jsonPath: .spec.retain + name: Retain + type: integer + - description: The concurrent job to run by each cron job + jsonPath: .spec.concurrency + name: Concurrency + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Specify the labels + jsonPath: .spec.labels + name: Labels + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: RecurringJob is where Longhorn stores recurring job object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume + jsonPath: .spec.groups + name: Groups + type: string + - description: Should be one of "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup" or "backup-force-create" + jsonPath: .spec.task + name: Task + type: string + - description: The cron expression represents recurring job scheduling + jsonPath: .spec.cron + name: Cron + type: string + - description: The number of snapshots/backups to keep for the volume + jsonPath: .spec.retain + name: Retain + type: integer + - description: The concurrent job to run by each cron job + jsonPath: .spec.concurrency + name: Concurrency + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Specify the labels + jsonPath: .spec.labels + name: Labels + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: RecurringJob is where Longhorn stores recurring job object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RecurringJobSpec defines the desired state of the Longhorn recurring job + properties: + concurrency: + description: The concurrency of taking the snapshot/backup. + type: integer + cron: + description: The cron setting. + type: string + groups: + description: The recurring job group. + items: + type: string + type: array + labels: + additionalProperties: + type: string + description: The label of the snapshot/backup. + type: object + name: + description: The recurring job name. + type: string + retain: + description: The retain count of the snapshot/backup. + type: integer + task: + description: The recurring job task. Can be "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup" or "backup-force-create". + enum: + - snapshot + - snapshot-force-create + - snapshot-cleanup + - snapshot-delete + - backup + - backup-force-create + type: string + type: object + status: + description: RecurringJobStatus defines the observed state of the Longhorn recurring job + properties: + ownerID: + description: The owner ID which is responsible to reconcile this recurring job CR. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: replicas.longhorn.io +spec: + group: longhorn.io + names: + kind: Replica + listKind: ReplicaList + plural: replicas + shortNames: + - lhr + singular: replica + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the replica + jsonPath: .status.currentState + name: State + type: string + - description: The node that the replica is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk that the replica is on + jsonPath: .spec.diskID + name: Disk + type: string + - description: The instance manager of the replica + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the replica + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Replica is where Longhorn stores replica object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The current state of the replica + jsonPath: .status.currentState + name: State + type: string + - description: The node that the replica is on + jsonPath: .spec.nodeID + name: Node + type: string + - description: The disk that the replica is on + jsonPath: .spec.diskID + name: Disk + type: string + - description: The instance manager of the replica + jsonPath: .status.instanceManagerName + name: InstanceManager + type: string + - description: The current image of the replica + jsonPath: .status.currentImage + name: Image + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Replica is where Longhorn stores replica object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ReplicaSpec defines the desired state of the Longhorn replica + properties: + active: + type: boolean + backingImage: + type: string + baseImage: + description: Deprecated. Rename to BackingImage + type: string + dataDirectoryName: + type: string + dataPath: + description: Deprecated + type: string + desireState: + type: string + diskID: + type: string + diskPath: + type: string + engineImage: + type: string + engineName: + type: string + failedAt: + type: string + hardNodeAffinity: + type: string + healthyAt: + type: string + logRequested: + type: boolean + nodeID: + type: string + rebuildRetryCount: + type: integer + revisionCounterDisabled: + type: boolean + salvageRequested: + type: boolean + unmapMarkDiskChainRemovedEnabled: + type: boolean + volumeName: + type: string + volumeSize: + format: int64 + type: string + type: object + status: + description: ReplicaStatus defines the observed state of the Longhorn replica + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentState: + type: string + evictionRequested: + type: boolean + instanceManagerName: + type: string + ip: + type: string + logFetched: + type: boolean + ownerID: + type: string + port: + type: integer + salvageExecuted: + type: boolean + started: + type: boolean + storageIP: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: settings.longhorn.io +spec: + group: longhorn.io + names: + kind: Setting + listKind: SettingList + plural: settings + shortNames: + - lhs + singular: setting + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The value of the setting + jsonPath: .value + name: Value + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Setting is where Longhorn stores setting object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + value: + type: string + required: + - value + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The value of the setting + jsonPath: .value + name: Value + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Setting is where Longhorn stores setting object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + value: + type: string + required: + - value + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: sharemanagers.longhorn.io +spec: + group: longhorn.io + names: + kind: ShareManager + listKind: ShareManagerList + plural: sharemanagers + shortNames: + - lhsm + singular: sharemanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the share manager + jsonPath: .status.state + name: State + type: string + - description: The node that the share manager is owned by + jsonPath: .status.ownerID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: ShareManager is where Longhorn stores share manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The state of the share manager + jsonPath: .status.state + name: State + type: string + - description: The node that the share manager is owned by + jsonPath: .status.ownerID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: ShareManager is where Longhorn stores share manager object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ShareManagerSpec defines the desired state of the Longhorn share manager + properties: + image: + type: string + type: object + status: + description: ShareManagerStatus defines the observed state of the Longhorn share manager + properties: + endpoint: + type: string + ownerID: + type: string + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: snapshots.longhorn.io +spec: + group: longhorn.io + names: + kind: Snapshot + listKind: SnapshotList + plural: snapshots + shortNames: + - lhsnap + singular: snapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The volume that this snapshot belongs to + jsonPath: .spec.volume + name: Volume + type: string + - description: Timestamp when the point-in-time snapshot was taken + jsonPath: .status.creationTime + name: CreationTime + type: string + - description: Indicates if the snapshot is ready to be used to restore/backup a volume + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the minimum size of volume required to rehydrate from this snapshot + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The actual size of the snapshot + jsonPath: .status.size + name: Size + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Snapshot is the Schema for the snapshots API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SnapshotSpec defines the desired state of Longhorn Snapshot + properties: + createSnapshot: + description: require creating a new snapshot + type: boolean + labels: + additionalProperties: + type: string + description: The labels of snapshot + nullable: true + type: object + volume: + description: the volume that this snapshot belongs to. This field is immutable after creation. Required + type: string + required: + - volume + type: object + status: + description: SnapshotStatus defines the observed state of Longhorn Snapshot + properties: + checksum: + type: string + children: + additionalProperties: + type: boolean + nullable: true + type: object + creationTime: + type: string + error: + type: string + labels: + additionalProperties: + type: string + nullable: true + type: object + markRemoved: + type: boolean + ownerID: + type: string + parent: + type: string + readyToUse: + type: boolean + restoreSize: + format: int64 + type: integer + size: + format: int64 + type: integer + userCreated: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: supportbundles.longhorn.io +spec: + group: longhorn.io + names: + kind: SupportBundle + listKind: SupportBundleList + plural: supportbundles + shortNames: + - lhbundle + singular: supportbundle + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the support bundle + jsonPath: .status.state + name: State + type: string + - description: The issue URL + jsonPath: .spec.issueURL + name: Issue + type: string + - description: A brief description of the issue + jsonPath: .spec.description + name: Description + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SupportBundle is where Longhorn stores support bundle object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SupportBundleSpec defines the desired state of the Longhorn SupportBundle + properties: + description: + description: A brief description of the issue + type: string + issueURL: + description: The issue URL + nullable: true + type: string + nodeID: + description: The preferred responsible controller node ID. + type: string + required: + - description + type: object + status: + description: SupportBundleStatus defines the observed state of the Longhorn SupportBundle + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + type: array + filename: + type: string + filesize: + format: int64 + type: integer + image: + description: The support bundle manager image + type: string + managerIP: + description: The support bundle manager IP + type: string + ownerID: + description: The current responsible controller node ID + type: string + progress: + type: integer + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: systembackups.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemBackup + listKind: SystemBackupList + plural: systembackups + shortNames: + - lhsb + singular: systembackup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system backup Longhorn version + jsonPath: .status.version + name: Version + type: string + - description: The system backup state + jsonPath: .status.state + name: State + type: string + - description: The system backup creation time + jsonPath: .status.createdAt + name: Created + type: string + - description: The last time that the system backup was synced into the cluster + jsonPath: .status.lastSyncedAt + name: LastSyncedAt + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemBackup is where Longhorn stores system backup object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SystemBackupSpec defines the desired state of the Longhorn SystemBackup + type: object + status: + description: SystemBackupStatus defines the observed state of the Longhorn SystemBackup + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + createdAt: + description: The system backup creation time. + format: date-time + type: string + gitCommit: + description: The saved Longhorn manager git commit. + nullable: true + type: string + lastSyncedAt: + description: The last time that the system backup was synced into the cluster. + format: date-time + nullable: true + type: string + managerImage: + description: The saved manager image. + type: string + ownerID: + description: The node ID of the responsible controller to reconcile this SystemBackup. + type: string + state: + description: The system backup state. + type: string + version: + description: The saved Longhorn version. + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: systemrestores.longhorn.io +spec: + group: longhorn.io + names: + kind: SystemRestore + listKind: SystemRestoreList + plural: systemrestores + shortNames: + - lhsr + singular: systemrestore + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The system restore state + jsonPath: .status.state + name: State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: SystemRestore is where Longhorn stores system restore object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SystemRestoreSpec defines the desired state of the Longhorn SystemRestore + properties: + systemBackup: + description: The system backup name in the object store. + type: string + required: + - systemBackup + type: object + status: + description: SystemRestoreStatus defines the observed state of the Longhorn SystemRestore + properties: + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + ownerID: + description: The node ID of the responsible controller to reconcile this SystemRestore. + type: string + sourceURL: + description: The source system backup URL. + type: string + state: + description: The system restore state. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + labels: {{- include "longhorn.labels" . | nindent 4 }} + longhorn-manager: "" + name: volumes.longhorn.io +spec: + preserveUnknownFields: false + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: longhorn-conversion-webhook + namespace: {{ include "release_namespace" . }} + path: /v1/webhook/conversion + port: 9443 + conversionReviewVersions: + - v1beta2 + - v1beta1 + group: longhorn.io + names: + kind: Volume + listKind: VolumeList + plural: volumes + shortNames: + - lhv + singular: volume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The state of the volume + jsonPath: .status.state + name: State + type: string + - description: The robustness of the volume + jsonPath: .status.robustness + name: Robustness + type: string + - description: The scheduled condition of the volume + jsonPath: .status.conditions['scheduled']['status'] + name: Scheduled + type: string + - description: The size of the volume + jsonPath: .spec.size + name: Size + type: string + - description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Volume is where Longhorn stores volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + x-kubernetes-preserve-unknown-fields: true + status: + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The state of the volume + jsonPath: .status.state + name: State + type: string + - description: The robustness of the volume + jsonPath: .status.robustness + name: Robustness + type: string + - description: The scheduled condition of the volume + jsonPath: .status.conditions[?(@.type=='Schedulable')].status + name: Scheduled + type: string + - description: The size of the volume + jsonPath: .spec.size + name: Size + type: string + - description: The node that the volume is currently attaching to + jsonPath: .status.currentNodeID + name: Node + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Volume is where Longhorn stores volume object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeSpec defines the desired state of the Longhorn volume + properties: + Standby: + type: boolean + accessMode: + enum: + - rwo + - rwx + type: string + backingImage: + type: string + baseImage: + description: Deprecated. Rename to BackingImage + type: string + dataLocality: + enum: + - disabled + - best-effort + - strict-local + type: string + dataSource: + type: string + disableFrontend: + type: boolean + diskSelector: + items: + type: string + type: array + encrypted: + type: boolean + engineImage: + type: string + fromBackup: + type: string + restoreVolumeRecurringJob: + enum: + - ignored + - enabled + - disabled + type: string + frontend: + enum: + - blockdev + - iscsi + - "" + type: string + lastAttachedBy: + type: string + migratable: + type: boolean + migrationNodeID: + type: string + nodeID: + type: string + nodeSelector: + items: + type: string + type: array + numberOfReplicas: + type: integer + recurringJobs: + description: Deprecated. Replaced by a separate resource named "RecurringJob" + items: + description: 'Deprecated: This field is useless and has been replaced by the RecurringJob CRD' + properties: + concurrency: + type: integer + cron: + type: string + groups: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + retain: + type: integer + task: + enum: + - snapshot + - snapshot-force-create + - snapshot-cleanup + - snapshot-delete + - backup + - backup-force-create + type: string + type: object + type: array + replicaAutoBalance: + enum: + - ignored + - disabled + - least-effort + - best-effort + type: string + revisionCounterDisabled: + type: boolean + size: + format: int64 + type: string + snapshotDataIntegrity: + enum: + - ignored + - disabled + - enabled + - fast-check + type: string + staleReplicaTimeout: + type: integer + unmapMarkSnapChainRemoved: + enum: + - ignored + - disabled + - enabled + type: string + type: object + status: + description: VolumeStatus defines the observed state of the Longhorn volume + properties: + actualSize: + format: int64 + type: integer + cloneStatus: + properties: + snapshot: + type: string + sourceVolume: + type: string + state: + type: string + type: object + conditions: + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + nullable: true + type: array + currentImage: + type: string + currentNodeID: + type: string + expansionRequired: + type: boolean + frontendDisabled: + type: boolean + isStandby: + type: boolean + kubernetesStatus: + properties: + lastPVCRefAt: + type: string + lastPodRefAt: + type: string + namespace: + description: determine if PVC/Namespace is history or not + type: string + pvName: + type: string + pvStatus: + type: string + pvcName: + type: string + workloadsStatus: + description: determine if Pod/Workload is history or not + items: + properties: + podName: + type: string + podStatus: + type: string + workloadName: + type: string + workloadType: + type: string + type: object + nullable: true + type: array + type: object + lastBackup: + type: string + lastBackupAt: + type: string + lastDegradedAt: + type: string + ownerID: + type: string + pendingNodeID: + type: string + remountRequestedAt: + type: string + restoreInitiated: + type: boolean + restoreRequired: + type: boolean + robustness: + type: string + shareEndpoint: + type: string + shareState: + type: string + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/packages/longhorn-crd/longhorn-1.4/generated-changes/exclude/questions.yaml b/packages/longhorn-crd/longhorn-1.4/generated-changes/exclude/questions.yaml index 508a35bcb0..a55737d603 100644 --- a/packages/longhorn-crd/longhorn-1.4/generated-changes/exclude/questions.yaml +++ b/packages/longhorn-crd/longhorn-1.4/generated-changes/exclude/questions.yaml @@ -17,7 +17,7 @@ questions: label: Longhorn Manager Image Repository group: "Longhorn Images Settings" - variable: image.longhorn.manager.tag - default: v1.4.3 + default: v1.4.4 description: "Specify Longhorn Manager Image Tag" type: string label: Longhorn Manager Image Tag @@ -29,7 +29,7 @@ questions: label: Longhorn Engine Image Repository group: "Longhorn Images Settings" - variable: image.longhorn.engine.tag - default: v1.4.3 + default: v1.4.4 description: "Specify Longhorn Engine Image Tag" type: string label: Longhorn Engine Image Tag @@ -41,7 +41,7 @@ questions: label: Longhorn UI Image Repository group: "Longhorn Images Settings" - variable: image.longhorn.ui.tag - default: v1.4.3 + default: v1.4.4 description: "Specify Longhorn UI Image Tag" type: string label: Longhorn UI Image Tag @@ -53,7 +53,7 @@ questions: label: Longhorn Instance Manager Image Repository group: "Longhorn Images Settings" - variable: image.longhorn.instanceManager.tag - default: v1.4.3 + default: v1.4.4 description: "Specify Longhorn Instance Manager Image Tag" type: string label: Longhorn Instance Manager Image Tag @@ -65,7 +65,7 @@ questions: label: Longhorn Share Manager Image Repository group: "Longhorn Images Settings" - variable: image.longhorn.shareManager.tag - default: v1.4.3 + default: v1.4.4 description: "Specify Longhorn Share Manager Image Tag" type: string label: Longhorn Share Manager Image Tag @@ -77,7 +77,7 @@ questions: label: Longhorn Backing Image Manager Image Repository group: "Longhorn Images Settings" - variable: image.longhorn.backingImageManager.tag - default: v1.4.3 + default: v1.4.4 description: "Specify Longhorn Backing Image Manager Image Tag" type: string label: Longhorn Backing Image Manager Image Tag @@ -89,7 +89,7 @@ questions: label: Longhorn Support Bundle Kit Image Repository group: "Longhorn Images Settings" - variable: image.longhorn.supportBundleKit.tag - default: v0.0.25 + default: v0.0.27 description: "Specify Longhorn Support Bundle Manager Image Tag" type: string label: Longhorn Support Bundle Kit Image Tag @@ -721,18 +721,18 @@ Set the value to **0** to disable backup restore." group: "Longhorn Storage Class Settings" type: string default: -- variable: defaultSettings.defaultNodeSelector.enable - description: "Enable recurring Node selector for Longhorn StorageClass" +- variable: persistence.defaultNodeSelector.enable + description: "Enable Node selector for Longhorn StorageClass" group: "Longhorn Storage Class Settings" label: Enable Storage Class Node Selector type: boolean default: false show_subquestion_if: true subquestions: - - variable: defaultSettings.defaultNodeSelector.selector + - variable: persistence.defaultNodeSelector.selector label: Storage Class Node Selector - description: 'We use NodeSelector when we want to bind PVC via StorageClass into desired mountpoint on the nodes tagged whith its value' - group: "Longhorn Default Settings" + description: 'We use NodeSelector when we want to bind PVC via StorageClass into desired mountpoint on the nodes tagged with its value' + group: "Longhorn Storage Class Settings" type: string default: - variable: persistence.backingImage.enable diff --git a/packages/longhorn-crd/longhorn-1.4/generated-changes/exclude/values.yaml b/packages/longhorn-crd/longhorn-1.4/generated-changes/exclude/values.yaml index ac90a15bd9..94a5ea1c64 100644 --- a/packages/longhorn-crd/longhorn-1.4/generated-changes/exclude/values.yaml +++ b/packages/longhorn-crd/longhorn-1.4/generated-changes/exclude/values.yaml @@ -25,25 +25,25 @@ image: longhorn: engine: repository: longhornio/longhorn-engine - tag: v1.4.3 + tag: v1.4.4 manager: repository: longhornio/longhorn-manager - tag: v1.4.3 + tag: v1.4.4 ui: repository: longhornio/longhorn-ui - tag: v1.4.3 + tag: v1.4.4 instanceManager: repository: longhornio/longhorn-instance-manager - tag: v1.4.3 + tag: v1.4.4 shareManager: repository: longhornio/longhorn-share-manager - tag: v1.4.3 + tag: v1.4.4 backingImageManager: repository: longhornio/backing-image-manager - tag: v1.4.3 + tag: v1.4.4 supportBundleKit: repository: longhornio/support-bundle-kit - tag: v0.0.25 + tag: v0.0.27 csi: attacher: repository: longhornio/csi-attacher diff --git a/packages/longhorn-crd/longhorn-1.4/generated-changes/patch/Chart.yaml.patch b/packages/longhorn-crd/longhorn-1.4/generated-changes/patch/Chart.yaml.patch index c1d1e65413..0bb3127caf 100644 --- a/packages/longhorn-crd/longhorn-1.4/generated-changes/patch/Chart.yaml.patch +++ b/packages/longhorn-crd/longhorn-1.4/generated-changes/patch/Chart.yaml.patch @@ -7,7 +7,7 @@ + catalog.cattle.io/namespace: longhorn-system + catalog.cattle.io/release-name: longhorn-crd apiVersion: v1 - appVersion: v1.4.3 + appVersion: v1.4.4 -description: Longhorn is a distributed block storage system for Kubernetes. -home: https://github.com/longhorn/longhorn -icon: https://raw.githubusercontent.com/cncf/artwork/master/projects/longhorn/icon/color/longhorn-icon-color.png @@ -36,4 +36,4 @@ +description: Installs the CRDs for longhorn. +name: longhorn-crd +type: application - version: 1.4.3 + version: 1.4.4 diff --git a/packages/longhorn-crd/longhorn-1.4/package.yaml b/packages/longhorn-crd/longhorn-1.4/package.yaml index 6e154f6b08..0d94fbcd3f 100644 --- a/packages/longhorn-crd/longhorn-1.4/package.yaml +++ b/packages/longhorn-crd/longhorn-1.4/package.yaml @@ -1,5 +1,5 @@ url: https://github.com/longhorn/charts.git subdirectory: charts/longhorn -commit: 1256dae9a9a3b2eccf675083624a971831894d12 -version: 103.1.0 -doNotRelease: false \ No newline at end of file +commit: 5b345f285cff1963429e349192f8be704d488a84 +version: 103.1.1 +doNotRelease: false diff --git a/packages/longhorn/longhorn-1.4/charts/.helmignore b/packages/longhorn/longhorn-1.4/charts/.helmignore new file mode 100755 index 0000000000..f0c1319444 --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/packages/longhorn/longhorn-1.4/charts/Chart.yaml b/packages/longhorn/longhorn-1.4/charts/Chart.yaml new file mode 100755 index 0000000000..00f83b693c --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/Chart.yaml @@ -0,0 +1,40 @@ +annotations: + catalog.cattle.io/auto-install: longhorn-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Longhorn + catalog.cattle.io/kube-version: '>= 1.21.0-0' + catalog.cattle.io/namespace: longhorn-system + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: longhorn.io/v1beta1 + catalog.cattle.io/rancher-version: '>= 2.8.0-0 < 2.9.0-0' + catalog.cattle.io/release-name: longhorn + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/upstream-version: 1.4.4 +apiVersion: v1 +appVersion: v1.4.4 +description: Longhorn is a distributed block storage system for Kubernetes. +home: https://github.com/longhorn/longhorn +icon: https://raw.githubusercontent.com/cncf/artwork/master/projects/longhorn/icon/color/longhorn-icon-color.png +keywords: +- longhorn +- storage +- distributed +- block +- device +- iscsi +- nfs +kubeVersion: '>=1.21.0-0' +maintainers: +- email: maintainers@longhorn.io + name: Longhorn maintainers +name: longhorn +sources: +- https://github.com/longhorn/longhorn +- https://github.com/longhorn/longhorn-engine +- https://github.com/longhorn/longhorn-instance-manager +- https://github.com/longhorn/longhorn-share-manager +- https://github.com/longhorn/longhorn-manager +- https://github.com/longhorn/longhorn-ui +- https://github.com/longhorn/longhorn-tests +- https://github.com/longhorn/backing-image-manager +version: 1.4.4 diff --git a/packages/longhorn/longhorn-1.4/charts/README.md b/packages/longhorn/longhorn-1.4/charts/README.md new file mode 100755 index 0000000000..60595a86b6 --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/README.md @@ -0,0 +1,49 @@ +# Longhorn Chart + +> **Important**: Please install the Longhorn chart in the `longhorn-system` namespace only. + +> **Warning**: Longhorn doesn't support downgrading from a higher version to a lower version. + +## Source Code + +Longhorn is 100% open source software. Project source code is spread across a number of repos: + +1. Longhorn Engine -- Core controller/replica logic https://github.com/longhorn/longhorn-engine +2. Longhorn Instance Manager -- Controller/replica instance lifecycle management https://github.com/longhorn/longhorn-instance-manager +3. Longhorn Share Manager -- NFS provisioner that exposes Longhorn volumes as ReadWriteMany volumes. https://github.com/longhorn/longhorn-share-manager +4. Backing Image Manager -- Backing image file lifecycle management. https://github.com/longhorn/backing-image-manager +5. Longhorn Manager -- Longhorn orchestration, includes CSI driver for Kubernetes https://github.com/longhorn/longhorn-manager +6. Longhorn UI -- Dashboard https://github.com/longhorn/longhorn-ui + +## Prerequisites + +1. A container runtime compatible with Kubernetes (Docker v1.13+, containerd v1.3.7+, etc.) +2. Kubernetes >= v1.21 +3. Make sure `bash`, `curl`, `findmnt`, `grep`, `awk` and `blkid` has been installed in all nodes of the Kubernetes cluster. +4. Make sure `open-iscsi` has been installed, and the `iscsid` daemon is running on all nodes of the Kubernetes cluster. For GKE, recommended Ubuntu as guest OS image since it contains `open-iscsi` already. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `enablePSP` set to `false` if it has been previously set to `true`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, you may have to clean up your Helm release secrets. +Upon setting `enablePSP` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Longhorn docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Uninstallation + +To prevent Longhorn from being accidentally uninstalled (which leads to data lost), we introduce a new setting, deleting-confirmation-flag. If this flag is **false**, the Longhorn uninstallation job will fail. Set this flag to **true** to allow Longhorn uninstallation. You can set this flag using setting page in Longhorn UI or `kubectl -n longhorn-system patch -p '{"value": "true"}' --type=merge lhs deleting-confirmation-flag` + +To prevent damage to the Kubernetes cluster, we recommend deleting all Kubernetes workloads using Longhorn volumes (PersistentVolume, PersistentVolumeClaim, StorageClass, Deployment, StatefulSet, DaemonSet, etc). + +From Rancher Cluster Explorer UI, navigate to Apps page, delete app `longhorn` then app `longhorn-crd` in Installed Apps tab. + + +--- +Please see [link](https://github.com/longhorn/longhorn) for more information. diff --git a/packages/longhorn/longhorn-1.4/charts/app-readme.md b/packages/longhorn/longhorn-1.4/charts/app-readme.md new file mode 100755 index 0000000000..321e5193c4 --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/app-readme.md @@ -0,0 +1,27 @@ +# Longhorn + +Longhorn is a lightweight, reliable and easy to use distributed block storage system for Kubernetes. Once deployed, users can leverage persistent volumes provided by Longhorn. + +Longhorn creates a dedicated storage controller for each volume and synchronously replicates the volume across multiple replicas stored on multiple nodes. The storage controller and replicas are themselves orchestrated using Kubernetes. Longhorn supports snapshots, backups and even allows you to schedule recurring snapshots and backups! + +**Important**: Please install Longhorn chart in `longhorn-system` namespace only. + +**Warning**: Longhorn doesn't support downgrading from a higher version to a lower version. + +[Chart Documentation](https://github.com/longhorn/longhorn/blob/master/chart/README.md) + + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `enablePSP` set to `false` if it has been previously set to `true`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `enablePSP` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. \ No newline at end of file diff --git a/packages/longhorn/longhorn-1.4/charts/questions.yaml b/packages/longhorn/longhorn-1.4/charts/questions.yaml new file mode 100755 index 0000000000..9879b587fc --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/questions.yaml @@ -0,0 +1,850 @@ +categories: +- storage +namespace: longhorn-system +questions: +- variable: image.defaultImage + default: "true" + description: "Use default Longhorn images" + label: Use Default Images + type: boolean + show_subquestion_if: false + group: "Longhorn Images" + subquestions: + - variable: image.longhorn.manager.repository + default: rancher/mirrored-longhornio-longhorn-manager + description: "Specify Longhorn Manager Image Repository" + type: string + label: Longhorn Manager Image Repository + group: "Longhorn Images Settings" + - variable: image.longhorn.manager.tag + default: v1.4.4 + description: "Specify Longhorn Manager Image Tag" + type: string + label: Longhorn Manager Image Tag + group: "Longhorn Images Settings" + - variable: image.longhorn.engine.repository + default: rancher/mirrored-longhornio-longhorn-engine + description: "Specify Longhorn Engine Image Repository" + type: string + label: Longhorn Engine Image Repository + group: "Longhorn Images Settings" + - variable: image.longhorn.engine.tag + default: v1.4.4 + description: "Specify Longhorn Engine Image Tag" + type: string + label: Longhorn Engine Image Tag + group: "Longhorn Images Settings" + - variable: image.longhorn.ui.repository + default: rancher/mirrored-longhornio-longhorn-ui + description: "Specify Longhorn UI Image Repository" + type: string + label: Longhorn UI Image Repository + group: "Longhorn Images Settings" + - variable: image.longhorn.ui.tag + default: v1.4.4 + description: "Specify Longhorn UI Image Tag" + type: string + label: Longhorn UI Image Tag + group: "Longhorn Images Settings" + - variable: image.longhorn.instanceManager.repository + default: rancher/mirrored-longhornio-longhorn-instance-manager + description: "Specify Longhorn Instance Manager Image Repository" + type: string + label: Longhorn Instance Manager Image Repository + group: "Longhorn Images Settings" + - variable: image.longhorn.instanceManager.tag + default: v1.4.4 + description: "Specify Longhorn Instance Manager Image Tag" + type: string + label: Longhorn Instance Manager Image Tag + group: "Longhorn Images Settings" + - variable: image.longhorn.shareManager.repository + default: rancher/mirrored-longhornio-longhorn-share-manager + description: "Specify Longhorn Share Manager Image Repository" + type: string + label: Longhorn Share Manager Image Repository + group: "Longhorn Images Settings" + - variable: image.longhorn.shareManager.tag + default: v1.4.4 + description: "Specify Longhorn Share Manager Image Tag" + type: string + label: Longhorn Share Manager Image Tag + group: "Longhorn Images Settings" + - variable: image.longhorn.backingImageManager.repository + default: rancher/mirrored-longhornio-backing-image-manager + description: "Specify Longhorn Backing Image Manager Image Repository" + type: string + label: Longhorn Backing Image Manager Image Repository + group: "Longhorn Images Settings" + - variable: image.longhorn.backingImageManager.tag + default: v1.4.4 + description: "Specify Longhorn Backing Image Manager Image Tag" + type: string + label: Longhorn Backing Image Manager Image Tag + group: "Longhorn Images Settings" + - variable: image.longhorn.supportBundleKit.repository + default: rancher/mirrored-longhornio-support-bundle-kit + description: "Specify Longhorn Support Bundle Manager Image Repository" + type: string + label: Longhorn Support Bundle Kit Image Repository + group: "Longhorn Images Settings" + - variable: image.longhorn.supportBundleKit.tag + default: v0.0.27 + description: "Specify Longhorn Support Bundle Manager Image Tag" + type: string + label: Longhorn Support Bundle Kit Image Tag + group: "Longhorn Images Settings" + - variable: image.csi.attacher.repository + default: rancher/mirrored-longhornio-csi-attacher + description: "Specify CSI attacher image repository. Leave blank to autodetect." + type: string + label: Longhorn CSI Attacher Image Repository + group: "Longhorn CSI Driver Images" + - variable: image.csi.attacher.tag + default: v3.4.0 + description: "Specify CSI attacher image tag. Leave blank to autodetect." + type: string + label: Longhorn CSI Attacher Image Tag + group: "Longhorn CSI Driver Images" + - variable: image.csi.provisioner.repository + default: rancher/mirrored-longhornio-csi-provisioner + description: "Specify CSI provisioner image repository. Leave blank to autodetect." + type: string + label: Longhorn CSI Provisioner Image Repository + group: "Longhorn CSI Driver Images" + - variable: image.csi.provisioner.tag + default: v2.1.2 + description: "Specify CSI provisioner image tag. Leave blank to autodetect." + type: string + label: Longhorn CSI Provisioner Image Tag + group: "Longhorn CSI Driver Images" + - variable: image.csi.nodeDriverRegistrar.repository + default: rancher/mirrored-longhornio-csi-node-driver-registrar + description: "Specify CSI Node Driver Registrar image repository. Leave blank to autodetect." + type: string + label: Longhorn CSI Node Driver Registrar Image Repository + group: "Longhorn CSI Driver Images" + - variable: image.csi.nodeDriverRegistrar.tag + default: v2.5.0 + description: "Specify CSI Node Driver Registrar image tag. Leave blank to autodetect." + type: string + label: Longhorn CSI Node Driver Registrar Image Tag + group: "Longhorn CSI Driver Images" + - variable: image.csi.resizer.repository + default: rancher/mirrored-longhornio-csi-resizer + description: "Specify CSI Driver Resizer image repository. Leave blank to autodetect." + type: string + label: Longhorn CSI Driver Resizer Image Repository + group: "Longhorn CSI Driver Images" + - variable: image.csi.resizer.tag + default: v1.3.0 + description: "Specify CSI Driver Resizer image tag. Leave blank to autodetect." + type: string + label: Longhorn CSI Driver Resizer Image Tag + group: "Longhorn CSI Driver Images" + - variable: image.csi.snapshotter.repository + default: rancher/mirrored-longhornio-csi-snapshotter + description: "Specify CSI Driver Snapshotter image repository. Leave blank to autodetect." + type: string + label: Longhorn CSI Driver Snapshotter Image Repository + group: "Longhorn CSI Driver Images" + - variable: image.csi.snapshotter.tag + default: v5.0.1 + description: "Specify CSI Driver Snapshotter image tag. Leave blank to autodetect." + type: string + label: Longhorn CSI Driver Snapshotter Image Tag + group: "Longhorn CSI Driver Images" + - variable: image.csi.livenessProbe.repository + default: rancher/mirrored-longhornio-livenessprobe + description: "Specify CSI liveness probe image repository. Leave blank to autodetect." + type: string + label: Longhorn CSI Liveness Probe Image Repository + group: "Longhorn CSI Driver Images" + - variable: image.csi.livenessProbe.tag + default: v2.8.0 + description: "Specify CSI liveness probe image tag. Leave blank to autodetect." + type: string + label: Longhorn CSI Liveness Probe Image Tag + group: "Longhorn CSI Driver Images" +- variable: privateRegistry.registryUrl + label: Private registry URL + description: "URL of private registry. Leave blank to apply system default registry." + group: "Private Registry Settings" + type: string + default: "" +- variable: privateRegistry.registrySecret + label: Private registry secret name + description: "If create a new private registry secret is true, create a Kubernetes secret with this name; else use the existing secret of this name. Use it to pull images from your private registry." + group: "Private Registry Settings" + type: string + default: "" +- variable: privateRegistry.createSecret + default: "true" + description: "Create a new private registry secret" + type: boolean + group: "Private Registry Settings" + label: Create Secret for Private Registry Settings + show_subquestion_if: true + subquestions: + - variable: privateRegistry.registryUser + label: Private registry user + description: "User used to authenticate to private registry." + type: string + default: "" + - variable: privateRegistry.registryPasswd + label: Private registry password + description: "Password used to authenticate to private registry." + type: password + default: "" +- variable: longhorn.default_setting + default: "false" + description: "Customize the default settings before installing Longhorn for the first time. This option will only work if the cluster hasn't installed Longhorn." + label: "Customize Default Settings" + type: boolean + show_subquestion_if: true + group: "Longhorn Default Settings" + subquestions: + - variable: csi.kubeletRootDir + default: + description: "Specify kubelet root-dir. Leave blank to autodetect." + type: string + label: Kubelet Root Directory + group: "Longhorn CSI Driver Settings" + - variable: csi.attacherReplicaCount + type: int + default: 3 + min: 1 + max: 10 + description: "Specify replica count of CSI Attacher. By default 3." + label: Longhorn CSI Attacher replica count + group: "Longhorn CSI Driver Settings" + - variable: csi.provisionerReplicaCount + type: int + default: 3 + min: 1 + max: 10 + description: "Specify replica count of CSI Provisioner. By default 3." + label: Longhorn CSI Provisioner replica count + group: "Longhorn CSI Driver Settings" + - variable: csi.resizerReplicaCount + type: int + default: 3 + min: 1 + max: 10 + description: "Specify replica count of CSI Resizer. By default 3." + label: Longhorn CSI Resizer replica count + group: "Longhorn CSI Driver Settings" + - variable: csi.snapshotterReplicaCount + type: int + default: 3 + min: 1 + max: 10 + description: "Specify replica count of CSI Snapshotter. By default 3." + label: Longhorn CSI Snapshotter replica count + group: "Longhorn CSI Driver Settings" + - variable: defaultSettings.backupTarget + label: Backup Target + description: "The endpoint used to access the backupstore. NFS and S3 are supported." + group: "Longhorn Default Settings" + type: string + default: + - variable: defaultSettings.backupTargetCredentialSecret + label: Backup Target Credential Secret + description: "The name of the Kubernetes secret associated with the backup target." + group: "Longhorn Default Settings" + type: string + default: + - variable: defaultSettings.allowRecurringJobWhileVolumeDetached + label: Allow Recurring Job While Volume Is Detached + description: 'If this setting is enabled, Longhorn will automatically attaches the volume and takes snapshot/backup when it is the time to do recurring snapshot/backup. +Note that the volume is not ready for workload during the period when the volume was automatically attached. Workload will have to wait until the recurring job finishes.' + group: "Longhorn Default Settings" + type: boolean + default: "false" + - variable: defaultSettings.createDefaultDiskLabeledNodes + label: Create Default Disk on Labeled Nodes + description: 'Create default Disk automatically only on Nodes with the label "node.longhorn.io/create-default-disk=true" if no other disks exist. If disabled, the default disk will be created on all new nodes when each node is first added.' + group: "Longhorn Default Settings" + type: boolean + default: "false" + - variable: defaultSettings.defaultDataPath + label: Default Data Path + description: 'Default path to use for storing data on a host. By default "/var/lib/longhorn/"' + group: "Longhorn Default Settings" + type: string + default: "/var/lib/longhorn/" + - variable: defaultSettings.defaultDataLocality + label: Default Data Locality + description: 'We say a Longhorn volume has data locality if there is a local replica of the volume on the same node as the pod which is using the volume. +This setting specifies the default data locality when a volume is created from the Longhorn UI. For Kubernetes configuration, update the `dataLocality` in the StorageClass +The available modes are: +- **disabled**. This is the default option. There may or may not be a replica on the same node as the attached volume (workload) +- **best-effort**. This option instructs Longhorn to try to keep a replica on the same node as the attached volume (workload). Longhorn will not stop the volume, even if it cannot keep a replica local to the attached volume (workload) due to environment limitation, e.g. not enough disk space, incompatible disk tags, etc.' + group: "Longhorn Default Settings" + type: enum + options: + - "disabled" + - "best-effort" + default: "disabled" + - variable: defaultSettings.replicaSoftAntiAffinity + label: Replica Node Level Soft Anti-Affinity + description: 'Allow scheduling on nodes with existing healthy replicas of the same volume. By default false.' + group: "Longhorn Default Settings" + type: boolean + default: "false" + - variable: defaultSettings.replicaAutoBalance + label: Replica Auto Balance + description: 'Enable this setting automatically rebalances replicas when discovered an available node. +The available global options are: +- **disabled**. This is the default option. No replica auto-balance will be done. +- **least-effort**. This option instructs Longhorn to balance replicas for minimal redundancy. +- **best-effort**. This option instructs Longhorn to balance replicas for even redundancy. +Longhorn also support individual volume setting. The setting can be specified in volume.spec.replicaAutoBalance, this overrules the global setting. +The available volume spec options are: +- **ignored**. This is the default option that instructs Longhorn to inherit from the global setting. +- **disabled**. This option instructs Longhorn no replica auto-balance should be done. +- **least-effort**. This option instructs Longhorn to balance replicas for minimal redundancy. +- **best-effort**. This option instructs Longhorn to balance replicas for even redundancy.' + group: "Longhorn Default Settings" + type: enum + options: + - "disabled" + - "least-effort" + - "best-effort" + default: "disabled" + - variable: defaultSettings.storageOverProvisioningPercentage + label: Storage Over Provisioning Percentage + description: "The over-provisioning percentage defines how much storage can be allocated relative to the hard drive's capacity. By default 200." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 200 + - variable: defaultSettings.storageMinimalAvailablePercentage + label: Storage Minimal Available Percentage + description: "If the minimum available disk capacity exceeds the actual percentage of available disk capacity, the disk becomes unschedulable until more space is freed up. By default 25." + group: "Longhorn Default Settings" + type: int + min: 0 + max: 100 + default: 25 + - variable: defaultSettings.upgradeChecker + label: Enable Upgrade Checker + description: 'Upgrade Checker will check for new Longhorn version periodically. When there is a new version available, a notification will appear in the UI. By default true.' + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.defaultReplicaCount + label: Default Replica Count + description: "The default number of replicas when a volume is created from the Longhorn UI. For Kubernetes configuration, update the `numberOfReplicas` in the StorageClass. By default 3." + group: "Longhorn Default Settings" + type: int + min: 1 + max: 20 + default: 3 + - variable: defaultSettings.defaultLonghornStaticStorageClass + label: Default Longhorn Static StorageClass Name + description: "The 'storageClassName' is given to PVs and PVCs that are created for an existing Longhorn volume. The StorageClass name can also be used as a label, so it is possible to use a Longhorn StorageClass to bind a workload to an existing PV without creating a Kubernetes StorageClass object. By default 'longhorn-static'." + group: "Longhorn Default Settings" + type: string + default: "longhorn-static" + - variable: defaultSettings.backupstorePollInterval + label: Backupstore Poll Interval + description: "In seconds. The backupstore poll interval determines how often Longhorn checks the backupstore for new backups. Set to 0 to disable the polling. By default 300." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 300 + - variable: defaultSettings.failedBackupTTL + label: Failed Backup Time to Live + description: "In minutes. This setting determines how long Longhorn will keep the backup resource that was failed. Set to 0 to disable the auto-deletion. +Failed backups will be checked and cleaned up during backupstore polling which is controlled by **Backupstore Poll Interval** setting. +Hence this value determines the minimal wait interval of the cleanup. And the actual cleanup interval is multiple of **Backupstore Poll Interval**. +Disabling **Backupstore Poll Interval** also means to disable failed backup auto-deletion." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 1440 + - variable: defaultSettings.restoreVolumeRecurringJobs + label: Restore Volume Recurring Jobs + description: "Restore recurring jobs from the backup volume on the backup target and create recurring jobs if not exist during a backup restoration. +Longhorn also supports individual volume setting. The setting can be specified on Backup page when making a backup restoration, this overrules the global setting. +The available volume setting options are: +- **ignored**. This is the default option that instructs Longhorn to inherit from the global setting. +- **enabled**. This option instructs Longhorn to restore recurring jobs/groups from the backup target forcibly. +- **disabled**. This option instructs Longhorn no restoring recurring jobs/groups should be done." + group: "Longhorn Default Settings" + type: boolean + default: "false" + - variable: defaultSettings.recurringSuccessfulJobsHistoryLimit + label: Cronjob Successful Jobs History Limit + description: "This setting specifies how many successful backup or snapshot job histories should be retained. History will not be retained if the value is 0." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 1 + - variable: defaultSettings.recurringFailedJobsHistoryLimit + label: Cronjob Failed Jobs History Limit + description: "This setting specifies how many failed backup or snapshot job histories should be retained. History will not be retained if the value is 0." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 1 + - variable: defaultSettings.supportBundleFailedHistoryLimit + label: SupportBundle Failed History Limit + description: "This setting specifies how many failed support bundles can exist in the cluster. +The retained failed support bundle is for analysis purposes and needs to clean up manually. +Set this value to **0** to have Longhorn automatically purge all failed support bundles." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 1 + - variable: defaultSettings.autoSalvage + label: Automatic salvage + description: "If enabled, volumes will be automatically salvaged when all the replicas become faulty e.g. due to network disconnection. Longhorn will try to figure out which replica(s) are usable, then use them for the volume. By default true." + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly + label: Automatically Delete Workload Pod when The Volume Is Detached Unexpectedly + description: 'If enabled, Longhorn will automatically delete the workload pod that is managed by a controller (e.g. deployment, statefulset, daemonset, etc...) when Longhorn volume is detached unexpectedly (e.g. during Kubernetes upgrade, Docker reboot, or network disconnect). By deleting the pod, its controller restarts the pod and Kubernetes handles volume reattachment and remount. +If disabled, Longhorn will not delete the workload pod that is managed by a controller. You will have to manually restart the pod to reattach and remount the volume. +**Note:** This setting does not apply to the workload pods that do not have a controller. Longhorn never deletes them.' + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.disableSchedulingOnCordonedNode + label: Disable Scheduling On Cordoned Node + description: "Disable Longhorn manager to schedule replica on Kubernetes cordoned node. By default true." + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.replicaZoneSoftAntiAffinity + label: Replica Zone Level Soft Anti-Affinity + description: "Allow scheduling new Replicas of Volume to the Nodes in the same Zone as existing healthy Replicas. Nodes don't belong to any Zone will be treated as in the same Zone. Notice that Longhorn relies on label `topology.kubernetes.io/zone=` in the Kubernetes node object to identify the zone. By default true." + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.nodeDownPodDeletionPolicy + label: Pod Deletion Policy When Node is Down + description: "Defines the Longhorn action when a Volume is stuck with a StatefulSet/Deployment Pod on a node that is down. +- **do-nothing** is the default Kubernetes behavior of never force deleting StatefulSet/Deployment terminating pods. Since the pod on the node that is down isn't removed, Longhorn volumes are stuck on nodes that are down. +- **delete-statefulset-pod** Longhorn will force delete StatefulSet terminating pods on nodes that are down to release Longhorn volumes so that Kubernetes can spin up replacement pods. +- **delete-deployment-pod** Longhorn will force delete Deployment terminating pods on nodes that are down to release Longhorn volumes so that Kubernetes can spin up replacement pods. +- **delete-both-statefulset-and-deployment-pod** Longhorn will force delete StatefulSet/Deployment terminating pods on nodes that are down to release Longhorn volumes so that Kubernetes can spin up replacement pods." + group: "Longhorn Default Settings" + type: enum + options: + - "do-nothing" + - "delete-statefulset-pod" + - "delete-deployment-pod" + - "delete-both-statefulset-and-deployment-pod" + default: "do-nothing" + - variable: defaultSettings.allowNodeDrainWithLastHealthyReplica + label: Allow Node Drain with the Last Healthy Replica + description: "By default, Longhorn will block `kubectl drain` action on a node if the node contains the last healthy replica of a volume. +If this setting is enabled, Longhorn will **not** block `kubectl drain` action on a node even if the node contains the last healthy replica of a volume." + group: "Longhorn Default Settings" + type: boolean + default: "false" + - variable: defaultSettings.nodeDrainPolicy + label: Node Drain Policy + description: "Define the policy to use when a node with the last healthy replica of a volume is drained. +- **block-if-contains-last-replica** Longhorn will block the drain when the node contains the last healthy replica of a volume. +- **allow-if-replica-is-stopped** Longhorn will allow the drain when the node contains the last healthy replica of a volume but the replica is stopped. WARNING: possible data loss if the node is removed after draining. Select this option if you want to drain the node and do in-place upgrade/maintenance. +- **always-allow** Longhorn will allow the drain even though the node contains the last healthy replica of a volume. WARNING: possible data loss if the node is removed after draining. Also possible data corruption if the last replica was running during the draining." + group: "Longhorn Default Settings" + type: enum + options: + - "block-if-contains-last-replica" + - "allow-if-replica-is-stopped" + - "always-allow" + default: "block-if-contains-last-replica" + - variable: defaultSettings.mkfsExt4Parameters + label: Custom mkfs.ext4 parameters + description: "Allows setting additional filesystem creation parameters for ext4. For older host kernels it might be necessary to disable the optional ext4 metadata_csum feature by specifying `-O ^64bit,^metadata_csum`." + group: "Longhorn Default Settings" + type: string + - variable: defaultSettings.disableReplicaRebuild + label: Disable Replica Rebuild + description: "This setting disable replica rebuild cross the whole cluster, eviction and data locality feature won't work if this setting is true. But doesn't have any impact to any current replica rebuild and restore disaster recovery volume." + group: "Longhorn Default Settings" + type: boolean + default: "false" + - variable: defaultSettings.replicaReplenishmentWaitInterval + label: Replica Replenishment Wait Interval + description: "In seconds. The interval determines how long Longhorn will wait at least in order to reuse the existing data on a failed replica rather than directly creating a new replica for a degraded volume. +Warning: This option works only when there is a failed replica in the volume. And this option may block the rebuilding for a while in the case." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 600 + - variable: defaultSettings.concurrentReplicaRebuildPerNodeLimit + label: Concurrent Replica Rebuild Per Node Limit + description: "This setting controls how many replicas on a node can be rebuilt simultaneously. +Typically, Longhorn can block the replica starting once the current rebuilding count on a node exceeds the limit. But when the value is 0, it means disabling the replica rebuilding. +WARNING: +- The old setting \"Disable Replica Rebuild\" is replaced by this setting. +- Different from relying on replica starting delay to limit the concurrent rebuilding, if the rebuilding is disabled, replica object replenishment will be directly skipped. +- When the value is 0, the eviction and data locality feature won't work. But this shouldn't have any impact to any current replica rebuild and backup restore." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 5 + - variable: defaultSettings.concurrentVolumeBackupRestorePerNodeLimit + label: Concurrent Volume Backup Restore Per Node Limit + description: "This setting controls how many volumes on a node can restore the backup concurrently. +Longhorn blocks the backup restore once the restoring volume count exceeds the limit. +Set the value to **0** to disable backup restore." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 5 + - variable: defaultSettings.disableRevisionCounter + label: Disable Revision Counter + description: "This setting is only for volumes created by UI. By default, this is false meaning there will be a reivision counter file to track every write to the volume. During salvage recovering Longhorn will pick the replica with largest reivision counter as candidate to recover the whole volume. If revision counter is disabled, Longhorn will not track every write to the volume. During the salvage recovering, Longhorn will use the 'volume-head-xxx.img' file last modification time and file size to pick the replica candidate to recover the whole volume." + group: "Longhorn Default Settings" + type: boolean + default: "false" + - variable: defaultSettings.systemManagedPodsImagePullPolicy + label: System Managed Pod Image Pull Policy + description: "This setting defines the Image Pull Policy of Longhorn system managed pods, e.g. instance manager, engine image, CSI driver, etc. The new Image Pull Policy will only apply after the system managed pods restart." + group: "Longhorn Default Settings" + type: enum + options: + - "if-not-present" + - "always" + - "never" + default: "if-not-present" + - variable: defaultSettings.allowVolumeCreationWithDegradedAvailability + label: Allow Volume Creation with Degraded Availability + description: "This setting allows user to create and attach a volume that doesn't have all the replicas scheduled at the time of creation." + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.autoCleanupSystemGeneratedSnapshot + label: Automatically Cleanup System Generated Snapshot + description: "This setting enables Longhorn to automatically cleanup the system generated snapshot after replica rebuild is done." + group: "Longhorn Default Settings" + type: boolean + default: "true" + - variable: defaultSettings.concurrentAutomaticEngineUpgradePerNodeLimit + label: Concurrent Automatic Engine Upgrade Per Node Limit + description: "This setting controls how Longhorn automatically upgrades volumes' engines to the new default engine image after upgrading Longhorn manager. The value of this setting specifies the maximum number of engines per node that are allowed to upgrade to the default engine image at the same time. If the value is 0, Longhorn will not automatically upgrade volumes' engines to default version." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 0 + - variable: defaultSettings.backingImageCleanupWaitInterval + label: Backing Image Cleanup Wait Interval + description: "This interval in minutes determines how long Longhorn will wait before cleaning up the backing image file when there is no replica in the disk using it." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 60 + - variable: defaultSettings.backingImageRecoveryWaitInterval + label: Backing Image Recovery Wait Interval + description: "This interval in seconds determines how long Longhorn will wait before re-downloading the backing image file when all disk files of this backing image become failed or unknown. + WARNING: + - This recovery only works for the backing image of which the creation type is \"download\". + - File state \"unknown\" means the related manager pods on the pod is not running or the node itself is down/disconnected." + group: "Longhorn Default Settings" + type: int + min: 0 + default: 300 + - variable: defaultSettings.guaranteedEngineManagerCPU + label: Guaranteed Engine Manager CPU + description: "This integer value indicates how many percentage of the total allocatable CPU on each node will be reserved for each engine manager Pod. For example, 10 means 10% of the total CPU on a node will be allocated to each engine manager pod on this node. This will help maintain engine stability during high node workload. + In order to prevent unexpected volume engine crash as well as guarantee a relative acceptable IO performance, you can use the following formula to calculate a value for this setting: + Guaranteed Engine Manager CPU = The estimated max Longhorn volume engine count on a node * 0.1 / The total allocatable CPUs on the node * 100. + The result of above calculation doesn't mean that's the maximum CPU resources the Longhorn workloads require. To fully exploit the Longhorn volume I/O performance, you can allocate/guarantee more CPU resources via this setting. + If it's hard to estimate the usage now, you can leave it with the default value, which is 12%. Then you can tune it when there is no running workload using Longhorn volumes. + WARNING: + - Value 0 means unsetting CPU requests for engine manager pods. + - Considering the possible new instance manager pods in the further system upgrade, this integer value is range from 0 to 40. And the sum with setting 'Guaranteed Engine Manager CPU' should not be greater than 40. + - One more set of instance manager pods may need to be deployed when the Longhorn system is upgraded. If current available CPUs of the nodes are not enough for the new instance manager pods, you need to detach the volumes using the oldest instance manager pods so that Longhorn can clean up the old pods automatically and release the CPU resources. And the new pods with the latest instance manager image will be launched then. + - This global setting will be ignored for a node if the field \"EngineManagerCPURequest\" on the node is set. + - After this setting is changed, all engine manager pods using this global setting on all the nodes will be automatically restarted. In other words, DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES." + group: "Longhorn Default Settings" + type: int + min: 0 + max: 40 + default: 12 + - variable: defaultSettings.guaranteedReplicaManagerCPU + label: Guaranteed Replica Manager CPU + description: "This integer value indicates how many percentage of the total allocatable CPU on each node will be reserved for each replica manager Pod. 10 means 10% of the total CPU on a node will be allocated to each replica manager pod on this node. This will help maintain replica stability during high node workload. + In order to prevent unexpected volume replica crash as well as guarantee a relative acceptable IO performance, you can use the following formula to calculate a value for this setting: + Guaranteed Replica Manager CPU = The estimated max Longhorn volume replica count on a node * 0.1 / The total allocatable CPUs on the node * 100. + The result of above calculation doesn't mean that's the maximum CPU resources the Longhorn workloads require. To fully exploit the Longhorn volume I/O performance, you can allocate/guarantee more CPU resources via this setting. + If it's hard to estimate the usage now, you can leave it with the default value, which is 12%. Then you can tune it when there is no running workload using Longhorn volumes. + WARNING: + - Value 0 means unsetting CPU requests for replica manager pods. + - Considering the possible new instance manager pods in the further system upgrade, this integer value is range from 0 to 40. And the sum with setting 'Guaranteed Replica Manager CPU' should not be greater than 40. + - One more set of instance manager pods may need to be deployed when the Longhorn system is upgraded. If current available CPUs of the nodes are not enough for the new instance manager pods, you need to detach the volumes using the oldest instance manager pods so that Longhorn can clean up the old pods automatically and release the CPU resources. And the new pods with the latest instance manager image will be launched then. + - This global setting will be ignored for a node if the field \"ReplicaManagerCPURequest\" on the node is set. + - After this setting is changed, all replica manager pods using this global setting on all the nodes will be automatically restarted. In other words, DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES." + group: "Longhorn Default Settings" + type: int + min: 0 + max: 40 + default: 12 +- variable: defaultSettings.kubernetesClusterAutoscalerEnabled + label: Kubernetes Cluster Autoscaler Enabled (Experimental) + description: "Enabling this setting will notify Longhorn that the cluster is using Kubernetes Cluster Autoscaler. + Longhorn prevents data loss by only allowing the Cluster Autoscaler to scale down a node that met all conditions: + - No volume attached to the node. + - Is not the last node containing the replica of any volume. + - Is not running backing image components pod. + - Is not running share manager components pod." + group: "Longhorn Default Settings" + type: boolean + default: false +- variable: defaultSettings.orphanAutoDeletion + label: Orphaned Data Cleanup + description: "This setting allows Longhorn to delete the orphan resource and its corresponding orphaned data automatically like stale replicas. Orphan resources on down or unknown nodes will not be cleaned up automatically." + group: "Longhorn Default Settings" + type: boolean + default: false +- variable: defaultSettings.storageNetwork + label: Storage Network + description: "Longhorn uses the storage network for in-cluster data traffic. Leave this blank to use the Kubernetes cluster network. + To segregate the storage network, input the pre-existing NetworkAttachmentDefinition in \"/\" format. + WARNING: + - The cluster must have pre-existing Multus installed, and NetworkAttachmentDefinition IPs are reachable between nodes. + - DO NOT CHANGE THIS SETTING WITH ATTACHED VOLUMES. Longhorn will try to block this setting update when there are attached volumes. + - When applying the setting, Longhorn will restart all manager, instance-manager, and backing-image-manager pods." + group: "Longhorn Default Settings" + type: string + default: +- variable: defaultSettings.deletingConfirmationFlag + label: Deleting Confirmation Flag + description: "This flag is designed to prevent Longhorn from being accidentally uninstalled which will lead to data lost. + Set this flag to **true** to allow Longhorn uninstallation. + If this flag **false**, Longhorn uninstallation job will fail. " + group: "Longhorn Default Settings" + type: boolean + default: "false" +- variable: defaultSettings.engineReplicaTimeout + label: Timeout between Engine and Replica + description: "In seconds. The setting specifies the timeout between the engine and replica(s), and the value should be between 8 to 30 seconds. The default value is 8 seconds." + group: "Longhorn Default Settings" + type: int + default: "8" +- variable: defaultSettings.snapshotDataIntegrity + label: Snapshot Data Integrity + description: "This setting allows users to enable or disable snapshot hashing and data integrity checking. + Available options are + - **disabled**: Disable snapshot disk file hashing and data integrity checking. + - **enabled**: Enables periodic snapshot disk file hashing and data integrity checking. To detect the filesystem-unaware corruption caused by bit rot or other issues in snapshot disk files, Longhorn system periodically hashes files and finds corrupted ones. Hence, the system performance will be impacted during the periodical checking. + - **fast-check**: Enable snapshot disk file hashing and fast data integrity checking. Longhorn system only hashes snapshot disk files if their are not hashed or the modification time are changed. In this mode, filesystem-unaware corruption cannot be detected, but the impact on system performance can be minimized." + group: "Longhorn Default Settings" + type: string + default: "disabled" +- variable: defaultSettings.snapshotDataIntegrityImmediateCheckAfterSnapshotCreation + label: Immediate Snapshot Data Integrity Check After Creating a Snapshot + description: "Hashing snapshot disk files impacts the performance of the system. The immediate snapshot hashing and checking can be disabled to minimize the impact after creating a snapshot." + group: "Longhorn Default Settings" + type: boolean + default: "false" +- variable: defaultSettings.snapshotDataIntegrityCronjob + label: Snapshot Data Integrity Check CronJob + description: "Unix-cron string format. The setting specifies when Longhorn checks the data integrity of snapshot disk files. + Warning: Hashing snapshot disk files impacts the performance of the system. It is recommended to run data integrity checks during off-peak times and to reduce the frequency of checks." + group: "Longhorn Default Settings" + type: string + default: "0 0 */7 * *" +- variable: defaultSettings.removeSnapshotsDuringFilesystemTrim + label: Remove Snapshots During Filesystem Trim + description: "This setting allows Longhorn filesystem trim feature to automatically mark the latest snapshot and its ancestors as removed and stops at the snapshot containing multiple children.\n\n + Since Longhorn filesystem trim feature can be applied to the volume head and the followed continuous removed or system snapshots only.\n\n + Notice that trying to trim a removed files from a valid snapshot will do nothing but the filesystem will discard this kind of in-memory trimmable file info.\n\n + Later on if you mark the snapshot as removed and want to retry the trim, you may need to unmount and remount the filesystem so that the filesystem can recollect the trimmable file info." + group: "Longhorn Default Settings" + type: boolean + default: "false" +- variable: defaultSettings.fastReplicaRebuildEnabled + label: Fast Replica Rebuild Enabled + description: "This feature supports the fast replica rebuilding. It relies on the checksum of snapshot disk files, so setting the snapshot-data-integrity to **enable** or **fast-check** is a prerequisite." + group: "Longhorn Default Settings" + type: boolean + default: false +- variable: defaultSettings.replicaFileSyncHttpClientTimeout + label: Timeout of HTTP Client to Replica File Sync Server + description: "In seconds. The setting specifies the HTTP client timeout to the file sync server." + group: "Longhorn Default Settings" + type: int + default: "30" +- variable: persistence.defaultClass + default: "true" + description: "Set as default StorageClass for Longhorn" + label: Default Storage Class + group: "Longhorn Storage Class Settings" + required: true + type: boolean +- variable: persistence.reclaimPolicy + label: Storage Class Retain Policy + description: "Define reclaim policy (Retain or Delete)" + group: "Longhorn Storage Class Settings" + required: true + type: enum + options: + - "Delete" + - "Retain" + default: "Delete" +- variable: persistence.defaultClassReplicaCount + description: "Set replica count for Longhorn StorageClass" + label: Default Storage Class Replica Count + group: "Longhorn Storage Class Settings" + type: int + min: 1 + max: 10 + default: 3 +- variable: persistence.defaultDataLocality + description: "Set data locality for Longhorn StorageClass" + label: Default Storage Class Data Locality + group: "Longhorn Storage Class Settings" + type: enum + options: + - "disabled" + - "best-effort" + default: "disabled" +- variable: persistence.recurringJobSelector.enable + description: "Enable recurring job selector for Longhorn StorageClass" + group: "Longhorn Storage Class Settings" + label: Enable Storage Class Recurring Job Selector + type: boolean + default: false + show_subquestion_if: true + subquestions: + - variable: persistence.recurringJobSelector.jobList + description: 'Recurring job selector list for Longhorn StorageClass. Please be careful of quotes of input. e.g., [{"name":"backup", "isGroup":true}]' + label: Storage Class Recurring Job Selector List + group: "Longhorn Storage Class Settings" + type: string + default: +- variable: persistence.defaultNodeSelector.enable + description: "Enable Node selector for Longhorn StorageClass" + group: "Longhorn Storage Class Settings" + label: Enable Storage Class Node Selector + type: boolean + default: false + show_subquestion_if: true + subquestions: + - variable: persistence.defaultNodeSelector.selector + label: Storage Class Node Selector + description: 'We use NodeSelector when we want to bind PVC via StorageClass into desired mountpoint on the nodes tagged with its value' + group: "Longhorn Storage Class Settings" + type: string + default: +- variable: persistence.backingImage.enable + description: "Set backing image for Longhorn StorageClass" + group: "Longhorn Storage Class Settings" + label: Default Storage Class Backing Image + type: boolean + default: false + show_subquestion_if: true + subquestions: + - variable: persistence.backingImage.name + description: 'Specify a backing image that will be used by Longhorn volumes in Longhorn StorageClass. If not exists, the backing image data source type and backing image data source parameters should be specified so that Longhorn will create the backing image before using it.' + label: Storage Class Backing Image Name + group: "Longhorn Storage Class Settings" + type: string + default: + - variable: persistence.backingImage.expectedChecksum + description: 'Specify the expected SHA512 checksum of the selected backing image in Longhorn StorageClass. + WARNING: + - If the backing image name is not specified, setting this field is meaningless. + - It is not recommended to set this field if the data source type is \"export-from-volume\".' + label: Storage Class Backing Image Expected SHA512 Checksum + group: "Longhorn Storage Class Settings" + type: string + default: + - variable: persistence.backingImage.dataSourceType + description: 'Specify the data source type for the backing image used in Longhorn StorageClass. + If the backing image does not exists, Longhorn will use this field to create a backing image. Otherwise, Longhorn will use it to verify the selected backing image. + WARNING: + - If the backing image name is not specified, setting this field is meaningless. + - As for backing image creation with data source type \"upload\", it is recommended to do it via UI rather than StorageClass here. Uploading requires file data sending to the Longhorn backend after the object creation, which is complicated if you want to handle it manually.' + label: Storage Class Backing Image Data Source Type + group: "Longhorn Storage Class Settings" + type: enum + options: + - "" + - "download" + - "upload" + - "export-from-volume" + default: "" + - variable: persistence.backingImage.dataSourceParameters + description: "Specify the data source parameters for the backing image used in Longhorn StorageClass. + If the backing image does not exists, Longhorn will use this field to create a backing image. Otherwise, Longhorn will use it to verify the selected backing image. + This option accepts a json string of a map. e.g., '{\"url\":\"https://backing-image-example.s3-region.amazonaws.com/test-backing-image\"}'. + WARNING: + - If the backing image name is not specified, setting this field is meaningless. + - Be careful of the quotes here." + label: Storage Class Backing Image Data Source Parameters + group: "Longhorn Storage Class Settings" + type: string + default: +- variable: persistence.removeSnapshotsDuringFilesystemTrim + description: "Allow automatically removing snapshots during filesystem trim for Longhorn StorageClass" + label: Default Storage Class Remove Snapshots During Filesystem Trim + group: "Longhorn Storage Class Settings" + type: enum + options: + - "ignored" + - "enabled" + - "disabled" + default: "ignored" +- variable: ingress.enabled + default: "false" + description: "Expose app using Layer 7 Load Balancer - ingress" + type: boolean + group: "Services and Load Balancing" + label: Expose app using Layer 7 Load Balancer + show_subquestion_if: true + subquestions: + - variable: ingress.host + default: "xip.io" + description: "layer 7 Load Balancer hostname" + type: hostname + required: true + label: Layer 7 Load Balancer Hostname + - variable: ingress.path + default: "/" + description: "If ingress is enabled you can set the default ingress path" + type: string + required: true + label: Ingress Path +- variable: service.ui.type + default: "Rancher-Proxy" + description: "Define Longhorn UI service type" + type: enum + options: + - "ClusterIP" + - "NodePort" + - "LoadBalancer" + - "Rancher-Proxy" + label: Longhorn UI Service + show_if: "ingress.enabled=false" + group: "Services and Load Balancing" + show_subquestion_if: "NodePort" + subquestions: + - variable: service.ui.nodePort + default: "" + description: "NodePort port number(to set explicitly, choose port between 30000-32767)" + type: int + min: 30000 + max: 32767 + show_if: "service.ui.type=NodePort||service.ui.type=LoadBalancer" + label: UI Service NodePort number +- variable: enablePSP + default: "false" + description: "Setup a pod security policy for Longhorn workloads." + label: Pod Security Policy + type: boolean + group: "Other Settings" +- variable: global.cattle.windowsCluster.enabled + default: "false" + description: "Enable this to allow Longhorn to run on the Rancher deployed Windows cluster." + label: Rancher Windows Cluster + type: boolean + group: "Other Settings" diff --git a/packages/longhorn/longhorn-1.4/charts/templates/NOTES.txt b/packages/longhorn/longhorn-1.4/charts/templates/NOTES.txt new file mode 100755 index 0000000000..cca7cd77b9 --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/NOTES.txt @@ -0,0 +1,5 @@ +Longhorn is now installed on the cluster! + +Please wait a few minutes for other Longhorn components such as CSI deployments, Engine Images, and Instance Managers to be initialized. + +Visit our documentation at https://longhorn.io/docs/ diff --git a/packages/longhorn/longhorn-1.4/charts/templates/_helpers.tpl b/packages/longhorn/longhorn-1.4/charts/templates/_helpers.tpl new file mode 100755 index 0000000000..3fbc2ac02f --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/_helpers.tpl @@ -0,0 +1,66 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "longhorn.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "longhorn.fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + + +{{- define "longhorn.managerIP" -}} +{{- $fullname := (include "longhorn.fullname" .) -}} +{{- printf "http://%s-backend:9500" $fullname | trunc 63 | trimSuffix "-" -}} +{{- end -}} + + +{{- define "secret" }} +{{- printf "{\"auths\": {\"%s\": {\"auth\": \"%s\"}}}" .Values.privateRegistry.registryUrl (printf "%s:%s" .Values.privateRegistry.registryUser .Values.privateRegistry.registryPasswd | b64enc) | b64enc }} +{{- end }} + +{{- /* +longhorn.labels generates the standard Helm labels. +*/ -}} +{{- define "longhorn.labels" -}} +app.kubernetes.io/name: {{ template "longhorn.name" . }} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion }} +{{- end -}} + + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{- define "registry_url" -}} +{{- if .Values.privateRegistry.registryUrl -}} +{{- printf "%s/" .Values.privateRegistry.registryUrl -}} +{{- else -}} +{{ include "system_default_registry" . }} +{{- end -}} +{{- end -}} + +{{- /* + define the longhorn release namespace +*/ -}} +{{- define "release_namespace" -}} +{{- if .Values.namespaceOverride -}} +{{- .Values.namespaceOverride -}} +{{- else -}} +{{- .Release.Namespace -}} +{{- end -}} +{{- end -}} diff --git a/packages/longhorn/longhorn-1.4/charts/templates/clusterrole.yaml b/packages/longhorn/longhorn-1.4/charts/templates/clusterrole.yaml new file mode 100755 index 0000000000..bf28a4785b --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/clusterrole.yaml @@ -0,0 +1,60 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: longhorn-role + labels: {{- include "longhorn.labels" . | nindent 4 }} +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - "*" +- apiGroups: [""] + resources: ["pods", "events", "persistentvolumes", "persistentvolumeclaims","persistentvolumeclaims/status", "nodes", "proxy/nodes", "pods/log", "secrets", "services", "endpoints", "configmaps", "serviceaccounts"] + verbs: ["*"] +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] +- apiGroups: ["apps"] + resources: ["daemonsets", "statefulsets", "deployments"] + verbs: ["*"] +- apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["*"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets", "podsecuritypolicies"] + verbs: ["*"] +- apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["watch", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "volumeattachments", "volumeattachments/status", "csinodes", "csidrivers"] + verbs: ["*"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"] + verbs: ["*"] +- apiGroups: ["longhorn.io"] + resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", + "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", + "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", + "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", + "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", + "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", + "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status"] + verbs: ["*"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] +- apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list"] +- apiGroups: ["apiregistration.k8s.io"] + resources: ["apiservices"] + verbs: ["list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["get", "list", "create", "patch", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings", "clusterrolebindings", "clusterroles"] + verbs: ["*"] diff --git a/packages/longhorn/longhorn-1.4/charts/templates/clusterrolebinding.yaml b/packages/longhorn/longhorn-1.4/charts/templates/clusterrolebinding.yaml new file mode 100755 index 0000000000..8ab944b238 --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/clusterrolebinding.yaml @@ -0,0 +1,27 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-bind + labels: {{- include "longhorn.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: longhorn-role +subjects: +- kind: ServiceAccount + name: longhorn-service-account + namespace: {{ include "release_namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: longhorn-support-bundle + labels: {{- include "longhorn.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: longhorn-support-bundle + namespace: {{ include "release_namespace" . }} diff --git a/packages/longhorn/longhorn-1.4/charts/templates/daemonset-sa.yaml b/packages/longhorn/longhorn-1.4/charts/templates/daemonset-sa.yaml new file mode 100755 index 0000000000..63f98cd127 --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/daemonset-sa.yaml @@ -0,0 +1,147 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-manager + name: longhorn-manager + namespace: {{ include "release_namespace" . }} +spec: + selector: + matchLabels: + app: longhorn-manager + template: + metadata: + labels: {{- include "longhorn.labels" . | nindent 8 }} + app: longhorn-manager + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + initContainers: + - name: wait-longhorn-admission-webhook + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" -k https://longhorn-admission-webhook:9443/v1/healthz) != "200" ]; do echo waiting; sleep 2; done'] + containers: + - name: longhorn-manager + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + privileged: true + command: + - longhorn-manager + - -d + {{- if eq .Values.longhornManager.log.format "json" }} + - -j + {{- end }} + - daemon + - --engine-image + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.engine.repository }}:{{ .Values.image.longhorn.engine.tag }}" + - --instance-manager-image + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.instanceManager.repository }}:{{ .Values.image.longhorn.instanceManager.tag }}" + - --share-manager-image + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.shareManager.repository }}:{{ .Values.image.longhorn.shareManager.tag }}" + - --backing-image-manager-image + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.backingImageManager.repository }}:{{ .Values.image.longhorn.backingImageManager.tag }}" + - --support-bundle-manager-image + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.supportBundleKit.repository }}:{{ .Values.image.longhorn.supportBundleKit.tag }}" + - --manager-image + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }}" + - --service-account + - longhorn-service-account + ports: + - containerPort: 9500 + name: manager + readinessProbe: + tcpSocket: + port: 9500 + volumeMounts: + - name: dev + mountPath: /host/dev/ + - name: proc + mountPath: /host/proc/ + - name: longhorn + mountPath: /var/lib/longhorn/ + mountPropagation: Bidirectional + - name: longhorn-grpc-tls + mountPath: /tls-files/ + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumes: + - name: dev + hostPath: + path: /dev/ + - name: proc + hostPath: + path: /proc/ + - name: longhorn + hostPath: + path: /var/lib/longhorn/ + - name: longhorn-grpc-tls + secret: + secretName: longhorn-grpc-tls + optional: true + {{- if .Values.privateRegistry.registrySecret }} + imagePullSecrets: + - name: {{ .Values.privateRegistry.registrySecret }} + {{- end }} + {{- if .Values.longhornManager.priorityClass }} + priorityClassName: {{ .Values.longhornManager.priorityClass | quote }} + {{- end }} + {{- if or .Values.longhornManager.tolerations .Values.global.cattle.windowsCluster.enabled }} + tolerations: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} +{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} + {{- end }} + {{- if .Values.longhornManager.tolerations }} +{{ toYaml .Values.longhornManager.tolerations | indent 6 }} + {{- end }} + {{- end }} + {{- if or .Values.longhornManager.nodeSelector .Values.global.cattle.windowsCluster.enabled }} + nodeSelector: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} +{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.longhornManager.nodeSelector }} +{{ toYaml .Values.longhornManager.nodeSelector | indent 8 }} + {{- end }} + {{- end }} + serviceAccountName: longhorn-service-account + updateStrategy: + rollingUpdate: + maxUnavailable: "100%" +--- +apiVersion: v1 +kind: Service +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-manager + name: longhorn-backend + namespace: {{ include "release_namespace" . }} + {{- if .Values.longhornManager.serviceAnnotations }} + annotations: +{{ toYaml .Values.longhornManager.serviceAnnotations | indent 4 }} + {{- end }} +spec: + type: {{ .Values.service.manager.type }} + sessionAffinity: ClientIP + selector: + app: longhorn-manager + ports: + - name: manager + port: 9500 + targetPort: manager + {{- if .Values.service.manager.nodePort }} + nodePort: {{ .Values.service.manager.nodePort }} + {{- end }} diff --git a/packages/longhorn/longhorn-1.4/charts/templates/default-setting.yaml b/packages/longhorn/longhorn-1.4/charts/templates/default-setting.yaml new file mode 100755 index 0000000000..f65b7f1e5b --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/default-setting.yaml @@ -0,0 +1,80 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-default-setting + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} +data: + default-setting.yaml: |- + {{ if not (kindIs "invalid" .Values.defaultSettings.backupTarget) }}backup-target: {{ .Values.defaultSettings.backupTarget }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.backupTargetCredentialSecret) }}backup-target-credential-secret: {{ .Values.defaultSettings.backupTargetCredentialSecret }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.allowRecurringJobWhileVolumeDetached) }}allow-recurring-job-while-volume-detached: {{ .Values.defaultSettings.allowRecurringJobWhileVolumeDetached }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.createDefaultDiskLabeledNodes) }}create-default-disk-labeled-nodes: {{ .Values.defaultSettings.createDefaultDiskLabeledNodes }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.defaultDataPath) }}default-data-path: {{ .Values.defaultSettings.defaultDataPath }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.replicaSoftAntiAffinity) }}replica-soft-anti-affinity: {{ .Values.defaultSettings.replicaSoftAntiAffinity }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.replicaAutoBalance) }}replica-auto-balance: {{ .Values.defaultSettings.replicaAutoBalance }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.storageOverProvisioningPercentage) }}storage-over-provisioning-percentage: {{ .Values.defaultSettings.storageOverProvisioningPercentage }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.storageMinimalAvailablePercentage) }}storage-minimal-available-percentage: {{ .Values.defaultSettings.storageMinimalAvailablePercentage }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.upgradeChecker) }}upgrade-checker: {{ .Values.defaultSettings.upgradeChecker }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.defaultReplicaCount) }}default-replica-count: {{ .Values.defaultSettings.defaultReplicaCount }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.defaultDataLocality) }}default-data-locality: {{ .Values.defaultSettings.defaultDataLocality }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.defaultLonghornStaticStorageClass) }}default-longhorn-static-storage-class: {{ .Values.defaultSettings.defaultLonghornStaticStorageClass }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.backupstorePollInterval) }}backupstore-poll-interval: {{ .Values.defaultSettings.backupstorePollInterval }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.failedBackupTTL) }}failed-backup-ttl: {{ .Values.defaultSettings.failedBackupTTL }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.restoreVolumeRecurringJobs) }}restore-volume-recurring-jobs: {{ .Values.defaultSettings.restoreVolumeRecurringJobs }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.recurringSuccessfulJobsHistoryLimit) }}recurring-successful-jobs-history-limit: {{ .Values.defaultSettings.recurringSuccessfulJobsHistoryLimit }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.recurringFailedJobsHistoryLimit) }}recurring-failed-jobs-history-limit: {{ .Values.defaultSettings.recurringFailedJobsHistoryLimit }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.supportBundleFailedHistoryLimit) }}support-bundle-failed-history-limit: {{ .Values.defaultSettings.supportBundleFailedHistoryLimit }}{{ end }} + {{- if or (not (kindIs "invalid" .Values.defaultSettings.taintToleration)) (.Values.global.cattle.windowsCluster.enabled) }} + taint-toleration: {{ $windowsDefaultSettingTaintToleration := list }}{{ $defaultSettingTaintToleration := list -}} + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.defaultSetting.taintToleration -}} + {{- $windowsDefaultSettingTaintToleration = .Values.global.cattle.windowsCluster.defaultSetting.taintToleration -}} + {{- end -}} + {{- if not (kindIs "invalid" .Values.defaultSettings.taintToleration) -}} + {{- $defaultSettingTaintToleration = .Values.defaultSettings.taintToleration -}} + {{- end -}} + {{- $taintToleration := list $windowsDefaultSettingTaintToleration $defaultSettingTaintToleration }}{{ join ";" (compact $taintToleration) -}} + {{- end }} + {{- if or (not (kindIs "invalid" .Values.defaultSettings.systemManagedComponentsNodeSelector)) (.Values.global.cattle.windowsCluster.enabled) }} + system-managed-components-node-selector: {{ $windowsDefaultSettingNodeSelector := list }}{{ $defaultSettingNodeSelector := list -}} + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.defaultSetting.systemManagedComponentsNodeSelector -}} + {{ $windowsDefaultSettingNodeSelector = .Values.global.cattle.windowsCluster.defaultSetting.systemManagedComponentsNodeSelector -}} + {{- end -}} + {{- if not (kindIs "invalid" .Values.defaultSettings.systemManagedComponentsNodeSelector) -}} + {{- $defaultSettingNodeSelector = .Values.defaultSettings.systemManagedComponentsNodeSelector -}} + {{- end -}} + {{- $nodeSelector := list $windowsDefaultSettingNodeSelector $defaultSettingNodeSelector }}{{ join ";" (compact $nodeSelector) -}} + {{- end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.priorityClass) }}priority-class: {{ .Values.defaultSettings.priorityClass }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.autoSalvage) }}auto-salvage: {{ .Values.defaultSettings.autoSalvage }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly) }}auto-delete-pod-when-volume-detached-unexpectedly: {{ .Values.defaultSettings.autoDeletePodWhenVolumeDetachedUnexpectedly }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.disableSchedulingOnCordonedNode) }}disable-scheduling-on-cordoned-node: {{ .Values.defaultSettings.disableSchedulingOnCordonedNode }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.replicaZoneSoftAntiAffinity) }}replica-zone-soft-anti-affinity: {{ .Values.defaultSettings.replicaZoneSoftAntiAffinity }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.nodeDownPodDeletionPolicy) }}node-down-pod-deletion-policy: {{ .Values.defaultSettings.nodeDownPodDeletionPolicy }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.allowNodeDrainWithLastHealthyReplica) }}allow-node-drain-with-last-healthy-replica: {{ .Values.defaultSettings.allowNodeDrainWithLastHealthyReplica }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.nodeDrainPolicy) }}node-drain-policy: {{ .Values.defaultSettings.nodeDrainPolicy }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.mkfsExt4Parameters) }}mkfs-ext4-parameters: {{ .Values.defaultSettings.mkfsExt4Parameters }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.disableReplicaRebuild) }}disable-replica-rebuild: {{ .Values.defaultSettings.disableReplicaRebuild }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.replicaReplenishmentWaitInterval) }}replica-replenishment-wait-interval: {{ .Values.defaultSettings.replicaReplenishmentWaitInterval }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.concurrentReplicaRebuildPerNodeLimit) }}concurrent-replica-rebuild-per-node-limit: {{ .Values.defaultSettings.concurrentReplicaRebuildPerNodeLimit }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.concurrentVolumeBackupRestorePerNodeLimit) }}concurrent-volume-backup-restore-per-node-limit: {{ .Values.defaultSettings.concurrentVolumeBackupRestorePerNodeLimit }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.disableRevisionCounter) }}disable-revision-counter: {{ .Values.defaultSettings.disableRevisionCounter }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.systemManagedPodsImagePullPolicy) }}system-managed-pods-image-pull-policy: {{ .Values.defaultSettings.systemManagedPodsImagePullPolicy }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.allowVolumeCreationWithDegradedAvailability) }}allow-volume-creation-with-degraded-availability: {{ .Values.defaultSettings.allowVolumeCreationWithDegradedAvailability }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.autoCleanupSystemGeneratedSnapshot) }}auto-cleanup-system-generated-snapshot: {{ .Values.defaultSettings.autoCleanupSystemGeneratedSnapshot }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.concurrentAutomaticEngineUpgradePerNodeLimit) }}concurrent-automatic-engine-upgrade-per-node-limit: {{ .Values.defaultSettings.concurrentAutomaticEngineUpgradePerNodeLimit }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.backingImageCleanupWaitInterval) }}backing-image-cleanup-wait-interval: {{ .Values.defaultSettings.backingImageCleanupWaitInterval }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.backingImageRecoveryWaitInterval) }}backing-image-recovery-wait-interval: {{ .Values.defaultSettings.backingImageRecoveryWaitInterval }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.guaranteedEngineManagerCPU) }}guaranteed-engine-manager-cpu: {{ .Values.defaultSettings.guaranteedEngineManagerCPU }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.guaranteedReplicaManagerCPU) }}guaranteed-replica-manager-cpu: {{ .Values.defaultSettings.guaranteedReplicaManagerCPU }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.kubernetesClusterAutoscalerEnabled) }}kubernetes-cluster-autoscaler-enabled: {{ .Values.defaultSettings.kubernetesClusterAutoscalerEnabled }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.orphanAutoDeletion) }}orphan-auto-deletion: {{ .Values.defaultSettings.orphanAutoDeletion }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.storageNetwork) }}storage-network: {{ .Values.defaultSettings.storageNetwork }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.deletingConfirmationFlag) }}deleting-confirmation-flag: {{ .Values.defaultSettings.deletingConfirmationFlag }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.engineReplicaTimeout) }}engine-replica-timeout: {{ .Values.defaultSettings.engineReplicaTimeout }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.snapshotDataIntegrity) }}snapshot-data-integrity: {{ .Values.defaultSettings.snapshotDataIntegrity }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.snapshotDataIntegrityImmediateCheckAfterSnapshotCreation) }}snapshot-data-integrity-immediate-check-after-snapshot-creation: {{ .Values.defaultSettings.snapshotDataIntegrityImmediateCheckAfterSnapshotCreation }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.snapshotDataIntegrityCronjob) }}snapshot-data-integrity-cronjob: {{ .Values.defaultSettings.snapshotDataIntegrityCronjob }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.removeSnapshotsDuringFilesystemTrim) }}remove-snapshots-during-filesystem-trim: {{ .Values.defaultSettings.removeSnapshotsDuringFilesystemTrim }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.fastReplicaRebuildEnabled) }}fast-replica-rebuild-enabled: {{ .Values.defaultSettings.fastReplicaRebuildEnabled }}{{ end }} + {{ if not (kindIs "invalid" .Values.defaultSettings.replicaFileSyncHttpClientTimeout) }}replica-file-sync-http-client-timeout: {{ .Values.defaultSettings.replicaFileSyncHttpClientTimeout }}{{ end }} \ No newline at end of file diff --git a/packages/longhorn/longhorn-1.4/charts/templates/deployment-driver.yaml b/packages/longhorn/longhorn-1.4/charts/templates/deployment-driver.yaml new file mode 100755 index 0000000000..f162fbf791 --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/deployment-driver.yaml @@ -0,0 +1,118 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: longhorn-driver-deployer + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app: longhorn-driver-deployer + template: + metadata: + labels: {{- include "longhorn.labels" . | nindent 8 }} + app: longhorn-driver-deployer + spec: + initContainers: + - name: wait-longhorn-manager + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done'] + containers: + - name: longhorn-driver-deployer + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - longhorn-manager + - -d + - deploy-driver + - --manager-image + - "{{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }}" + - --manager-url + - http://longhorn-backend:9500/v1 + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + {{- if .Values.csi.kubeletRootDir }} + - name: KUBELET_ROOT_DIR + value: {{ .Values.csi.kubeletRootDir }} + {{- end }} + {{- if and .Values.image.csi.attacher.repository .Values.image.csi.attacher.tag }} + - name: CSI_ATTACHER_IMAGE + value: "{{ template "registry_url" . }}{{ .Values.image.csi.attacher.repository }}:{{ .Values.image.csi.attacher.tag }}" + {{- end }} + {{- if and .Values.image.csi.provisioner.repository .Values.image.csi.provisioner.tag }} + - name: CSI_PROVISIONER_IMAGE + value: "{{ template "registry_url" . }}{{ .Values.image.csi.provisioner.repository }}:{{ .Values.image.csi.provisioner.tag }}" + {{- end }} + {{- if and .Values.image.csi.nodeDriverRegistrar.repository .Values.image.csi.nodeDriverRegistrar.tag }} + - name: CSI_NODE_DRIVER_REGISTRAR_IMAGE + value: "{{ template "registry_url" . }}{{ .Values.image.csi.nodeDriverRegistrar.repository }}:{{ .Values.image.csi.nodeDriverRegistrar.tag }}" + {{- end }} + {{- if and .Values.image.csi.resizer.repository .Values.image.csi.resizer.tag }} + - name: CSI_RESIZER_IMAGE + value: "{{ template "registry_url" . }}{{ .Values.image.csi.resizer.repository }}:{{ .Values.image.csi.resizer.tag }}" + {{- end }} + {{- if and .Values.image.csi.snapshotter.repository .Values.image.csi.snapshotter.tag }} + - name: CSI_SNAPSHOTTER_IMAGE + value: "{{ template "registry_url" . }}{{ .Values.image.csi.snapshotter.repository }}:{{ .Values.image.csi.snapshotter.tag }}" + {{- end }} + {{- if and .Values.image.csi.livenessProbe.repository .Values.image.csi.livenessProbe.tag }} + - name: CSI_LIVENESS_PROBE_IMAGE + value: "{{ template "registry_url" . }}{{ .Values.image.csi.livenessProbe.repository }}:{{ .Values.image.csi.livenessProbe.tag }}" + {{- end }} + {{- if .Values.csi.attacherReplicaCount }} + - name: CSI_ATTACHER_REPLICA_COUNT + value: {{ .Values.csi.attacherReplicaCount | quote }} + {{- end }} + {{- if .Values.csi.provisionerReplicaCount }} + - name: CSI_PROVISIONER_REPLICA_COUNT + value: {{ .Values.csi.provisionerReplicaCount | quote }} + {{- end }} + {{- if .Values.csi.resizerReplicaCount }} + - name: CSI_RESIZER_REPLICA_COUNT + value: {{ .Values.csi.resizerReplicaCount | quote }} + {{- end }} + {{- if .Values.csi.snapshotterReplicaCount }} + - name: CSI_SNAPSHOTTER_REPLICA_COUNT + value: {{ .Values.csi.snapshotterReplicaCount | quote }} + {{- end }} + + {{- if .Values.privateRegistry.registrySecret }} + imagePullSecrets: + - name: {{ .Values.privateRegistry.registrySecret }} + {{- end }} + {{- if .Values.longhornDriver.priorityClass }} + priorityClassName: {{ .Values.longhornDriver.priorityClass | quote }} + {{- end }} + {{- if or .Values.longhornDriver.tolerations .Values.global.cattle.windowsCluster.enabled }} + tolerations: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} +{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} + {{- end }} + {{- if .Values.longhornDriver.tolerations }} +{{ toYaml .Values.longhornDriver.tolerations | indent 6 }} + {{- end }} + {{- end }} + {{- if or .Values.longhornDriver.nodeSelector .Values.global.cattle.windowsCluster.enabled }} + nodeSelector: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} +{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.longhornDriver.nodeSelector }} +{{ toYaml .Values.longhornDriver.nodeSelector | indent 8 }} + {{- end }} + {{- end }} + serviceAccountName: longhorn-service-account + securityContext: + runAsUser: 0 diff --git a/packages/longhorn/longhorn-1.4/charts/templates/deployment-recovery-backend.yaml b/packages/longhorn/longhorn-1.4/charts/templates/deployment-recovery-backend.yaml new file mode 100755 index 0000000000..81c8abad5b --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/deployment-recovery-backend.yaml @@ -0,0 +1,83 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-recovery-backend + name: longhorn-recovery-backend + namespace: {{ include "release_namespace" . }} +spec: + replicas: {{ .Values.longhornRecoveryBackend.replicas }} + selector: + matchLabels: + app: longhorn-recovery-backend + template: + metadata: + labels: {{- include "longhorn.labels" . | nindent 8 }} + app: longhorn-recovery-backend + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - longhorn-recovery-backend + topologyKey: kubernetes.io/hostname + containers: + - name: longhorn-recovery-backend + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + runAsUser: 2000 + command: + - longhorn-manager + - recovery-backend + - --service-account + - longhorn-service-account + ports: + - containerPort: 9600 + name: recov-backend + readinessProbe: + tcpSocket: + port: 9600 + initialDelaySeconds: 3 + periodSeconds: 5 + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.privateRegistry.registrySecret }} + imagePullSecrets: + - name: {{ .Values.privateRegistry.registrySecret }} + {{- end }} + {{- if .Values.longhornRecoveryBackend.priorityClass }} + priorityClassName: {{ .Values.longhornRecoveryBackend.priorityClass | quote }} + {{- end }} + {{- if or .Values.longhornRecoveryBackend.tolerations .Values.global.cattle.windowsCluster.enabled }} + tolerations: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} +{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} + {{- end }} + {{- if .Values.longhornRecoveryBackend.tolerations }} +{{ toYaml .Values.longhornRecoveryBackend.tolerations | indent 6 }} + {{- end }} + {{- end }} + {{- if or .Values.longhornRecoveryBackend.nodeSelector .Values.global.cattle.windowsCluster.enabled }} + nodeSelector: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} +{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.longhornRecoveryBackend.nodeSelector }} +{{ toYaml .Values.longhornRecoveryBackend.nodeSelector | indent 8 }} + {{- end }} + {{- end }} + serviceAccountName: longhorn-service-account diff --git a/packages/longhorn/longhorn-1.4/charts/templates/deployment-ui.yaml b/packages/longhorn/longhorn-1.4/charts/templates/deployment-ui.yaml new file mode 100755 index 0000000000..6bad5cd4ed --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/deployment-ui.yaml @@ -0,0 +1,114 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-ui + name: longhorn-ui + namespace: {{ include "release_namespace" . }} +spec: + replicas: {{ .Values.longhornUI.replicas }} + selector: + matchLabels: + app: longhorn-ui + template: + metadata: + labels: {{- include "longhorn.labels" . | nindent 8 }} + app: longhorn-ui + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - longhorn-ui + topologyKey: kubernetes.io/hostname + containers: + - name: longhorn-ui + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.ui.repository }}:{{ .Values.image.longhorn.ui.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + volumeMounts: + - name : nginx-cache + mountPath: /var/cache/nginx/ + - name : nginx-config + mountPath: /var/config/nginx/ + - name: var-run + mountPath: /var/run/ + ports: + - containerPort: 8000 + name: http + env: + - name: LONGHORN_MANAGER_IP + value: "http://longhorn-backend:9500" + - name: LONGHORN_UI_PORT + value: "8000" + volumes: + - emptyDir: {} + name: nginx-cache + - emptyDir: {} + name: nginx-config + - emptyDir: {} + name: var-run + {{- if .Values.privateRegistry.registrySecret }} + imagePullSecrets: + - name: {{ .Values.privateRegistry.registrySecret }} + {{- end }} + {{- if .Values.longhornUI.priorityClass }} + priorityClassName: {{ .Values.longhornUI.priorityClass | quote }} + {{- end }} + {{- if or .Values.longhornUI.tolerations .Values.global.cattle.windowsCluster.enabled }} + tolerations: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} +{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} + {{- end }} + {{- if .Values.longhornUI.tolerations }} +{{ toYaml .Values.longhornUI.tolerations | indent 6 }} + {{- end }} + {{- end }} + {{- if or .Values.longhornUI.nodeSelector .Values.global.cattle.windowsCluster.enabled }} + nodeSelector: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} +{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.longhornUI.nodeSelector }} +{{ toYaml .Values.longhornUI.nodeSelector | indent 8 }} + {{- end }} + {{- end }} +--- +kind: Service +apiVersion: v1 +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-ui + {{- if eq .Values.service.ui.type "Rancher-Proxy" }} + kubernetes.io/cluster-service: "true" + {{- end }} + name: longhorn-frontend + namespace: {{ include "release_namespace" . }} +spec: + {{- if eq .Values.service.ui.type "Rancher-Proxy" }} + type: ClusterIP + {{- else }} + type: {{ .Values.service.ui.type }} + {{- end }} + {{- if and .Values.service.ui.loadBalancerIP (eq .Values.service.ui.type "LoadBalancer") }} + loadBalancerIP: {{ .Values.service.ui.loadBalancerIP }} + {{- end }} + {{- if and (eq .Values.service.ui.type "LoadBalancer") .Values.service.ui.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.service.ui.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + selector: + app: longhorn-ui + ports: + - name: http + port: 80 + targetPort: http + {{- if .Values.service.ui.nodePort }} + nodePort: {{ .Values.service.ui.nodePort }} + {{- else }} + nodePort: null + {{- end }} diff --git a/packages/longhorn/longhorn-1.4/charts/templates/deployment-webhook.yaml b/packages/longhorn/longhorn-1.4/charts/templates/deployment-webhook.yaml new file mode 100755 index 0000000000..c4d353a90f --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/deployment-webhook.yaml @@ -0,0 +1,166 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-conversion-webhook + name: longhorn-conversion-webhook + namespace: {{ include "release_namespace" . }} +spec: + replicas: {{ .Values.longhornConversionWebhook.replicas }} + selector: + matchLabels: + app: longhorn-conversion-webhook + template: + metadata: + labels: {{- include "longhorn.labels" . | nindent 8 }} + app: longhorn-conversion-webhook + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - longhorn-conversion-webhook + topologyKey: kubernetes.io/hostname + containers: + - name: longhorn-conversion-webhook + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + runAsUser: 2000 + command: + - longhorn-manager + - conversion-webhook + - --service-account + - longhorn-service-account + ports: + - containerPort: 9443 + name: conversion-wh + readinessProbe: + tcpSocket: + port: 9443 + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.privateRegistry.registrySecret }} + imagePullSecrets: + - name: {{ .Values.privateRegistry.registrySecret }} + {{- end }} + {{- if .Values.longhornConversionWebhook.priorityClass }} + priorityClassName: {{ .Values.longhornConversionWebhook.priorityClass | quote }} + {{- end }} + {{- if or .Values.longhornConversionWebhook.tolerations .Values.global.cattle.windowsCluster.enabled }} + tolerations: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} +{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} + {{- end }} + {{- if .Values.longhornConversionWebhook.tolerations }} +{{ toYaml .Values.longhornConversionWebhook.tolerations | indent 6 }} + {{- end }} + {{- end }} + {{- if or .Values.longhornConversionWebhook.nodeSelector .Values.global.cattle.windowsCluster.enabled }} + nodeSelector: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} +{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.longhornConversionWebhook.nodeSelector }} +{{ toYaml .Values.longhornConversionWebhook.nodeSelector | indent 8 }} + {{- end }} + {{- end }} + serviceAccountName: longhorn-service-account +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-admission-webhook + name: longhorn-admission-webhook + namespace: {{ include "release_namespace" . }} +spec: + replicas: {{ .Values.longhornAdmissionWebhook.replicas }} + selector: + matchLabels: + app: longhorn-admission-webhook + template: + metadata: + labels: {{- include "longhorn.labels" . | nindent 8 }} + app: longhorn-admission-webhook + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - longhorn-admission-webhook + topologyKey: kubernetes.io/hostname + initContainers: + - name: wait-longhorn-conversion-webhook + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" -k https://longhorn-conversion-webhook:9443/v1/healthz) != "200" ]; do echo waiting; sleep 2; done'] + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + runAsUser: 2000 + containers: + - name: longhorn-admission-webhook + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + runAsUser: 2000 + command: + - longhorn-manager + - admission-webhook + - --service-account + - longhorn-service-account + ports: + - containerPort: 9443 + name: admission-wh + readinessProbe: + tcpSocket: + port: 9443 + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.privateRegistry.registrySecret }} + imagePullSecrets: + - name: {{ .Values.privateRegistry.registrySecret }} + {{- end }} + {{- if .Values.longhornAdmissionWebhook.priorityClass }} + priorityClassName: {{ .Values.longhornAdmissionWebhook.priorityClass | quote }} + {{- end }} + {{- if or .Values.longhornAdmissionWebhook.tolerations .Values.global.cattle.windowsCluster.enabled }} + tolerations: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} +{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} + {{- end }} + {{- if .Values.longhornAdmissionWebhook.tolerations }} +{{ toYaml .Values.longhornAdmissionWebhook.tolerations | indent 6 }} + {{- end }} + {{- end }} + {{- if or .Values.longhornAdmissionWebhook.nodeSelector .Values.global.cattle.windowsCluster.enabled }} + nodeSelector: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} +{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.longhornAdmissionWebhook.nodeSelector }} +{{ toYaml .Values.longhornAdmissionWebhook.nodeSelector | indent 8 }} + {{- end }} + {{- end }} + serviceAccountName: longhorn-service-account diff --git a/packages/longhorn/longhorn-1.4/charts/templates/ingress.yaml b/packages/longhorn/longhorn-1.4/charts/templates/ingress.yaml new file mode 100755 index 0000000000..ee47f8b8d9 --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/ingress.yaml @@ -0,0 +1,48 @@ +{{- if .Values.ingress.enabled }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else -}} +apiVersion: networking.k8s.io/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: longhorn-ingress + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-ingress + annotations: + {{- if .Values.ingress.secureBackends }} + ingress.kubernetes.io/secure-backends: "true" + {{- end }} + {{- range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + {{- if and .Values.ingress.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end }} + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - path: {{ default "" .Values.ingress.path }} + {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: ImplementationSpecific + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: longhorn-frontend + port: + number: 80 + {{- else }} + serviceName: longhorn-frontend + servicePort: 80 + {{- end }} +{{- if .Values.ingress.tls }} + tls: + - hosts: + - {{ .Values.ingress.host }} + secretName: {{ .Values.ingress.tlsSecret }} +{{- end }} +{{- end }} diff --git a/packages/longhorn/longhorn-1.4/charts/templates/postupgrade-job.yaml b/packages/longhorn/longhorn-1.4/charts/templates/postupgrade-job.yaml new file mode 100755 index 0000000000..b9b2eeb214 --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/postupgrade-job.yaml @@ -0,0 +1,58 @@ +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + "helm.sh/hook": post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation + name: longhorn-post-upgrade + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} +spec: + activeDeadlineSeconds: 900 + backoffLimit: 1 + template: + metadata: + name: longhorn-post-upgrade + labels: {{- include "longhorn.labels" . | nindent 8 }} + spec: + containers: + - name: longhorn-post-upgrade + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + privileged: true + command: + - longhorn-manager + - post-upgrade + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + {{- if .Values.privateRegistry.registrySecret }} + imagePullSecrets: + - name: {{ .Values.privateRegistry.registrySecret }} + {{- end }} + {{- if .Values.longhornManager.priorityClass }} + priorityClassName: {{ .Values.longhornManager.priorityClass | quote }} + {{- end }} + serviceAccountName: longhorn-service-account + {{- if or .Values.longhornManager.tolerations .Values.global.cattle.windowsCluster.enabled }} + tolerations: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} +{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} + {{- end }} + {{- if .Values.longhornManager.tolerations }} +{{ toYaml .Values.longhornManager.tolerations | indent 6 }} + {{- end }} + {{- end }} + {{- if or .Values.longhornManager.nodeSelector .Values.global.cattle.windowsCluster.enabled }} + nodeSelector: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} +{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.longhornManager.nodeSelector }} +{{ toYaml .Values.longhornManager.nodeSelector | indent 8 }} + {{- end }} + {{- end }} diff --git a/packages/longhorn/longhorn-1.4/charts/templates/psp.yaml b/packages/longhorn/longhorn-1.4/charts/templates/psp.yaml new file mode 100755 index 0000000000..a2dfc05bef --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/psp.yaml @@ -0,0 +1,66 @@ +{{- if .Values.enablePSP }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: longhorn-psp + labels: {{- include "longhorn.labels" . | nindent 4 }} +spec: + privileged: true + allowPrivilegeEscalation: true + requiredDropCapabilities: + - NET_RAW + allowedCapabilities: + - SYS_ADMIN + hostNetwork: false + hostIPC: false + hostPID: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + fsGroup: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - configMap + - downwardAPI + - emptyDir + - secret + - projected + - hostPath +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: longhorn-psp-role + labels: {{- include "longhorn.labels" . | nindent 4 }} + namespace: {{ include "release_namespace" . }} +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - longhorn-psp +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: longhorn-psp-binding + labels: {{- include "longhorn.labels" . | nindent 4 }} + namespace: {{ include "release_namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: longhorn-psp-role +subjects: +- kind: ServiceAccount + name: longhorn-service-account + namespace: {{ include "release_namespace" . }} +- kind: ServiceAccount + name: default + namespace: {{ include "release_namespace" . }} +{{- end }} diff --git a/packages/longhorn/longhorn-1.4/charts/templates/registry-secret.yaml b/packages/longhorn/longhorn-1.4/charts/templates/registry-secret.yaml new file mode 100755 index 0000000000..3c6b1dc510 --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/registry-secret.yaml @@ -0,0 +1,13 @@ +{{- if .Values.privateRegistry.createSecret }} +{{- if .Values.privateRegistry.registrySecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.privateRegistry.registrySecret }} + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: {{ template "secret" . }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/packages/longhorn/longhorn-1.4/charts/templates/serviceaccount.yaml b/packages/longhorn/longhorn-1.4/charts/templates/serviceaccount.yaml new file mode 100755 index 0000000000..a563d68ca8 --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/serviceaccount.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-service-account + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-support-bundle + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} \ No newline at end of file diff --git a/packages/longhorn/longhorn-1.4/charts/templates/services.yaml b/packages/longhorn/longhorn-1.4/charts/templates/services.yaml new file mode 100755 index 0000000000..cd008db04f --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/services.yaml @@ -0,0 +1,74 @@ +apiVersion: v1 +kind: Service +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-conversion-webhook + name: longhorn-conversion-webhook + namespace: {{ include "release_namespace" . }} +spec: + type: ClusterIP + sessionAffinity: ClientIP + selector: + app: longhorn-conversion-webhook + ports: + - name: conversion-webhook + port: 9443 + targetPort: conversion-wh +--- +apiVersion: v1 +kind: Service +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-admission-webhook + name: longhorn-admission-webhook + namespace: {{ include "release_namespace" . }} +spec: + type: ClusterIP + sessionAffinity: ClientIP + selector: + app: longhorn-admission-webhook + ports: + - name: admission-webhook + port: 9443 + targetPort: admission-wh +--- +apiVersion: v1 +kind: Service +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + app: longhorn-recovery-backend + name: longhorn-recovery-backend + namespace: {{ include "release_namespace" . }} +spec: + type: ClusterIP + sessionAffinity: ClientIP + selector: + app: longhorn-recovery-backend + ports: + - name: recovery-backend + port: 9600 + targetPort: recov-backend +--- +apiVersion: v1 +kind: Service +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + name: longhorn-engine-manager + namespace: {{ include "release_namespace" . }} +spec: + clusterIP: None + selector: + longhorn.io/component: instance-manager + longhorn.io/instance-manager-type: engine +--- +apiVersion: v1 +kind: Service +metadata: + labels: {{- include "longhorn.labels" . | nindent 4 }} + name: longhorn-replica-manager + namespace: {{ include "release_namespace" . }} +spec: + clusterIP: None + selector: + longhorn.io/component: instance-manager + longhorn.io/instance-manager-type: replica diff --git a/packages/longhorn/longhorn-1.4/charts/templates/storageclass.yaml b/packages/longhorn/longhorn-1.4/charts/templates/storageclass.yaml new file mode 100755 index 0000000000..68325177e8 --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/storageclass.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-storageclass + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} +data: + storageclass.yaml: | + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: longhorn + annotations: + storageclass.kubernetes.io/is-default-class: {{ .Values.persistence.defaultClass | quote }} + provisioner: driver.longhorn.io + allowVolumeExpansion: true + reclaimPolicy: "{{ .Values.persistence.reclaimPolicy }}" + volumeBindingMode: Immediate + parameters: + numberOfReplicas: "{{ .Values.persistence.defaultClassReplicaCount }}" + staleReplicaTimeout: "30" + fromBackup: "" + {{- if .Values.persistence.defaultFsType }} + fsType: "{{ .Values.persistence.defaultFsType }}" + {{- end }} + {{- if .Values.persistence.defaultMkfsParams }} + mkfsParams: "{{ .Values.persistence.defaultMkfsParams }}" + {{- end }} + {{- if .Values.persistence.migratable }} + migratable: "{{ .Values.persistence.migratable }}" + {{- end }} + {{- if .Values.persistence.backingImage.enable }} + backingImage: {{ .Values.persistence.backingImage.name }} + backingImageDataSourceType: {{ .Values.persistence.backingImage.dataSourceType }} + backingImageDataSourceParameters: {{ .Values.persistence.backingImage.dataSourceParameters }} + backingImageChecksum: {{ .Values.persistence.backingImage.expectedChecksum }} + {{- end }} + {{- if .Values.persistence.recurringJobSelector.enable }} + recurringJobSelector: '{{ .Values.persistence.recurringJobSelector.jobList }}' + {{- end }} + dataLocality: {{ .Values.persistence.defaultDataLocality | quote }} + {{- if .Values.persistence.defaultNodeSelector.enable }} + nodeSelector: "{{ .Values.persistence.defaultNodeSelector.selector }}" + {{- end }} diff --git a/packages/longhorn/longhorn-1.4/charts/templates/tls-secrets.yaml b/packages/longhorn/longhorn-1.4/charts/templates/tls-secrets.yaml new file mode 100755 index 0000000000..74c43426de --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/tls-secrets.yaml @@ -0,0 +1,16 @@ +{{- if .Values.ingress.enabled }} +{{- range .Values.ingress.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .name }} + namespace: {{ include "release_namespace" $ }} + labels: {{- include "longhorn.labels" $ | nindent 4 }} + app: longhorn +type: kubernetes.io/tls +data: + tls.crt: {{ .certificate | b64enc }} + tls.key: {{ .key | b64enc }} +--- +{{- end }} +{{- end }} diff --git a/packages/longhorn/longhorn-1.4/charts/templates/uninstall-job.yaml b/packages/longhorn/longhorn-1.4/charts/templates/uninstall-job.yaml new file mode 100755 index 0000000000..989933d96b --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/uninstall-job.yaml @@ -0,0 +1,59 @@ +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + name: longhorn-uninstall + namespace: {{ include "release_namespace" . }} + labels: {{- include "longhorn.labels" . | nindent 4 }} +spec: + activeDeadlineSeconds: 900 + backoffLimit: 1 + template: + metadata: + name: longhorn-uninstall + labels: {{- include "longhorn.labels" . | nindent 8 }} + spec: + containers: + - name: longhorn-uninstall + image: {{ template "registry_url" . }}{{ .Values.image.longhorn.manager.repository }}:{{ .Values.image.longhorn.manager.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + privileged: true + command: + - longhorn-manager + - uninstall + - --force + env: + - name: LONGHORN_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: Never + {{- if .Values.privateRegistry.registrySecret }} + imagePullSecrets: + - name: {{ .Values.privateRegistry.registrySecret }} + {{- end }} + {{- if .Values.longhornManager.priorityClass }} + priorityClassName: {{ .Values.longhornManager.priorityClass | quote }} + {{- end }} + serviceAccountName: longhorn-service-account + {{- if or .Values.longhornManager.tolerations .Values.global.cattle.windowsCluster.enabled }} + tolerations: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.tolerations }} +{{ toYaml .Values.global.cattle.windowsCluster.tolerations | indent 6 }} + {{- end }} + {{- if .Values.longhornManager.tolerations }} +{{ toYaml .Values.longhornManager.tolerations | indent 6 }} + {{- end }} + {{- end }} + {{- if or .Values.longhornManager.nodeSelector .Values.global.cattle.windowsCluster.enabled }} + nodeSelector: + {{- if and .Values.global.cattle.windowsCluster.enabled .Values.global.cattle.windowsCluster.nodeSelector }} +{{ toYaml .Values.global.cattle.windowsCluster.nodeSelector | indent 8 }} + {{- end }} + {{- if or .Values.longhornManager.nodeSelector }} +{{ toYaml .Values.longhornManager.nodeSelector | indent 8 }} + {{- end }} + {{- end }} diff --git a/packages/longhorn/longhorn-1.4/charts/templates/userroles.yaml b/packages/longhorn/longhorn-1.4/charts/templates/userroles.yaml new file mode 100644 index 0000000000..c8eeef5088 --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/userroles.yaml @@ -0,0 +1,50 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: "longhorn-admin" + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: +- apiGroups: [ "longhorn.io" ] + resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", + "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", + "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", + "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", + "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", + "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", + "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status"] + verbs: [ "*" ] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: "longhorn-edit" + labels: + rbac.authorization.k8s.io/aggregate-to-edit: "true" +rules: +- apiGroups: [ "longhorn.io" ] + resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", + "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", + "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", + "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", + "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", + "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", + "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status"] + verbs: [ "*" ] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: "longhorn-view" + labels: + rbac.authorization.k8s.io/aggregate-to-view: "true" +rules: +- apiGroups: [ "longhorn.io" ] + resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings", + "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status", + "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status", + "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status", + "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status", + "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status", + "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status"] + verbs: [ "get", "list", "watch" ] \ No newline at end of file diff --git a/packages/longhorn/longhorn-1.4/charts/templates/validate-install-crd.yaml b/packages/longhorn/longhorn-1.4/charts/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..cd2471281e --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/validate-install-crd.yaml @@ -0,0 +1,33 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "longhorn.io/v1beta1/BackingImageDataSource" false -}} +# {{- set $found "longhorn.io/v1beta1/BackingImageManager" false -}} +# {{- set $found "longhorn.io/v1beta1/BackingImage" false -}} +# {{- set $found "longhorn.io/v1beta1/Backup" false -}} +# {{- set $found "longhorn.io/v1beta1/BackupTarget" false -}} +# {{- set $found "longhorn.io/v1beta1/BackupVolume" false -}} +# {{- set $found "longhorn.io/v1beta1/EngineImage" false -}} +# {{- set $found "longhorn.io/v1beta1/Engine" false -}} +# {{- set $found "longhorn.io/v1beta1/InstanceManager" false -}} +# {{- set $found "longhorn.io/v1beta1/Node" false -}} +# {{- set $found "longhorn.io/v1beta2/Orphan" false -}} +# {{- set $found "longhorn.io/v1beta1/RecurringJob" false -}} +# {{- set $found "longhorn.io/v1beta1/Replica" false -}} +# {{- set $found "longhorn.io/v1beta1/Setting" false -}} +# {{- set $found "longhorn.io/v1beta1/ShareManager" false -}} +# {{- set $found "longhorn.io/v1beta2/Snapshot" false -}} +# {{- set $found "longhorn.io/v1beta2/SupportBundle" false -}} +# {{- set $found "longhorn.io/v1beta2/SystemBackup" false -}} +# {{- set $found "longhorn.io/v1beta2/SystemRestore" false -}} +# {{- set $found "longhorn.io/v1beta1/Volume" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install the corresponding CRD chart before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} \ No newline at end of file diff --git a/packages/longhorn/longhorn-1.4/charts/templates/validate-psp-install.yaml b/packages/longhorn/longhorn-1.4/charts/templates/validate-psp-install.yaml new file mode 100755 index 0000000000..0df98e3657 --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.enablePSP }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} \ No newline at end of file diff --git a/packages/longhorn/longhorn-1.4/charts/values.yaml b/packages/longhorn/longhorn-1.4/charts/values.yaml new file mode 100755 index 0000000000..045cdd69f8 --- /dev/null +++ b/packages/longhorn/longhorn-1.4/charts/values.yaml @@ -0,0 +1,333 @@ +# Default values for longhorn. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. +global: + cattle: + systemDefaultRegistry: "" + windowsCluster: + # Enable this to allow Longhorn to run on the Rancher deployed Windows cluster + enabled: false + # Tolerate Linux node taint + tolerations: + - key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" + # Select Linux nodes + nodeSelector: + kubernetes.io/os: "linux" + # Recognize toleration and node selector for Longhorn run-time created components + defaultSetting: + taintToleration: cattle.io/os=linux:NoSchedule + systemManagedComponentsNodeSelector: kubernetes.io/os:linux + +image: + longhorn: + engine: + repository: rancher/mirrored-longhornio-longhorn-engine + tag: v1.4.4 + manager: + repository: rancher/mirrored-longhornio-longhorn-manager + tag: v1.4.4 + ui: + repository: rancher/mirrored-longhornio-longhorn-ui + tag: v1.4.4 + instanceManager: + repository: rancher/mirrored-longhornio-longhorn-instance-manager + tag: v1.4.4 + shareManager: + repository: rancher/mirrored-longhornio-longhorn-share-manager + tag: v1.4.4 + backingImageManager: + repository: rancher/mirrored-longhornio-backing-image-manager + tag: v1.4.4 + supportBundleKit: + repository: rancher/mirrored-longhornio-support-bundle-kit + tag: v0.0.27 + csi: + attacher: + repository: rancher/mirrored-longhornio-csi-attacher + tag: v3.4.0 + provisioner: + repository: rancher/mirrored-longhornio-csi-provisioner + tag: v2.1.2 + nodeDriverRegistrar: + repository: rancher/mirrored-longhornio-csi-node-driver-registrar + tag: v2.5.0 + resizer: + repository: rancher/mirrored-longhornio-csi-resizer + tag: v1.3.0 + snapshotter: + repository: rancher/mirrored-longhornio-csi-snapshotter + tag: v5.0.1 + livenessProbe: + repository: rancher/mirrored-longhornio-livenessprobe + tag: v2.8.0 + pullPolicy: IfNotPresent + +service: + ui: + type: ClusterIP + nodePort: null + manager: + type: ClusterIP + nodePort: "" + loadBalancerIP: "" + loadBalancerSourceRanges: "" + +persistence: + defaultClass: true + defaultFsType: ext4 + defaultMkfsParams: "" + defaultClassReplicaCount: 3 + defaultDataLocality: disabled # best-effort otherwise + reclaimPolicy: Delete + migratable: false + recurringJobSelector: + enable: false + jobList: [] + backingImage: + enable: false + name: ~ + dataSourceType: ~ + dataSourceParameters: ~ + expectedChecksum: ~ + defaultNodeSelector: + enable: false # disable by default + selector: "" + removeSnapshotsDuringFilesystemTrim: ignored # "enabled" or "disabled" otherwise + +csi: + kubeletRootDir: ~ + attacherReplicaCount: ~ + provisionerReplicaCount: ~ + resizerReplicaCount: ~ + snapshotterReplicaCount: ~ + +defaultSettings: + backupTarget: ~ + backupTargetCredentialSecret: ~ + allowRecurringJobWhileVolumeDetached: ~ + createDefaultDiskLabeledNodes: ~ + defaultDataPath: ~ + defaultDataLocality: ~ + replicaSoftAntiAffinity: ~ + replicaAutoBalance: ~ + storageOverProvisioningPercentage: ~ + storageMinimalAvailablePercentage: ~ + upgradeChecker: ~ + defaultReplicaCount: ~ + defaultLonghornStaticStorageClass: ~ + backupstorePollInterval: ~ + failedBackupTTL: ~ + restoreVolumeRecurringJobs: ~ + recurringSuccessfulJobsHistoryLimit: ~ + recurringFailedJobsHistoryLimit: ~ + supportBundleFailedHistoryLimit: ~ + taintToleration: ~ + systemManagedComponentsNodeSelector: ~ + priorityClass: ~ + autoSalvage: ~ + autoDeletePodWhenVolumeDetachedUnexpectedly: ~ + disableSchedulingOnCordonedNode: ~ + replicaZoneSoftAntiAffinity: ~ + nodeDownPodDeletionPolicy: ~ + allowNodeDrainWithLastHealthyReplica: ~ + nodeDrainPolicy: ~ + mkfsExt4Parameters: ~ + disableReplicaRebuild: ~ + replicaReplenishmentWaitInterval: ~ + concurrentReplicaRebuildPerNodeLimit: ~ + concurrentVolumeBackupRestorePerNodeLimit: ~ + disableRevisionCounter: ~ + systemManagedPodsImagePullPolicy: ~ + allowVolumeCreationWithDegradedAvailability: ~ + autoCleanupSystemGeneratedSnapshot: ~ + concurrentAutomaticEngineUpgradePerNodeLimit: ~ + backingImageCleanupWaitInterval: ~ + backingImageRecoveryWaitInterval: ~ + guaranteedEngineManagerCPU: ~ + guaranteedReplicaManagerCPU: ~ + kubernetesClusterAutoscalerEnabled: ~ + orphanAutoDeletion: ~ + storageNetwork: ~ + deletingConfirmationFlag: ~ + engineReplicaTimeout: ~ + snapshotDataIntegrity: ~ + snapshotDataIntegrityImmediateCheckAfterSnapshotCreation: ~ + snapshotDataIntegrityCronjob: ~ + removeSnapshotsDuringFilesystemTrim: ~ + fastReplicaRebuildEnabled: ~ + replicaFileSyncHttpClientTimeout: ~ +privateRegistry: + createSecret: ~ + registryUrl: ~ + registryUser: ~ + registryPasswd: ~ + registrySecret: ~ + +longhornManager: + log: + ## Allowed values are `plain` or `json`. + format: plain + priorityClass: ~ + tolerations: [] + ## If you want to set tolerations for Longhorn Manager DaemonSet, delete the `[]` in the line above + ## and uncomment this example block + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + nodeSelector: {} + ## If you want to set node selector for Longhorn Manager DaemonSet, delete the `{}` in the line above + ## and uncomment this example block + # label-key1: "label-value1" + # label-key2: "label-value2" + serviceAnnotations: {} + ## If you want to set annotations for the Longhorn Manager service, delete the `{}` in the line above + ## and uncomment this example block + # annotation-key1: "annotation-value1" + # annotation-key2: "annotation-value2" + +longhornDriver: + priorityClass: ~ + tolerations: [] + ## If you want to set tolerations for Longhorn Driver Deployer Deployment, delete the `[]` in the line above + ## and uncomment this example block + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + nodeSelector: {} + ## If you want to set node selector for Longhorn Driver Deployer Deployment, delete the `{}` in the line above + ## and uncomment this example block + # label-key1: "label-value1" + # label-key2: "label-value2" + +longhornUI: + replicas: 2 + priorityClass: ~ + tolerations: [] + ## If you want to set tolerations for Longhorn UI Deployment, delete the `[]` in the line above + ## and uncomment this example block + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + nodeSelector: {} + ## If you want to set node selector for Longhorn UI Deployment, delete the `{}` in the line above + ## and uncomment this example block + # label-key1: "label-value1" + # label-key2: "label-value2" + +longhornConversionWebhook: + replicas: 2 + priorityClass: ~ + tolerations: [] + ## If you want to set tolerations for Longhorn conversion webhook Deployment, delete the `[]` in the line above + ## and uncomment this example block + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + nodeSelector: {} + ## If you want to set node selector for Longhorn conversion webhook Deployment, delete the `{}` in the line above + ## and uncomment this example block + # label-key1: "label-value1" + # label-key2: "label-value2" + +longhornAdmissionWebhook: + replicas: 2 + priorityClass: ~ + tolerations: [] + ## If you want to set tolerations for Longhorn admission webhook Deployment, delete the `[]` in the line above + ## and uncomment this example block + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + nodeSelector: {} + ## If you want to set node selector for Longhorn admission webhook Deployment, delete the `{}` in the line above + ## and uncomment this example block + # label-key1: "label-value1" + # label-key2: "label-value2" + +longhornRecoveryBackend: + replicas: 2 + priorityClass: ~ + tolerations: [] + ## If you want to set tolerations for Longhorn recovery backend Deployment, delete the `[]` in the line above + ## and uncomment this example block + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + nodeSelector: {} + ## If you want to set node selector for Longhorn recovery backend Deployment, delete the `{}` in the line above + ## and uncomment this example block + # label-key1: "label-value1" + # label-key2: "label-value2" + +ingress: + ## Set to true to enable ingress record generation + enabled: false + + ## Add ingressClassName to the Ingress + ## Can replace the kubernetes.io/ingress.class annotation on v1.18+ + ingressClassName: ~ + + host: sslip.io + + ## Set this to true in order to enable TLS on the ingress record + tls: false + + ## Enable this in order to enable that the backend service will be connected at port 443 + secureBackends: false + + ## If TLS is set to true, you must declare what secret will store the key/certificate for TLS + tlsSecret: longhorn.local-tls + + ## If ingress is enabled you can set the default ingress path + ## then you can access the UI by using the following full path {{host}}+{{path}} + path: / + + ## Ingress annotations done as key:value pairs + ## If you're using kube-lego, you will want to add: + ## kubernetes.io/tls-acme: true + ## + ## For a full list of possible ingress annotations, please see + ## ref: https://github.com/kubernetes/ingress-nginx/blob/master/docs/annotations.md + ## + ## If tls is set to true, annotation ingress.kubernetes.io/secure-backends: "true" will automatically be set + annotations: + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: true + + secrets: + ## If you're providing your own certificates, please use this to add the certificates as secrets + ## key and certificate should start with -----BEGIN CERTIFICATE----- or + ## -----BEGIN RSA PRIVATE KEY----- + ## + ## name should line up with a tlsSecret set further up + ## If you're using kube-lego, this is unneeded, as it will create the secret for you if it is not set + ## + ## It is also possible to create and manage the certificates outside of this helm chart + ## Please see README.md for more information + # - name: longhorn.local-tls + # key: + # certificate: + +# For Kubernetes < v1.25, if your cluster enables Pod Security Policy admission controller, +# set this to `true` to ship longhorn-psp which allow privileged Longhorn pods to start +enablePSP: false + +## Specify override namespace, specifically this is useful for using longhorn as sub-chart +## and its release namespace is not the `longhorn-system` +namespaceOverride: "" + +# Annotations to add to the Longhorn Manager DaemonSet Pods. Optional. +annotations: {} + +serviceAccount: + # Annotations to add to the service account + annotations: {} diff --git a/packages/longhorn/longhorn-1.4/generated-changes/patch/Chart.yaml.patch b/packages/longhorn/longhorn-1.4/generated-changes/patch/Chart.yaml.patch index e179748246..b39ef71f98 100644 --- a/packages/longhorn/longhorn-1.4/generated-changes/patch/Chart.yaml.patch +++ b/packages/longhorn/longhorn-1.4/generated-changes/patch/Chart.yaml.patch @@ -7,12 +7,12 @@ + catalog.cattle.io/display-name: Longhorn + catalog.cattle.io/kube-version: '>= 1.21.0-0' + catalog.cattle.io/namespace: longhorn-system -+ catalog.cattle.io/os: linux ++ catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: longhorn.io/v1beta1 + catalog.cattle.io/rancher-version: '>= 2.8.0-0 < 2.9.0-0' + catalog.cattle.io/release-name: longhorn + catalog.cattle.io/type: cluster-tool -+ catalog.cattle.io/upstream-version: 1.4.3 ++ catalog.cattle.io/upstream-version: 1.4.4 apiVersion: v1 - appVersion: v1.4.3 + appVersion: v1.4.4 description: Longhorn is a distributed block storage system for Kubernetes. diff --git a/packages/longhorn/longhorn-1.4/generated-changes/patch/values.yaml.patch b/packages/longhorn/longhorn-1.4/generated-changes/patch/values.yaml.patch index 780f8d1a8e..018c12872b 100644 --- a/packages/longhorn/longhorn-1.4/generated-changes/patch/values.yaml.patch +++ b/packages/longhorn/longhorn-1.4/generated-changes/patch/values.yaml.patch @@ -6,31 +6,31 @@ engine: - repository: longhornio/longhorn-engine + repository: rancher/mirrored-longhornio-longhorn-engine - tag: v1.4.3 + tag: v1.4.4 manager: - repository: longhornio/longhorn-manager + repository: rancher/mirrored-longhornio-longhorn-manager - tag: v1.4.3 + tag: v1.4.4 ui: - repository: longhornio/longhorn-ui + repository: rancher/mirrored-longhornio-longhorn-ui - tag: v1.4.3 + tag: v1.4.4 instanceManager: - repository: longhornio/longhorn-instance-manager + repository: rancher/mirrored-longhornio-longhorn-instance-manager - tag: v1.4.3 + tag: v1.4.4 shareManager: - repository: longhornio/longhorn-share-manager + repository: rancher/mirrored-longhornio-longhorn-share-manager - tag: v1.4.3 + tag: v1.4.4 backingImageManager: - repository: longhornio/backing-image-manager + repository: rancher/mirrored-longhornio-backing-image-manager - tag: v1.4.3 + tag: v1.4.4 supportBundleKit: - repository: longhornio/support-bundle-kit + repository: rancher/mirrored-longhornio-support-bundle-kit - tag: v0.0.25 + tag: v0.0.27 csi: attacher: - repository: longhornio/csi-attacher diff --git a/packages/longhorn/longhorn-1.4/package.yaml b/packages/longhorn/longhorn-1.4/package.yaml index 6e154f6b08..0d94fbcd3f 100644 --- a/packages/longhorn/longhorn-1.4/package.yaml +++ b/packages/longhorn/longhorn-1.4/package.yaml @@ -1,5 +1,5 @@ url: https://github.com/longhorn/charts.git subdirectory: charts/longhorn -commit: 1256dae9a9a3b2eccf675083624a971831894d12 -version: 103.1.0 -doNotRelease: false \ No newline at end of file +commit: 5b345f285cff1963429e349192f8be704d488a84 +version: 103.1.1 +doNotRelease: false From 1b2f4285e2ff6785711a8871a7bb588a1d097c1c Mon Sep 17 00:00:00 2001 From: James Lu Date: Fri, 3 Nov 2023 14:34:33 +0800 Subject: [PATCH 3/5] make charts: release longhorn v1.4.4 into Rancher 2.8 Longhorn 6894 Signed-off-by: James Lu --- .../longhorn-crd-103.1.1+up1.4.4.tgz | Bin 0 -> 10552 bytes assets/longhorn/longhorn-103.1.1+up1.4.4.tgz | Bin 0 -> 24114 bytes .../longhorn-crd/103.1.1+up1.4.4}/Chart.yaml | 2 +- .../longhorn-crd/103.1.1+up1.4.4}/README.md | 0 .../103.1.1+up1.4.4}/templates/_helpers.tpl | 0 .../103.1.1+up1.4.4}/templates/crds.yaml | 0 .../longhorn/103.1.1+up1.4.4}/.helmignore | 0 .../longhorn/103.1.1+up1.4.4}/Chart.yaml | 2 +- .../longhorn/103.1.1+up1.4.4}/README.md | 0 .../longhorn/103.1.1+up1.4.4}/app-readme.md | 0 .../longhorn/103.1.1+up1.4.4}/questions.yaml | 0 .../103.1.1+up1.4.4}/templates/NOTES.txt | 0 .../103.1.1+up1.4.4}/templates/_helpers.tpl | 0 .../templates/clusterrole.yaml | 0 .../templates/clusterrolebinding.yaml | 0 .../templates/daemonset-sa.yaml | 0 .../templates/default-setting.yaml | 0 .../templates/deployment-driver.yaml | 0 .../deployment-recovery-backend.yaml | 0 .../templates/deployment-ui.yaml | 0 .../templates/deployment-webhook.yaml | 0 .../103.1.1+up1.4.4}/templates/ingress.yaml | 0 .../templates/postupgrade-job.yaml | 0 .../103.1.1+up1.4.4}/templates/psp.yaml | 0 .../templates/registry-secret.yaml | 0 .../templates/serviceaccount.yaml | 0 .../103.1.1+up1.4.4}/templates/services.yaml | 0 .../templates/storageclass.yaml | 0 .../templates/tls-secrets.yaml | 0 .../templates/uninstall-job.yaml | 0 .../103.1.1+up1.4.4}/templates/userroles.yaml | 0 .../templates/validate-install-crd.yaml | 0 .../templates/validate-psp-install.yaml | 0 .../longhorn/103.1.1+up1.4.4}/values.yaml | 0 index.yaml | 59 ++++++++++++++++++ release.yaml | 5 +- 36 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 assets/longhorn-crd/longhorn-crd-103.1.1+up1.4.4.tgz create mode 100644 assets/longhorn/longhorn-103.1.1+up1.4.4.tgz rename {packages/longhorn-crd/longhorn-1.4/charts => charts/longhorn-crd/103.1.1+up1.4.4}/Chart.yaml (92%) mode change 100755 => 100644 rename {packages/longhorn-crd/longhorn-1.4/charts => charts/longhorn-crd/103.1.1+up1.4.4}/README.md (100%) mode change 100755 => 100644 rename {packages/longhorn-crd/longhorn-1.4/charts => charts/longhorn-crd/103.1.1+up1.4.4}/templates/_helpers.tpl (100%) mode change 100755 => 100644 rename {packages/longhorn-crd/longhorn-1.4/charts => charts/longhorn-crd/103.1.1+up1.4.4}/templates/crds.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/.helmignore (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/Chart.yaml (98%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/README.md (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/app-readme.md (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/questions.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/NOTES.txt (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/_helpers.tpl (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/clusterrole.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/clusterrolebinding.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/daemonset-sa.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/default-setting.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/deployment-driver.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/deployment-recovery-backend.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/deployment-ui.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/deployment-webhook.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/ingress.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/postupgrade-job.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/psp.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/registry-secret.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/serviceaccount.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/services.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/storageclass.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/tls-secrets.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/uninstall-job.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/userroles.yaml (100%) rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/validate-install-crd.yaml (100%) rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/templates/validate-psp-install.yaml (100%) mode change 100755 => 100644 rename {packages/longhorn/longhorn-1.4/charts => charts/longhorn/103.1.1+up1.4.4}/values.yaml (100%) mode change 100755 => 100644 diff --git a/assets/longhorn-crd/longhorn-crd-103.1.1+up1.4.4.tgz b/assets/longhorn-crd/longhorn-crd-103.1.1+up1.4.4.tgz new file mode 100644 index 0000000000000000000000000000000000000000..1e8870925ecbbc428f8f48bbda42b975ee2a7f32 GIT binary patch literal 10552 zcmV-8DaY0yiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PKD1bKEwTZ$A64KxyxkliBH(6g$pcRi$ z9o%ML?ejS~IXQXu^r`xLa&prCd-C1Mw_lw;{qEb-)6>&u-+uqq$?22to_+fjIN2)| zo~Pgfvae2duB+X+&*XuGPyq#|gr7YEAb|pAbUja?5E+_ddYm98@Cu{!3@}KN4Pwq@ z8=R(yoPi_33UuU5Lf{th946=tRI`rXae=nZ(B+JkM~Vdgswi zS;OhcllkfV^xunISM{k!aJQ$i{sr3R85D>g|GYt2ju@Yde0N?ju>PNXfBLMo{-2#b z`#!G!`*?o+_4tqA25-**M*^&HhUC=$VJo{O*`PD<$K&6Adn8jmdhs!bBvp$@t{6a9 z>UW}q#v_w%22!-b1c9S!eaXZ}U{-&IMJB*p3!68xr1=M!6^N^pf4)JC;S_;?0l^BA zfM-wCPrSVKE?uP; zvJTk{4wfi^1xJ9>Edu{rED<9}Ag&v{!YE5QfDC~QZ?Qlr5EO_F=HTn)o$8E>H&-$( zCTqZRlzh|IFOs@u-~1m{&Djs*JE_c>Gfcz^9Q}dM{=kpGe4&}A&Z&zH>Q6VtTS(v< zvCFp>$z>}f*^V<~s3oM^KMI`yA`UR{O>sdbo$A5fOb#?}qDC1y_2x5|rt0~C^ zBq?Mm_-~YLfleZ)rM)7X!>rTj4u*L?zpmCnDaOyhufKv)T6C8BToSiI*>=u1$BI3u z^!8-_?YCL|#t3ilFThZq!32S$e?J08KdZs(5};>cI$Pf9V#~RHh1d;FkUJBWIsi#f zGmD|ck-if5HJQ&VH3Rg|Jf+k8Y}WA4<*3#Z7O$!)u+HcbW~J7nb6)6-<+O0D@#CXO z1)z-A9OJ0FfLa{2Zc*8Up9_{bo6)yKddVM?(IOjE4945Z^Y-9$T6O%#M`g?8`Y0za zFbXHA^i|DJq*^6hu0 zE&cDir_a8N`rmy#wkZX3j6MoPod@D_0axA_?$ zMV37RMsi7pJOf=lqm)^5@Gn3xNf8m?sbmF`h-#7>1?CJKNfT8|GO2oCvc_8}RG@&m zeSBUOjp-w)%gOev87=bKRFt1|F{ZV;r+(BWUdj?LpnzAZL`@0i;(vP*y~13mgn3pl zn0cvFY50aR@useQ2C~f(r{*Ij>mq~9SHvR#ctWL>Td|h(5ddaHREaYH(^QQy%-%}w zjo3NOiY=+?Yna*Zq%lx1CcB;ssTddwA*U%QxYm|)MsKi^q4J9%4Jg`_^#N9lZUL`* z41j-dO5Q@TIRkT9Y{BQImMd9_NumuXm?Imh%36i;|1xO9wSp zF(vnycUFF*%;)Q7NJ@f$RF4tkl4}%MDjD+)VgfLf&^5uoRHbok)FPlX0Rhyy zColsyQgitefFuQ5cn27g<${7(CERG`2E3*W0Zdl(3`iS~pB*2su_#4KLbuz3U~zY> zqyt_Sf--)bq8pSQbG)8GmTa&lKruGarF+ARqGDdeOo8xbvMf{!eoV*PN~?MECrD?rwhF;EqUT{odOHsu`GHE6x_0Nixi!Op;T-W03F_Vhp9eyh}+D7RPa4wbF3Jm^v@!VhVE*s5YsMw|yF-y+VX#vhW0-V($@R|TVMhPY$4 z>r7I5u2s;D{jh=%OxAA?y`)HyWpJ6Hj^$alx?9NowiUcKUAlAv6m^i3NRR)BL2>#BbfO7sx%Gw|1+9_flv`k{ni<;&S@_9$ZX zHiT`y?+c)Rt&7t%IMI2Nq8FMkiPE#hiOX;GHzzE=t%$fUOnz%M`=A!N77kXG3O!H; zV!?g(xysb2_R@M2WF?oiN@e>VjA5%fhImd1$IA>!zPDck*iAtW#imV0!E3hYs$Blx zsoZ+`xxx3Z>!yz}z0018W_L8ZquJd`9C7<7Vxx%tf<)|o0PZMaqlk?n_R}NVW*zPX zYd57--@=s7(yILvz_|AZj;qcI$`Oo+?*qOqXTXlwZP+A$Iev|alSzXC6wk-phw|C; z?j_F9Yv@38$Id=~%k*OuvvV&zZPV+S#6V3vh7v*9ov(x@v%Qe4GLh^|svD9qN}X5h z7bruV&mT{w#JDovdco!}f*6uYg^K_9ds*h+2gq{|tbV*;`^few6=!N@KLS6Ne2z$p zm|2U@({#AB9&%)1n5-9Qg%~2qK)ZU$azgO>=?CIR*WmWTjBIJavOTQ~KFAZN zxpKs%aF9~O$>YZYBv5JyD*$!C3g*=sL8;Q?HuQG-dUHGSMrH1ie!IA1?#pd9#RV2! zJJGeH7OC5(qs_5eXW)WCf!23t;4f&oq4Zk4z16?VA=Ku>IVCH+uBR2(TLkTS*53SI zRW`e|O0UzJ6AbLVXpTK$^Rn&w$(1F4d~efp$qMU`rSZZNw2 zV0nUo!U=4%zW)Dok}FIPmeU8;>-_5Tdg)J2r`KV;T}`ep+q*9S>_K}y#$quMgNYbS z#9&&9qnl%g?FkO{s{I?gZapaDp;x{rb2*$I=6(+W{s0qzV6sv8lD88Aqx~N=>>p)J zlrd4p^mOx~aQb*@mHb?GBSV>z zZSE*(2PV{qY^Xd3kJA0j1K;=I2-(#GqANS}V!3GH@H@FN`r%psQdsCimQt|1GeByM z;mbwv_Y2IgU*gR4>1&v~p#AAkq}>q;kKCyj{5u=$Dzww0Zg+d?pn2$;3+i80#l3ZI zqxi$Q$uh__o~$+z;ZjAN><-w~3A6`4&JOO31L5{^JYPXB7HIWc?15eEeV@l9z<Ra-f+fWRXft51PVp9?1q?XP12pd#1O!C(E(VZ5dq zDOLf$BT|FtZLpHRDqX9sueu*P_3zIEQRMx&P?1MKC;~ze5NaomPKPM!qNuxPQMa2T zB#OEy>Y}K-AKyojPw;vad81H2f@Lhy!+mOLYihfwV;_`nNwNXgM@Me@ zDssa!cO+TpxhVPSh`J>6mVCp^V89BZN-++VEfvVy;u7|elYC}Lf%+K*PVL#RP~_7* z3ZrMBPF%_8kO4KKM*CvG3KPsXMuG;7Gh&Rg5f!U+B|Iy|9{~bhmwa*Lly*|^V3*-^ zJk(ZIM)a*xf%Nj(Kv6}y6gjwsywr_uvDtIlSV}-ZwnpMUS5yVk-`kiq<{sCRFM(Mm zXJ2)510rio5THbXc=8iv023)?)PE&TxW{4k!rty}(w7&Ky4wv-^g!0wYI_^A;-U#9 z3C@g`VMhCW5wx`Y-jG4YQfBql=Q)xJsms#5+h01>LY=Z?qb9#z2dp4`C(hz_G4z{x zVHLEktdQCoX5OS~53dBcePKRRu4<9Wr_@)eyB;0=2^?LL(!VVKJEz+`LjtAppBHj@ z%OCGG7I73rqjLk%;PigKcokHO4rj1E#v?mlXMEfaj|(0OU5=S(qwC#alIa41S~;~< zSL4>HsU&cfMn~Tl<>=G;PTVB)Nj3=;`8(D9dazo)QvJ!}F*U=chsEVLHOsK&I2C1e z2ihc5DbP|NIFQF2CG(}#x;mAKX0QGaKs-&22}5%BW0(PbPoby&I^C6>OcQWkeVj~3 zPP0rSd^c4oZ5m{n%Vv@qd)31~8e+aADOPuG-focCC~&Mz66OHjK&;eqsm2rP>KqLV zR|y+*J#W3$MA^p}5BAIF=YxG$^WETDl)9BTI%K2pjKcGC6rQ_!b))c% z!ZQlb!EWBai&rnaNseA1YIC%IC4$P^R>`9AxMnTog21h|>1ff*1i^`*0`Jx7#P@l-Z{0W?5ySbUo5}4?04qFA4$RFtPN(lhi=o}p2 zmAA89YKLz^xNj4(kkEe07AT=C<%Sf1=`Ev6)VJNK zLH4W4%X_Qtaa>B(NV@@byItGQz?e->l9qA%uRH`b$N-`$3Z+`2QBrP>lnW3NDpk+P zA9k;*~gFh-Y9jk7!Rd}!C<{~%zzr6QqRKDZTF7JR=t#FQ;{sSBg+YX9zh#tN#%EPDK zyt{Ac`%s@i&~{bnPaap5sb9|sm#gYVe`>j^ssj%W=c;n~^u54J;@lUo!+ro$yW(Ny zRqt+$tU|Xs`!Qa*KJ7-zAjs4L+9o4#9x^^FN0FZYgIJeoU&Oj1))ld?R^sR=ilQru zuKRb>M9~#RR}@`?-MoJ{jb3=u+%%5E;QaXNK|KSSlF!F8V0wM3XCSEK?fZ57_ue#! z>>kQn9*Y;`3|fZA)$bTip9JjK=uu&U>Xhs8N?||2l`b=yT!VxbB+ScpzkBQ2x`nKJ z-`na+m1h*%szhelg|)I3*oC&jH?`W}2`a07O;fy5eh!~kXK1dfsCKR8ZLPvQy>Lh< zd2d@HP1CHMk(d}A$vCWkpPM0lJ0Y!Ykj*aq%(p_Cc@VfGa^#l4W*u?I(eKkAp%+rm ztwbdjW4xm;4H8o#Dsey30rw-lP=n*Tv&j0=aq-wgs>j6P;X&;ii>zGxcko=<3cTUn z_NncMID6?;OIG;m+x?R8H^p|Tui~p%nZ7D}d~+FEolDKPLBpc?vb!nWg?&l(qLhH3 z?pE0@+Fkx*>#e4sx_RRPUKM4n3qO8l_eg}#B77F%vv%U>`H3pQKm(i zcJD4BYuN>QgdE(dZykd2s;4e!v&G~UB5TQvPRD~YatLt;f}c4En(DZ*`zF|LGogDb zR^IKRJ_ksKpTE6)jfs=Zfq`?rbnaQj5B}~e`#^RiAx7gum>lqh97ie{bNfIVV2u;xi$}@VmMPwhhQ@M8s z-uLxXrGXaK^PU3}`%b{?AK>U-`qJG?yKHpPs-~`~<^J$^K6;!U`b*2_ahUxqNx_)1 ztGY0e7qKvTu6D+WlmohoDkv3h4l0LM8!VC;s!Pjtoo{FwQY9u_K$0L}roqT2m)TzH zT%B0t%jWk0g319fkVwb?T!unt57ZAb2p2`5C;~+hC~75+&W|YJqJ+C23Ad|PBucm_ z;i81QA6LjEFG!=%o)eoAfp5I1f*USbTDwPr@CZ%!RWOc`Vh)!XddVmes6B~?s71BQ zB&gDrw>Csvdsk+9BSngC9G@^xICi`@Aje3OGPAS4o_jsUT=BoheRPIDPxQo&WYJ6Owz%no zH00&fwK$Ki9O%6A@!a+{GPnOaDZ2nSGOnMY`=Le>(tiQc*h1VgZdC5w{a4PsNk+@5 zdbnA%+Z>R};bK3di1~R&2`20Q5`6w3u$LM6#ZILdW-&le46|rIG0Y-{S+w#x!z?VA zEG*_sXS^@9b_eEZkl20duXU-xwsnca1Z8qSt6}*a!_t?WzxbF#k~+nCpCFekL1mRD z+zYpE(r0hm*(lwizOW5kAKr|XwUB%4qpZ%R;#+Ggp=;8t?-f~_C8Mv3{` zf~`Nb8u;)uIHu!O9`@kv)f`yp7RRa3oeHXlU(*ylL~+S9_k8&9N2P#BdNEo1*!{@j zc3e6*Lu41LQ!W_Okm>8imVTcQ7r+a-qF6=GEflefNhqRQKZ z=-J^2FUMQ`sfsUe(x;!stV8i9B1IV*OABZXGn z>xTnlho~>`ZDb(>E95{ffxw`r@ED$mz<(hKywsKAr2r4@Qh*8(s680+RmW|iMo)LY z!&jjjQM)RLE_KiTiAx>Mjlk^&i4D@h(h4Y1RyR!V{P0k93FVx5qy< zAdQ7p&Ja-JB>sm76=$W2AkKrM|AL`?ScUHCu*zlNyw1LpU(Y=bzja-i zLo`>4dICvF9xCbOfRa8UyS-~DlCAdkv6bfD(WvSEkT+(gxgn_Pe>EW^{uJ@2h(EOw zM-Rw-$+-Xg`oEM1;)frMwEN~KlPb!({mDA3mu;!%?(0t}^QL=XqRfjjFUq`uZr-QQ zMJ~8NM~lU-`WBvAd`&(Y7LA!=etkfYd62eY3iX3-a3Yk+>^+%eY*ET?@6Fx2NPdQ`Y$YCf_z_l0J_U>LE*^s`itzZaR?D?pjTgI_;{`S2YurR;Lh|%sFXSZvj zr%TG6MY~b$9xm@d^$wA5aHe|v6blM=^969Bc+$cULGh%8_7hK9h$k(y^18&4)KLz- ztFn)M>1ROTj&VkhWN7-?4_njED0gKDi|ZwEuZ%9J0Rv3WLG^)HUl7&(Y|ugUVVPkN z?fs0=LHps@V-Vd#zWIGI%mAVWqhW$}VfyrX%}(FHclql_5?f+y<;je`g2FbTj*H+J z6Th*ay{g92dwv74F72SEo~i9M{D`-%sKEo&byW`IA!|EOgbG9Ju#{vsg3|M0fSuuQ|4gw2@DYweFE(#`UIj+pq1AJ zWyL)wpNsW6Y>qON6iklSKj?B7qjXW0^FMUiGEMDI9Rsydzt>D<|7h8%cDDUnt5-;H z-EK_QmF1#(2>fM(2;fM_w?`>j!6Fk!Ix)D#StgkXOj9Z5bT)kvjUEwcfuyf!rO7FZAxTG4NA`I7o(toV!PBAL{f{lhFOM^}`Yle=b*bl$sY z*<*NYd#dFq!K*uC@=IpqX+YznuH%*csuy>ezW2mtN`CaEIWHp!6G4~=!n6{{Jt$G1 zsP&0Thxk+dd$yu1K{I^c)r~n*2;<>sGYiy&e8b9S-A*(rTKSIwMETiQ~HW+YEH6%oH?qT7wu>k4y*> zS`ZP~Jm1D1ejVICnW=V9-ltO9ZJh5U!m{w5PmQAc(ixijlISKlQs`EAKlIZOhWL+g zkv8~lu%mPe>-BmwI&(~=1eX`aS3l^SZ_ddeAANL~d?Dd}77i{(lwi$6fjUuiKmmTU8<)dL%3hO2JkR7fxSh1T(R@3#>Au z>!}BHg)PCY7q__#GmDlC2w+-4LEWm zpBL6dEA8Tiz0yVKx+*WMdE4>AmKEyJcX(mVm&yyvtKtvfh1EDp8I5B#q=6KwROYqr zwhz{u&%y6zM#GN{nzl!5Fk*ud8yt{xKzFe#Wvf`(BbeC&c?N7|@_>A1pBE>b?j?zw zFml4k3H!SF;d8=hJvuMn?aD%(%$tBcqRu{{G|M?1dK?{N`eheJfEuIM9nZHSD9&tl-#>DJ?5!TaZ6- ziVuHIvCQqg*p^jpUUx0>vQED>PGH`)$LmTC;p5ZCLK%Gt^Qtz7sZ9m*p}#`CUtX{GRdWlV7ErRhhEjbEBs5F;P(45`Z-KQ z0keuoyMz5!LA&byIv<`wZp|RsV4b*%GOPFkc>eY>YT!`=j~e)74csiHVePxpPp@1j zs`LaWt0PUyOaq%M4DU7i4Os&S2)gwj;?ft2_6bfJ;`S+mojlX;g42>|pzp*;^<)m- zsda);rdA`~ZVRD;kKsxnR_T^=urQKvyt?1@%mMA4vn70QbBB$qa>kA^tR_#|4N9)L z>qMcxkk<{)QidEs)nQZ4ceDTW`C|+7uwY!qC<_aSJ>;sJ-UD06uFYPQdkEBp3G`o~3I}+wI~B9Nu$z1xHMo7Gd2V(St7qJt*>=GO;X3nxUN_1YOXNs-QNGV1Mg^ z2@Mjwh{Cci*}@d;y!YYc*YyF5n8P@KFy{pV?-#E;n1%XYe>P_s$d@B4c@CBgqm})# zY$Udd=^Q9gCvQ}J(Ru$_n6vi8>~SzPM!n7UqInz5+i2c~nz#1K3-mB`MNR)wY**fI zt&L=igS{o6l1mR%>00qu?w1D zbv7nauupHwT#CE8hX{(hy4z3O)%|*E&^lM?*(A^vJJji*$#xgs2a>k{I$|%O7+4o&F?0nqw#JG1k>8q2oeq9*`a z7J8-fwCH=){m>zDPw3h(fMpGlW6TzJb?lcnzgFDu`3&nQYB5a6~L43fbB^(~` zkrUB@hz>+_pqJ8!4($ERw^H95V?+D($!?oU7(oiEnWb`$YVR70bGqH)QB*<6Yb*9z zTJ~!!K%&a%(l#Cbwl&ZZwC$n=PwCR>4J)yfu@H{x^agShbot$QB*j1_^`ZPuK&5+A zm@Q_5Nb^7@V{EygdTw0Weotq&P1XHP>j?UMQJ{|r6Wbm2F?$%Cu#frX!F@^+{J>B9 z%CttlgCaksj(wvq4=)hK9;ZR5qU@E3RYa^JVimoTaU@nz?f{ztR@j`ke<><*ELA61 zaZ&ehU}BI9H6c{pdx*>6hXG#7j-80Sg!OHNUWN#Y&`bM?&`X3~T6ta2%YE6vhT(-7 z!AeaS7iMTwil$h3zj$>>tGl7Z$4kIJwW)h3gH~@PbX&6Jd0&RRyfmkMm8W)T3!%1A zxL{DA_1zix3tDa{)fZOY>fhxEYK``sk`-RplTlWKlg8}SOZGAEC3PLEBY$8|k+GRnjm_>ZSgpOmerQZ3L8MzG*T?$%qum5Wq+J3G_%|>Y(rEQe9_v3TXYc0^(;y9_u z58Yo0NYbVEB5L=MEig$C=dWq%z_t)^ojEiE?3Oy;ehk6WKEvDGU(p0+&a1wCJ(Zek zuN|M3h>ICottjgvwlhUTB4(MaF_g?JwVH$a)X@2oQ6f;AwSa*slwy8;g)()~eSi}LHfUNfi;kPDk1Iv=Lw zP54#$6K>a+S?+JmkYc+;?9VH61AO2XO=eABN7f#un`bal_ccgG1m#ejU0bOek|n3$ z$X*@YS>o-ArovO#d)7Kou;3_@#+TIbHjslQLi$Ggk~F0USu}w2#l>Ssg+o5`gc5BS zB#s$7+{>mA=J(4Ex*-)b?sb-WdvavyeG^PSzSH*fGxp5AKyo}zPf!|3Q#+F(0r9|C z^o^u|{Mwh>%~vSPB2?hmY|uAkRMK+NVeAsba5t3QHFX7Tz2DSx}V3;c9_!! zm}YX%3qgN`86+-MC|u_DXmS~XeX3laqSvcI++b-EpVLBGs~4vJFsU~E00p-X+{};* zDBz`>`z%Nsp47X3WoIS3U7d{@)CK`FWCYDZm|F zGMKZ6N5;UwA%bFHVEc)IfiWI%mB7xWz69n)Iy!~)~T*XDn ziM_#GP<96>1H@z64A7F_yz$y|gIRq8Fe-C0x#C;Ot}_Z#?*st-y-3sJznCIMc4DuU zKP25%8k+i>JF3xDIem=|D5>nky^vKF_O6FBk1T8$f9U70U;t9-DPJz^38-|P&BYG} zGG3rHgQ+J$Z6OQcTtp!~{cIX)iCd*D;t_1KVE`EO6-c09Ed35~>dS2%=cs z&-h}>g^McZRp9Y^zsm#rzKYKwo;o<4rN@gG&o5rTm~Yel)^T!ja`NoyQ}y@cDc zVQyr3R8em|NM&qo0PMZ{b{jX6I6VI@p8`u~j%|&ay4ucaH*-EkQetf!nMac2vzz2+ zgWVtrs|oY~Xi1Ed-?Mzr@;%viP`EVu!X`!1vNNMTIWg6ZOQBFGR28ZUDQBZG7i{Zj zjD_k=@g)7rx}WXs?d|9L`|$7f_ICB(-DfX${<5?GYwpZ?}I=DZ{gBjxrE6SNX=pYnhg;PH2#aOA7^eErLx#AIJQsFc`K#p9Z zSR~&}u!_fxm0}_k9a54UAc0vtCZe%eLS>fXX~gh^9H3XW?Z%>4`G7>%L`cfn0ebxR zZ_rL}cc-@JwU7peSkxI@fJizf4Wuu5r@Th>iSY6wDy`A39-}20?qR-9v?c>3}#%WIE zClLVJ_y67f7cZVy_y64&yAS*SeLRoQF&W}KRmczkaA?nQI*-uhm`cOk|L5?>SJ9A* z309iF4{1ttfnyST#lZGQDIeg}3LGjW z`WLc$goWpvj5K?l9-wX)zS<4>$ZTw;211X}3DdnIrQuW@;WXtp1%Ff=iJT$MkQx(o zZnj{QkSyg>lAyPyQMX;%MhP^T9H1diCGkC7@{|ay2zsR@1jKkkkmkM0mQd!qlcn?% zp(~hTED@A-Et(5Mkr7XIUkkk>?k=I&tUU$w8Y_|V*u-LOwJ4J_GT^PWFrTzQspp}x&o zl9C^&x@%vyel&mvqARLO9c}lvd%G_*N0pWauu@pJcGr$#Nh4Rc)Wu$Jzqbu1H?p(`O&UEDpYQBaMWO|=o^()JiO zaZa+7#`uWmOdX)T;_ES1_!W9LJ$b~{&jfuFSAYBM8LD7lB z@i9q>((fm9B(T!83rR$fI2VGl(SP$nc~3V&ODMB^;Dc9Gssr@%ubncxn^nVTkpCBc zCs<*FQ-hl7TR;p^T5q-A$ZbYqMUta2iLYcnG4Cx_&T2wPseAOuVG0eVu8vtgeiw%+ z5M;uy$%S1M@;KMX_>QK;2m+UaP7V+qF-_;_5$al5u8X)pT{jus0RJ7!Ftv=35_QhG zI;O(F?$}^?uJxCIh1EY=eo+4qtmW#*PFXGj2~tm4o?T)wBDR6>&5kq><39xP46J)hnzqkm#9`rMc4--N$NNds&dV88!^* zf)CXpQ}l2+q^$b!Fjw5JSc?*4PI-MzMBm|*vQeLim@uVjH7xccWpskm!)r`aJ(Ja9 zc{UO_AwYpdl;Ew;hW+4V*9(OejW0|`mP-^k*8q?{Pt#MTh`7e7`8dQhCCRsDS}tGt z`PZcl(!;rs?z??;k;gHSa+s(3<4dZmO<&OoRh5F@K{w5XN`i$cST9tQuA$~4va_Lf z$^})^04q3G`~s)f9`*W#VO@Qmyd4u(rnxuFv4_-Bgk6kQqNi;1njLYGaArubL^l7+ z8EK{~kQDe0>+=M9r<^%P>X*YANE4W{w^WT^VX0mcoT~BEl0?vuE)-UuTn**Pt=g|L zI}5vQ;G7I{nkFTP^(Vrp98U;SZ!uM6x`{c{Q%jgCx6vm;LmSLoQPLpP(Av2nv3hY2 znORCe?rtTN=DW`m37bJb*xxPR#fnCR7Z0;c6Re zw-Zt=Q0S0WR?&lyF;0m%arOdTj*D!J*`Zqv#bPsz<&1!3+HoLVf{l(i8&WX=Tz{8p z=}Uh#dZY#Cl1>QEEBlD%!Fre@;U}Z<)9GYF5{i{!jE6%-gqs>S#Vxgu1ZN-kz>)U? z`>)}QvO?P>G}&dMYhFxQ{8FjxD5Zp{V4#_x*H{s63i6v)xZzp3e)>kFl^4>{y?fKg zQr;xxcSTbjClv?EV9Kq9;?W~?sMiHaoS_U0g5GB-rtH0zE#7~SoW1WsVPhmaK=8Gh zeTKFoBcDBbgieQO%5!vsnL>&qNmN*_Y-q7SqGL=ZoLvyLX{d>Sq3iw6zrIJ*7`;-; z2*Lw?O-wJCB`9YxpJ>ix>}KQ^Pckif!5IcZkG$dPicGukqS^BFD6nF|zwYG>8&=5_ zg+BfNW^nup zgVa#h25Q?}^Dv99-B^P+r2&^q?lzR%?G~%YsO<-zi0n=4=-3!=-48v`e;N8;6a}Ai zA$V$db9zuPdwGC%KM`HLIsI28g_co&5wm~9nLPk~O9o@kukME!V&5&gG5!8462~2| z@I?^IVKSle-f0E9UjLOzWpN~Y5tQPzJgo{7mfR1a2-_*>n}j9*igdCt4n8ZHP&N`o zf(7-_BXj|i3if{eudxRsTLwlX5;QUv3(8ricKmOeIZP5)38?A}PoO|Zc|erMt=B~=%L5JlDH_^RW2~ShM*>zmK{qr_(SYb}hk>0BVTCd-6x!e4 z1I;MT1+h!E(&y*p*VIRF%pI$C~);QQ4zMrr38}AZ*}IsZ@v$ zQO2dD;j#)K-9#Dqt|Cb+2M}b~=<`|h@0Ll#MwH!d4N^XEuAB*vWW>1DeSD3I8CRT#3fOkP%SpBrs`5tTIvS~hSF57f@rf; z1W}HG6_aWFj`f6sLlwVaC?NR))42@%p^^kBFD$6(#`a-no=kx+hYLuKd7dUnDlGJ3 zQ)3kA=i8I-PtVZN$@%5!cc({(mnZNRaba2t%A8*uqW<~mPluN$=!cX4fiJ;SG0i}2 zC$LB4ndt@>D;8#Pm$$6SQesrgkBGHs0&ev z7*}?3Jwd^HoJw90x#F%ZjWctYUB@xcm81#L%LBj~lXQaOF&3tI)c30bou3>Y|9H}y zBmmJwvoy+#5z*{OS|})c_XZry?E0Dy*q;Fy2c6C%1S_`ix<>z{cbeU2n|jJ|X-gkVl*A^t>dGH7Y20*_Gd9!)X+c7oX zWe>du*Ls}xI=Es*bPgAXaSW%JFyFI#ayv6N#VagJHyHT;dhg7wv)7j=7d>^W*5zY5 zC;n%9@7Zp}|9Ah{&hrQV-}`vN_zlKykd5>qi4zztZ}|64$17lE2(U<{%Ei!JXWMC3tFm1l{6r&cu{aMR7`hBT28* z`H4!Zkmg(ok8|Cc^-Wb?tO$d0KxFt*9PYocr(ylS8woXXPSyYa z+5W@&zn|yh$E^)?O(zFN)S@X-(~NwhwJscw$pPBf`s0s|F4Z}?&GZu0OaWv!5NwDM zX?h)7F49VpUe#T{0(Idoii*#mL-kC+DX7uCpG@$HF6lN>^t%>^S&W|V!B0B5$cICE zi@H(KnASh_Zv)_w70P@cT1T0pf8;oYU_^wo3_$O7-V)O~lvlbBJq(EkB*t2LmV81& zkqHJoq$Ewu!EZ_@RI9kK(^83^45omL1AbHw`7Y$t*H(x$haw>-_hXtK4I}w{|f~pHdG&>t&L8h6W{?!r88y`V}j5d zYJx?AUTOuz6wzcEuE_En@{B>LUk&ANZOSBbs(P_K-$u9vY##XyCS)9{L?I$`Zm9 z+SjXBo z-HmRk2lR8d`)jE+^m8}kiR^BoF1gkktN!IRCf$gPx*^lxjaH9ujCGcd6vBSe_jh0rm<#3rt9V@R-~%5R92hs%`R!z2 z(QKUUgi4Wv{cVsZBMYOR3~LRAsi(49S`hVy}HlQ%HdnW(nT#53Er z-(Az2sA~w&1hh;XHP>DSq!~-<$uXAHMA0oUaO)Y|n+Kv@Kel)cQum?ZmCg3i^2H5f z9ZUr_z`??Qm$3h z8sGWi)y^SiHmq5C5%gjm?h7zWFJ`tMmI}LRdOxE#z1`RuYWdU(L7pjyp7IU&7nMHq zEFgf=i(0|(=giK1ERW+?r58b)CV(LP8PZO;Ec_p{=hVaonV4Zyr}QRl86@gpK&2P8 zf;Psf;mx9pFXmh%l!byueG!86JHM?7QG&*DHm|iCxmh`P+0v+s8*T*?XVmY_rPiU@ zO=S6by?uA8woosVTw%>Qtdo}6^mFU21bhmoG-)X^bKtOj608X9cW&=79zAuaX9|jS zkhLUesGX-|Kv`lKAL<5L#*Ehuv;MiI<$u1XUH`K)v!WU}NB`T|+uN(?e=nZzzj)C9 z?&Dcr|NGX^Mw=R_$!7&Mkf-FF40XGX`?SE=L8z{xHY=twA3!!Ztzw!w7o~IcdNs@2 zBU~zA^9zP%sRr{97^I>o>U>gEm$QR>5lD9_2=yMvmO*-TNO#2f=U!TWil>49dxbrc zcr6CNcK*M!`(nSE|MmIa-oyFty*wdP!dWH@{^Q+}N^ECIwgUNxvMgBLMY5#IvM+zQ z!+Mt)i4QtK!VsHpOU2_?25PGv%>u+N0b4N`HRPJP^T@w;BX8?eKe|b9eRQV-p`)*F zqvCH`R8{WCNVN6lO3YIuXGDl%$$O(n9bg$KbTQ7fMa zmQRfI^nCp2M+X&p)l=nOk+BH<@yEgZ@(Pba1jl?b!J2S?ek{k2H__uL*8jg5)0Ci} z(N|C6T%;(PpdA!R6mhhbkn1g$rzwhVQ1`DN$4X`IVxEvcx+uD;c@otzaPS}d`+Hm0 zJ6mH&C-K`;^zYxG?(X(>7ybI5DB+02V-9GfZ1kTZ`EGX0u!7D1XOhANAG%9Vm}zj~`Le~$bahLhRdevI z5y(oKz=ff$)@uU0l+JIBFo(GY7Y4XgsR`P~tbl8Tx*>7l!oWAxZNzvv+u$caeI*^@ z;^?ndYyyA%@UF%*E6A;>lQpI`tB4-D4B4RHKTN*7|9G~&9WYe0lTv^y9W*e}rAz{`tudDhcHUZ@Oz^m^lKx(=739Ki zs(FpYR!Rp&n(kUT!81IvDR91}2^AzZ{`WJxibf)fZRkZ)pDLBnkfuZyoe_3jM^gXw z@w>CbA5Sj&hesztTM*9mo#2!5B!K&&)y-?~P1jClE!;~QJGEIBSMAN%`Fj~pf(Bhb zXRnV>V07#0DkE`^@r0b=i5rtWjj4)FA?azlfd)bu!%R{@f{g^&QZ*#RnPk{fe-=&7 zr!(UU5H@MvoXw?AZi7lna@B%fTs9WVi#=ZOyC~w+LihuwxM-6#(W(MkRcI>(co(%w zug|K$v^VlQ`bW+cnGX{eGg}OU$D2L$%2~{NF4ENZ(U3#jcZ^wbw^kP_9Kvjz?3_M% z*V?}$t9?EXMdSvldCug{g{ZB)-a z&EH$rtl|5f?P}J={_b|QxzkzCd1k7W3|8~!xS<(8)B6P=c1oQc%cPGExV&wq5WuII zW$KcE`KHJpY7!2D@KQD!&O)h|zM}@ox*{YSuM#7bhs7$$tb*%OU%S~YQen4@v05p> z(;WM6ll|ZLR7ciLY4sz3IrqQ3*xs$if4*cEA&JU>QtgGW#AuEL?2XAZui3M+E39Rf0(}`WfO1*3$U%G zF(U*!&#ItQ6Mfe}i(+3QvMR!&z*gH{3-F8NAlHL{a*_9gi$3r{1oryK=yp-!6PC9O zXOZ64GOX5w>3Te-fL|F2l~++}%D`0=vUMD-&97wum2~bJWDhG=I5PTa3y75l)&kUw ztnnH+`zexqs3Oc1jj-RlTcDiL;yR%X(%6T9!eyBQm)?Fph(R-Jfo^UxeGRNF6!B{! zimO=k`pbfB`YpiE?PZ-~nVs=J9LvO%io6tl$Fa=rWi4YVUrV3?w{N1@-Bi#5W4ZP^ zIYD?QLdXpqJ)1Nm3vh(6p>?w=cgaMJU8Vl^iN#o!YiOMS&b~Y%1h>#Kp;+X|)~1G< z5B}_K)+lY%uDB=!4fqsgJWV6&)9TL^F=`F21=z}!8yZOSDsn=RY6q^;z?$}jSyz6T z{lOs6iZvyPr+xlsx4R~==iWqO;CBE^Uk+f8#-?PHnoAp6M>4mo^$w}|&Xn?SEa5CW zl;$qhv;DQ->H?cz`Ha?%W$|Im>7rp6&P4j;ZeyujKJ)Y`dU6*-=Tip&v=gb|7MqVB z(O0(8ajDbGa+m1f89*TG5pSf6UgiXrwn!fvx}a@e3Ic<)jBQ4UoloMz0?-C(rM}C zXWEWj{*5BVm$_%LxTtOBstmzsU6PFjKhJtx0HYNuUS#mgtO@-$t9U(V}geY@3Sm(3QNmL;s% zRxmqPVyn0_AUgMG;0}GSO?qv=C)W$Du_ro(sCQIJ1r?ff7~oCzD>>PV(r6dx-Ypt4tMR-k^^b#WGy3^ zeW$Y?L-ExSnTraFv%D4f+3l_w?Ale+6}X*raCg9KEw2Ug`FCLJ0TiVop98{|i~Q|c zE#sKq&sv64zp7hRA$J@}C0F1CQN(`ga|eZF@~h_uPB@ zjoz(``B{-slAD8bUoui~<$tU0J-4ehIflrJDjaM|`eRMoUSuzt+3GrN9~v7MQx?Be zDl<_D0l#ukYn(@N%Hn9OR2IchZXxzHua|HrpW7q-;knPJG5*7)WR6VI-&G?(=H!3c z+kLiQjsMu+eaQcJFHcMUCr<{gQ6NAek)fGovNkOp1>yq0ikqKz?w#x=s#L0i{5Yf8e@5RKn7U0)cYmrcpkkYK4b#Zo27 z668Swv{)pS3(m;K@@3>>DMOfxwDe^Tu?1yI89wIc_?VdtV|F@>MKgW0=ipei~h^b*JHSYDwWXMSW>Vgr9_=`u8yfF)4T)uhd1AzygIpj zcmDeI<-6n4^E%9>g*30(hIsf!k2OP}x;1kFLz$RcO4XLV8_3bc>ASnuFgAzlqt z#=$}JUz=|Q&p}yJg;vx7LpeS_{psZV-TBG)rx%y!hwDb-oKeg|=IpVohEzc${cRrY zxk9bTJ3qNN{ol3ITmyWwP*jF;HMBT`keY{@uwW~CF3t}77cXC5UalQQGhm&CraHu{ zp(>@<#3x&8M^?FLE4p5t{&aG7a&e*A*|#Ta#@I}dXW^^~@I|o}?yGL=sKgV_+w0Cx z`maup4&TiXIcnQ?V>h(~Pjx=kiEhV_f*`a(Ri8w&p4;OlTf~Wob$2)#myVr6jL38eZdcL#0 zT|WQW-PwD%|LK07JD>loEm2{Wwi+&LYA&(Fl_13#QG#$t{UyPZ z1x@nA^>j%@%?=F1t?%3J0qmdLX2K+Ll=Tu3x*}7c-~y0fzu(;svd!P^{*Rf6 z`_#$zpEo4m2ViQDESrMA+yh)~j?-h3Vy)wGmdFFN7cOBUC{O&SXa3WdlaT(?CyM{` z>D|>WX(a``y|Gm^@oLKtD)}lZc^fucmAAHv>sQjNy*+5_t7_}57`@ZX-@pF;x%HQa zXSJtY|Ig_Hs-EX`Va+_J>r%?w?sG#2h>a+_jba>+Lx;QSO>6Ld3(8h+ zYAf1+%LUpRfdb8~9H46~B9XUU_a^e-mbZB-``6nID*ITetXWeo;CS`=?E9Cm&(Gfd zczAaB{mFUxQnkYUqRoS}+t%)z(|7&X=Z(kGT@AIXT)PI1Ofoe+rs4p7ET-2qUZ2H9 ziiZ={TE09te?GPQ&FN~IcCG#@%JrMm2Yq@aeY%3cRz<2(cKurOo6`qHd1Xbpio|(S zx}rK9MN#F(uF7pWpH41D4Hs9tOP}Iq2Nrdbmq3?0L*<}mV97O1u#=d!tV`tVYybJRB31p>m#U?a-7&ZIK*im zJ!u2@RZzBT^Wv84&jPK|ODm9f0bx#87d#g+Imc{7WF6e_tD)S=T3V#l8bOza)poBM zPU{3PSI?PEka9liVH7~XF_TUZD?hVp{x=-y7vcJDqGcJYTjQ!VT{XxPWKDSTD zpU>b^XZtJJf95r}iaDs>_Jeh373)wtK3gpd?N!%rJF55hU`krmlr#sYcbxzG_q(WG zE`PtWJ5DB4wu|nwimY35KXm=9MR0F|`X1%>rtTlacK@tHY;Q(!@hmR2H>YQ6PR{h@ zi|=dA(G*>QkxqsOpLG-*Xz6w9dS}brH}cz3shjRm@mN4jn@wF`$TWYkTH0*PeGVlp zfaae|M|<$QZvt(mFtL(4IH$E$w82`_56a*w%3v!xTXlifn(J2uYu!C)f~#tR?I^w5 zv?{d>x+O=9%OFy|=x$S2_QCw!gdkaQ=56Pi0ZKxeu6xEg{L|nusGl z$*>@(`}c2ldOQCSZFf=c2xsP&BuZrOhkQV+QKI)fRrV*`j2fCN>4nHc#P%a)C{04Ww9OOJ*Fx~HATOG0W-!`xb4oE5fH$M4fOG%i2GVr zR83juYAR@>JE5ukM8CY9IH+zSxP2^nVG)GNKA(`uJm6^%;lV# zOZve)#<38f<$nC;Sk**kd#{BL=wmhW5X&b6t-OA{T?0RK0jfZ7wm^@zXg_phYQW=+ z6|Yje&{P`GoF$=X9_dL?fo2lBN@bxWRbct^do?{gi#`qVU&f_!*YSPegVp80Ir88B z{?7BN|JU~OhxosHc|tL3fK@y$4w3$w4@zQGO;YKONjmAt@s`$ox(7%Tj=RXQUZA*$ zJ*Dxqdw}4j%;T7lge02-GUS3p@Ydb^R2SYVecbnD6F64%nj90Hq?8e#SK~jx?5TG_ zJ{-O>H&^b=JvwS18d~jeDLgb2UO_ZV zQw~|L*>{+xx%l&1W$h)Fs~KaP8<}r|T_T&|Z1bH1;=x+GinX>0i>-FLrV8t~&AMJ5 zOtPz*WE(NL&)3Xi&6U>>retC#fN$kth{!#fw|Ma*3 z!MjKR&k=b_4$!%NdB~>VrS^)l{C39Iq5NL(JeyTF&z#9Fg>rI6!5~opI6(~Q=trEv zUkSfqH&`Tx{nJhu;}3rrOA`Fd1ph!{MG|k0(z0mH8Bq*y+{3vVb3uQD$q20chIGz( zO6E{YBy^b-jP7?-4y|!FLO7!ZA|p%sKu8h-AN~8pX&6w15~;CF#K0EFB?)S2(b0Y= zt%)U(_bp`!WutjG9N1#_h(?|gtGPHtwhXF+P(To~$w1}U)CEiw!LliUBcpYPW0f8E)B{;>bw$1}$o0EReHJ}J19@t~41 z!L2CRQR=Mgjz)z0vi_Nz1LZT;gvVDzm~Hffi_rb#RLDp zm#2-Z{d45~s3(+?Ci=`r%%P){5~l38+4+hZp{nICni1+%i3JJs8I>~v%-d`J86rB@ zHz!R%)uyC5wy0%BO5s1BXF~4tdZhJDM*~c?*=W3-rEUPj z0G32_>`;7w&Nw4AWO~pj--`Jp|GoUCH^LMzL zlS6yr^dnCoPGCY3ij`?S!var;3LFfLQ~m4VxzF7(tIKfUl`FhmigT%OO6<2wIw3q) z2dKN}Q=kt8pL~nst2{eE-R=xOw|bY~$xELw7MhVJ24W7RzFK!dKemQ8e!LpWzD8Q$ z$2Rfr=7LbFz6=->IucmnK^hFdcsINMQoVY|`mgz&Yf+A}(J8oY8jphlityvi#Zqd4 z`BbjxxA~jxRYgN z_2)uRHu`Tqa9&ik32G{IfF93gDD@^k@WCqzp(>AOL8%e-ipMww?W27)90z5aJv!SL z<5}P|RvT`~6>r|UEVU#H&$Wgn`nflL`!vXZDwX!=cTFb1dFQ{+UR2|MpYQHIod4d- zvsnDEi4biFN!3KM+%a##wHKEhzq*U$_|;q+OXIn&N~M=ERVsV2OBx;%q3DptSP}F) z8a&@8EcWqvJ;++6i(Qdv>9bb8zu@?~hV`E_P>s`cJ<(qa#DDG9&VQfnZ$GU6`*_wG z|0ReG@>(GJt0~-Tvm6rEa6gB5GusbKh21p0pV6D96gxvLpIRZv zGvgNG8}KhGedbv}0HqhTg5l4ZoqHqp2R^Wr5xfZ6Ovrs<9WW09<63Yh2rx%;dZ|NY|m!};I6JddpPZKTkXlrZ!p<@_qoP`7Q#wDkvUfp{$dKE2_V#F_q?H5B80@!wdfS+6LT&Hnwlr@Qv} z_1_zlbVA`?*md_Ycm3}?tLT4wyASt2-Ouv~^|4Y!Fo_gLX8zI5m@qWRX$sqUhT|(d zBC^+cgf8_euofeU8l`+-ZuX^Yw21^sv7*-mE|m%1W0rItAx1{POwf}|kRiP# ziAhcK?@xQ^HA|<6GpMOSLKzV#rHu4Cz2l2_7g~_(JVHl>{vUbaa6dDrBcOqG}8N zZ@};L2EU0d_`iEG9&PFWxj*GK+bS9|1}!wCDUqFxp1jFA8@&O(>TLAXB^!Ubf9&o( zdszSX@;pMXECrw=y}5Nde@7b|rxU%vF;g2G2Tk(6uVXeu*d7UbU${hq?)AR#(MFu5 z)1GPkEf$Qjk#5?zqqk}HSUJ`L328?H1OFKcK0z3b>1a#@vYApex56mpH^r-7r}O9$ zGUg(5#1qo#`1YwpJKNiTMLZ)6nVlXK#vV~M&Ii4iPy8L(Ta|

EI>!lqkqRmJSI{ZC(}*mz%S9!#i@uphERw|<$|FP&(MUcz|LGK4xj^A&h_i-*#{qK;n zWWtpGIT9q(|Kgjg_u%^aevn?#eGM2QdQIK8?kApJFdz4_Jg$F7%rZhi8e6fzvcU++Fe>mBM#5R)&VF^jln?cT0 zu9wFsCsLu;7nX7*)r*-bhh15Q6{ngqUa{b7TBww@a03s5S^gkO`yF!c(C z4*RFQPUldwT0vx{~8` zT6d5UG2~)`u%>}1OZ5(BulX@E>vcg`6Ysr>`yPyWilg^nTzjuoU8)L-$^^OAf@C`N z1(nFjd%HEBaYYU`HadStr$aR5x$kt!bK!N`rpa|mDd1P53G~)$iV;TdwQA>@dXJuv zYfamxg9&CiPSYuhQ-T@Fv%=-!sYXB5LrhbY5~n}E)Q#CFD}&#G;8%KERahA&VdVh_ zugv4I*%pSmPrQdLIg`QY|U7U?x`K2LPw$js>^0*RaMkLbezf^gA z<<)k8Y}1Z+YE1#W7&6o@jmyP9civ>2A?u$}nrFq0{dCg|)e=LRS_(&cZ|h(53v{A{ zWGSDLMAIo%njpqF9TqFWGYC+!sDzj9Zcqn;-Z%mqs$VWZqp-#65_LA ziP5&6B01)HnxFyEizxAilbAETcFS_2ZW|z*Xd+FB!Zei{5o3M>Qvn%gG$$@*(ocq$ zB0)z55`kr>dtl@SH4hV5HqNqiS}t&*FiWsVq~)28%FKmxol6djRAbUZ4^uQC8vi(s zX##gGq|<^zNrIl-jA=a9>m&g%6RfnvC)LwUbfXuF5LJ1e>y{4{Mr6^*kq8MleKOUnfLytdp zA@Hs1JhT3Ij3UUy_05Ec5kcu#wxH;JX}%H+JEr0@wqBSBzv=pBVb=~yYXVETo<=jy znhg7-?=n9qb4s*{s&9@`OedSAEJT~=7#M(Fr1}e1WSFNHL~WvDkR{+JQSsAWr}Ler zxO2>6%`L616`kB>DHlYbH>aD3;cGh53|DbB z1?NG^<13d84Ov!S)^vO5wN_`GDCFdCy~U*Dnm|RLS4Vk{3-$%ACJm+@-d+H(xu!xx zosa}lbR-^*!Vm+n5FC$fyD%6zW${>WW`uHUMbT~PJ;aqW7NALO>ri7dktDs=Dx)_Wki>31A>#aC3-i14_T02KY-y%Pi}$dvOAvV# zM=~ChBnQZ?=Soo->e%}K-s!X(|LTUn1&n^5)a3Uwm@UyUk8^)*{bD=S)*$7BtqIn9 z>lTcsFoHkrJSfx;3iX3R-L6n)b&b!-tf24(`rQJm_JgjylCE9C{FkO`e{M17;hFLH z&F~*Nk-$6Fd-v;K=E?s%d(U^DSLOfx?cMzc`Tss1E$fcBpe7p5O01nAw^l{G>v%fR z<*%`zCSj~e$z(Yw9L9B!W2iprPQQ_{pqaU!wfjaEOt(-zOca?5@;gAf)UhoEElYo8 z!GLqUv2>7TkT>sSK5*c@qr>7l){(h{vRj;``aXgx_L73?m|7BBL2#8Hq%aJ4YeI$4 zYC+^`QXVy?lPV+QA|o*!PU}aD=4kWW_iB=f8U#pi7B;!<4YuZ1hr%;3f-YPn;$o;! zcvQy0^-gcUw}1E1UE>PM4j?hMH>{_X-o}fm(d>!Yb<+04)att+px~uK}5ZgPNz! zk`p|)olk+S73O{hyo*!9>;~2`QW-=stwt1OFQNr7Q+9eskOa&F*GP62xMSeJZ@IKp$;@FHR?OT3O422m=s*plT^%Dc{^0 zT~W2r8nR|>)A!1}qukmLRDBu-QpH!DNVa?1z14jvW?lM<-g@X2n1Kf=W>;{&kt?1MMPk)k5Iws6L)ZRdG@+{;8$Z-S z5uSsgIw9=!_IulR8@R%wPc!0~q;Q9!20qM7(?ih49L$6s)1Qbc`au_KCyWqVb=uhN z?euo89WlMwK*6&;|X!FdI&xh^)z?3b~zhKzeME20U)2xz&R-!ubPf8l^+AL0fb|%3CxX6oH#ov`-8$e(T3!WJ z#qr}}y@0z%Z7M!Ba?LP3YUdpZA1#2xZo~QS zLPN0gYWuJ^(SQ%ntXi{9mhJUHZKuaGSCMp=kv4_;gr(Ek4$#zK8!_u`@G2jWl&Evg)iD*NIceGaY%L@>S5ZR6@*Ca{wj~V! zI;H|HcbUJt&oTps=B@DF=>P`GRGIYl;7_=o!rmEo` zvoHceWoSJ}u)Vs^y+M$3H3Y+?pG3ij@=!0eNXC+kQ>KbtE{?S@u35f~tWaE#9^}fv zEI}80;3031TS#I@z7U&k+12kb#3Mlxy$|Emi)PJ3Mny>!6s;oxF7&OZJ+M+LmXgO5 zLR-8;u}!>WGPd0h3Lq>dOyBx;OjBZFHjfGL*pjSs2;s550{U-0Ku`slI_OlQQoRyh z9=iZu=TS~2viGrV7W56Gg5CuuSj$CeI(6qJgGoFCo zX;q1EVCbTVGh~aSqI{VmY_%k;R>m4PS@s=Fkh_!II;~!36)phfNEv3 zEFuo^A^Np6sw0Gz+w6wPW=ZhXrOXr!h}BOMpVoxj*wiL4@eoGGY@#G11mh$j$vsb( z{reaz+{bDhPLQ7&tT6H)%d_FiK&m<$(WuE@vujO=5X28rP-Ct!7wlHut!pf{QaUK& z$+}C7y-_!7XnJuxyy7uVshZBjMX5j;A8!egc#3c}<{jd#Y(@i9*=jU`*kDop+V9ME z7qE4?VG~mC4?r>4@Tm-#2#v?O2@~&F)arFEL&hmVprE82s8far=n@C4%u%q9`UzYX zO&XpsM}Q5Bu}PcqECCI5Owjuz!0LOK>?{oJ@96yxUt^j=jO+xK2^Qp_6QPX_x7Ic` zn!}BZ#t0#!fPM0ZGNPV4gRT3!bA2#c;)VwO>P9Hmk$z&QOxWu0q%~ z6`VnQR!S#S*%>9hQ4b&>jOU}VVRjJY04chPG*|FJ;gK{^aZ4;}!tx3HX|QbXDT+|n zkwVwK2&Sf6VFF>DZ$Sp8wk)s#j47cq=GR1!1Ys613)s*r48&Zr>!|1MoaBuA#iOB%EJ%K9 zPdFnzXfY-Dj=aIOYye5l63pUhZ#mA>*g9~CuxVcdr;_^sa>^0`rX5-0 zdQFoYr;a(;DYDV{cD`cFG!ER30tw?Rjp}y|gtBQST+bfd8wNPHbL6_H%%B4xBXNBi zO&Z!cvuI@VW-L)QCW5MBW2*wwNG!7v&t}j|5q0_}r(c|0zXT9{>|6>5`j z!{>drJM|cg#H71?EK!Uz98)zdZ3Da8+skPE&M*a*sO>Uxcel3}!|IQQ{tn%WZO5uD zjePNTte%?f4`@7}1Z)9B3x^9tZev0cD@4UgGp+#hd^oG3Roe8c0g3s9NR%_XOG9a+ zQ$zf>NyMRbG(kv$@~k`q&(<^n))ZKsjc0Qv!{#428k6{{d@5#l+BY^`Ls+7`C09zA zJucRN04LCDYoW~8L?Xt5VRq~udTR|CPM6VDTuDCzoB9J+bVy@d9Pr>QBUm_yUOz?eW0}5|?cF=vTQEEP15G7edJOkI=^Ma0DK*nFr6 z3zIgH_^Ng+;rXO}XER1}v<+5ddk>1S*uW`xc?qt66W^P=i4QSNNn)?byL=Vm^*i|F z&oHbl{1v@!+^CrbAXg@1SD@5Y@TjN1V#@#{MR1DYa?hC6ccTSSfMhl{LDV)$C^_qO zEZ{Damp8-0zLvo1hAcX;i#vOKl@jzjZ&UMm#~^Ro#>PyN+t?^nhnIxKR>OtNoFOW_ zP=ixry!Uj+oqq$tOv@ODHeI$AYwY`?JPop`@nJh`z0NUUO^>a85WuV^1e?oD9Il5U zVk=l*Gv)5=@6RJ$dnv9tJ_=8wWZ52JMrco&N(WF4#|m9nzI28niyhexzHE^#oxd3Z zGp7@kR_t|6z^5JI;ehTnS2i@XM&3< zyrOZcw}6K0Gj1n$$VE&C>2x{MfIALp?y_PfSYk=7WY)7FUF2})KFm|SL|#%2^Yj&+ zP*py)f?&;CjSQ~5&pl{%GDz$!)3v(BY-e8z9cmxRnUB1L#$LA?G^8ty< zw@sFE@E9mImy*Rj%f$%nP%}nz-*c*=yTIvn#gugDfJQR-ShE{>_Yjx**2#h8LEAR0 z7lyl*$*btbEQA2CsX6(wr(=vLa7cyY2*yYZ8~6YB|CWVYdjM$j1ZsuM2$i;jC-N!i zm$@{i5lAm?{nIDjHMZg42TUSs+aMM(iDW)CD)@scwXdy2ko6B9}(yPoFPpr5h?JY(i+~Y9G@DnrPLAH5mv&~DQ zHto$d(q^0Q^?FauV5(>SR6_Ph7*QUK$hB_-!9YE?*1Sz*?*$TMz`5E)pk7wV?x`gV zn}gpv1Z+aK&R{HB_<$s&&SOAxk?`guY*?0sS<9~)WH|jb$(AQPXR6mJIla`6rI*CD zxgnwC)#MYvTF^b@AiEjxLeNT14?bgUMICjwn3_NtYhe!owN{^m7nhxu;VdLNa!rIC zFQ9dD&qhi!3m5)W`87M@LW_>zWK}YnSq-izi(WJ2N+Vyq%tc)-KMtGUrIHBSJ2jMy zleTF%mHoV&%R>0SoRLM&_I0^Cob7jOXZwuYxW?cB-|{s}YsMRm1}-4bA(nM70M|&* zR!TT~tmQ5+qJYV7YAU;}O<6M~u6Ll@P}EsfJjMk{wH(8lb&Gqic*avcn)b?Rc7NlH zeDhz>S?HdsXZP>kx*W_uxV0F=A}DX*_&S_IvEKlk&q@*yEYELPpC`xGf?=*O4hceU zPVOtRHypze$C>rk(=9YD8++1MGT`Ib9CjF!|4)8nFzKMd3j~LEnQRqzoB$x;I|o>G zj5Ogl%xa-LVqA?OFML(84(4P)#`v0YID;^}ODnUa*g9r(Z>(0vimVxA4_#1W+A{3e z8bNBnAqufMHm~|-!xe^t*ztRJ7;kY8Mh6Tw3U=NoMz@O2X*;P#P z<{G4uo5CT7!wkuc+Qj5VCTqY@2zVhr%!j)L>-P=zfU9u{>6j&V8q0#zbRXkdg15}} zTJVd_yC^hMX!Ua$ch#>8pu6V&P=aKxv;G+*2*H%SrE2^NOZC!hGj^9L3GJ|@*$f?+ zvch><*F!JM`%?o8ESP&$L^M~M_$#dv-TZsU9SbH$i~XgzuhDpKOI3A~gA;x)`7ItQ zu(83o+SphWCg)$dBw+Vz&SP%2>-$7CCWO4L_}pI70)tKXKLnRx3zSR z2Du9D3q}BdZhPqM;rZF=+4l#C}GYdF{rqF~atm)!iEb|E(60CAT&|qpUyi>S7 zC3=ni&-48ORh$2(RQ~-+J`oEj_I7(cCxe`(Ns~J7RMxOai{08G!jZc$)sVTA3k)G+ zo|590M{-TgCJ20@6eHr0bptwS5t?7#ctNv6-{#6+Qe_M(Y>Mb4!$vvOf5o{FT3oAw zX3RV0oKAx;P24ItfKNZq&MMrrWGP`(f`#lYrmD#|rI?_gFw&*(;-xanDW$=g0-Oo3 zLf}fP^>Qu}W3UmVmiMFx%CZK3k9`cWZfcAfN+3T#It@LE7`gFf;fMky1U8Pu@3Quy zvV(Fi2QIFpP3BPX6|6u&S;_Kkyda8(CU_bM1(sOcO+y&nKwzmma=~&b5@|cScs|#$ zCg#l2ba@%~iO}*_{kYGUudZs;C&C7tEOIzyO_jmfR1r+#B{%CJU=XK}q?0^Vm=U=D ztJk@lX2x;$ZzT;wEr)@Iy(w z>q^*v(Mzx6M21cTtA?kEN6IhV87%Xcu8|->`~jnM$ilSjb4-Ur0wV&C0725J4dAKa zAR#G+9R~(wuu)8F0qUj`J^~z38Ccc5L}UzTqQpIX?1@{L@`{24*@?UdZV833=I_GE z&2$(($v%oU6HahD!h~PzRQvtoX}8UB3bgHxsLMnDtj*b@rf`G@KJVQQ`AWp%;WqLqTOU zZUj~SG{9|e$41+*r_oNrp|$-zqZI}WI(Ul3h=3kb1ptF(ETM_b5@j0>ggpnr=@1E0 zsa;}9^`pK;M`aEosw1nm=){x9rgk(YIEij=Z+mnydTcNW%G!h{MeqQmXwbX{zDr0z z#p_sF3KNnBe|2T#lcn&X*WK^m zJLSmuMw_+rJz)ecfOZLwW;>Vny6H@1+m7wTy`!kJD@nX?SQ;Tz8mj^XO=)! zl7iE;b-IHtg3m(w@VGKpi`gi#1>~o~Ps?6P<^s%C5TRyxu359Cb6$~;EnAXqL((kB zors1U%c8uv!nx||U^tkOk9fttQBI{X!M6~jSj2fb7&0P)!%-|utmlnMaOJL1wtzMR zsfx3FlOXZ5>TlV!JvZX05k;<7cNzGa#k_gCa(JnZU5v8~D&?zg+{&G2P%$3oK4ikj zsw`?yvFR&e73<#fB47oCNdN{Ukg{1L2nV;u2#7T`DO%`KyIH$c;tq&Dw??$ZFzrg; z9o5#7bw^YS5+(eGrQF`E-hyqtH33?|fb3Ukxw!Y^1S=C4C}&p;TnzLOyyd_d(_s|e z%;>X^O7D_cZk-RUTPLXeZl~50rm5hUt{ciP-Jb8_I}ljmV)9?Q7MfqWF6aif+7z}2 z=XB)`%bnYWOEG58D=c8x+*1Z$fhW z-j{OQg}dg+6`Vn5x`^e2EP-sefwohxVBNB z@Y!-@HzR$Xn9~Syiziu1Hqp+uoxq*#zm|Knz2I^eMbN+Eja_*E8Y-dJD7`+ew~ckVtVF=c&nBZKl*;9B=nr zvqD>G1$_fsHj#=>?E1N_dE~lbRSc6GXuG$AwxEGVLXjnBrwz2Ty=|~*-1*@Oc|JsV zz^{oPj^+hUnAX$*a=_!V)Sv+`F5NGAj=5>m4Cx;^6`BS(8s>UaC%0M3t*Nm(#?!4C zbKr<>tC&Q{a8QQfn&OHr6sTs1sD>SKQfQjBXa)#!sTUgKH$`75q$sAU@D=c~hMwTf zBF1uO_pcyqF<)2Z%v7!Isg~t=w_UTLR^5bV@b;g8vfM(=nG@&C@F8)tJ#DNlMOm=# zBhDmENRi9Yoil4uv#KT=-fcyIVJ_fw#p(gx*IPAkG>HUedbbmNV&Y!+w~LrinNJK} z-4GtPu8hYnuE$32j6eaj$2|Mn#n@gmOA*p!#cv)CBvvDabq$F1k|#kr%c?zrddKzz zBv_{?6hVY$g%zj!#zinlHRiV8nMgL2=fE;v_kl?&OpnLYgh(}GaL&W5bo}8%ZEST@ zw34b54lIq|nl4saRK?UmQhr@K@S#zS;1~(RZ5S!e8N?B&F=4&3yiyK`^i^!+r9B!8 zy{m?lqzU?^Tiut>%_{h%TjF+x6%>6QT7ym5sv3_q!6us^mNl#(KLK@sc@cl(a-%gx z>{wF}5D~LkId8bobAS9AoxQ$9M=uZ0zCS^iFHbMf#mVL6>Dl+_?dj!9ba;7rc=Yn* z82$A6)tetrF4mc|WxpETw7aleWY47L3P$ei>I;;NoSIV-jaHY2s?9Dg4h4sE>y(GQ zLHSD+h#F{QA&IDxVEjQOdJu^kM52cE_-B%d8WzWcQ1l=aH3~(wO?honQPU#&R}zcb zgrXwq?8rDF9_EV67^g&-M0QE1bd8OT{;-8nQQGFcOVE>(+l&Z0(ZWvpw03b36!WMr zkqKm}3$EezIsV<{Lq6MfDdA>kbyVzz#15_698a^)CUAXmpHZIF<~&+geV}9 zm^0%|>(4mPxWDwkrF(WKB9JZDlsYw*MDGqA`fFDmuJl&!-B>COxJ{^R9F6%T;|vC! z@nl97ImV(`-jyodMK4~eo_cwzg9G_i!s8+vV|M6*+$m=Tp}mH0Br#dv?V&@BlA6w1 z!xOC_uqf%f-Znnuk&wSWcxR>vlUEel1qC=U+VA2JXN_YE=Jn~B5Q-< zv~MH}p*1h#{6*9aA(m!(rt3TJC8#kWhVvv1mW)kc8Hl!UR9a3`@4x^@M{peR4o0S# z>OrW>9iaK6cUp1sMC9K%cSXuZ5wNYOB;ww2Db((EY+VB`{`QEoA=UFuIr}b!D@T3K zOBm*m#r@Nk_~ueW%lU@Iiha5a3pE3=!JqpqLf6=^0f75SbE_XVPt4 znks#w8R!uF2bIW)aUnEOLB&FK&DsoS|K{TKpzg*7@QRI%%~>@dMUjaz#Wd}q)wz>% z(`uU=W5Ker;3b_9o(K1u+82JISevURrXn`7-M7ElnLt$<1{dC<@gjSSq1?h3pvc_h zs(!8KVsE?LYf0nOl=8Lzen-3-xv#+K*X{UpzAkzvJ=J=hFpf)(?K@W(*lEK8i}^K4 zb8iK3FD|?ntM{-OV>yN@&I_8lE=+nztJD|Sse?RrR&WWEz7_tbz6XC2y=Y&iO`o{V zC2W>HxAL-Wqf5~BOh6zB!mFa3;TyBnVXCibwK21 zO`#1_LIa#c+LL+XrpXfNI)=oAS^E0TUI>b^%@Vdb3 zJZ$PZvr$?dEC}v)kPXR$; zPHI8?m1}hkBKjhmVB<{L{DgCPx19rB{J)`ijm0C8|~9=;;G$qO5?=;o##H6f=)U?grhF(Cto_B|Drsy74D!d z2cf`9&>}Lo=2ttx;wm&07Ou2bqlOEF6c-ZVAW)no5Oy!UXQita7(IMe<#kj0j1k(sH=Fj>Fx&Q4bubTu}{|5_^@hbhX5g3(qIJo!T89&dTSystu@A zpfoId@SFIX`OBm*YixqVDW(%2F;y`)ntsk9FK-c6)tDN`w&c>|Jh7Fyut`pzHqOwU z2V|p{g4m@8r3YKRAok9KZJhY>jc3)n(dKDjm4)ChOfYGKxab)s-F!w? zy;fS0_3tr3Wedlv=^??sw zQP~i_^!U6PPbrlxtEY$h=6rmh1xG=KIot*EkDMzL)BxuSJqW|xL_dG*GCU#OgRTkC z?rx$kmEUV*cMtUF{`mEADSmVTY$89LJ4{;J1@d#2;u%lKf|Li*V)sgL(Bpm6SrcMp z0mF>htb*Eb6?HBR4*gVv8ht~&kPY`q^?y177pKw=P8l3sO1(89kghXMjfYs_k)ZGh zQm9ik2CWOs$}@5yua}m>h^ARYq}h5Ucxb&8)Vu?f0;ukbgi82ejklmZzA(wH?9DF) zR?K}%*4XOI!L{6*l+9vbr@`$ua2hbJL(c#cH~woxzSMYNo0uim+s=Td{V(*3GMeg^Lls}FuKcW*0z8fanT&12+O&BvK!~t z=gW7ix+!OepPwMBK)IbBdacGp+)zoHnxL==T@&F()l9ATOO%I3eXTe%mC`WOkpD|J z&rG1_roGg$LeL3Z_NaH+H>agEYo$$hOa$qnH<=56wYs4>NVB~X<0NG514Vuy9A6QZ z1nD$v5-A7yW_YdL1T->bZtw~7gqqtRAp4fPR%YR)1geiAVB{%yi6t~MaYvCm*~ZoS z7Y6(O&pJz}Ci1A}L0Di>_S@21&RUw4M3(>53pC95XCfdZ!svr6&Uhfx>jY;`oq7E6 zmu@c7U%CgsbjM0%@?dML?2Th1Y){FhjWc{BdoiDEDXj^Ys{GRZf0RhP9t8Z#}zadhlMfs1{8;cTblJK?>4a3v~k$F2{45O zH1FQQtD0Ta53?DMFN?}eAZJ*OD+mW@%=-m(!jy`)~pDIcD*gh^R04?X*sG*2wMG%$rl!8bH>zSYSz5|MkvGjR@E?g~tiZhERg5VQu10m2R6sH$srBc$np)BDyE;8j#Je=7Kq+8svwmxt%!d3YY4hiA3t{|^8F|NmJ`hminC0svKX B3+?~_ literal 0 HcmV?d00001 diff --git a/packages/longhorn-crd/longhorn-1.4/charts/Chart.yaml b/charts/longhorn-crd/103.1.1+up1.4.4/Chart.yaml old mode 100755 new mode 100644 similarity index 92% rename from packages/longhorn-crd/longhorn-1.4/charts/Chart.yaml rename to charts/longhorn-crd/103.1.1+up1.4.4/Chart.yaml index aef6266777..b205951bcb --- a/packages/longhorn-crd/longhorn-1.4/charts/Chart.yaml +++ b/charts/longhorn-crd/103.1.1+up1.4.4/Chart.yaml @@ -8,4 +8,4 @@ appVersion: v1.4.4 description: Installs the CRDs for longhorn. name: longhorn-crd type: application -version: 1.4.4 +version: 103.1.1+up1.4.4 diff --git a/packages/longhorn-crd/longhorn-1.4/charts/README.md b/charts/longhorn-crd/103.1.1+up1.4.4/README.md old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn-crd/longhorn-1.4/charts/README.md rename to charts/longhorn-crd/103.1.1+up1.4.4/README.md diff --git a/packages/longhorn-crd/longhorn-1.4/charts/templates/_helpers.tpl b/charts/longhorn-crd/103.1.1+up1.4.4/templates/_helpers.tpl old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn-crd/longhorn-1.4/charts/templates/_helpers.tpl rename to charts/longhorn-crd/103.1.1+up1.4.4/templates/_helpers.tpl diff --git a/packages/longhorn-crd/longhorn-1.4/charts/templates/crds.yaml b/charts/longhorn-crd/103.1.1+up1.4.4/templates/crds.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn-crd/longhorn-1.4/charts/templates/crds.yaml rename to charts/longhorn-crd/103.1.1+up1.4.4/templates/crds.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/.helmignore b/charts/longhorn/103.1.1+up1.4.4/.helmignore old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/.helmignore rename to charts/longhorn/103.1.1+up1.4.4/.helmignore diff --git a/packages/longhorn/longhorn-1.4/charts/Chart.yaml b/charts/longhorn/103.1.1+up1.4.4/Chart.yaml old mode 100755 new mode 100644 similarity index 98% rename from packages/longhorn/longhorn-1.4/charts/Chart.yaml rename to charts/longhorn/103.1.1+up1.4.4/Chart.yaml index 00f83b693c..527541a075 --- a/packages/longhorn/longhorn-1.4/charts/Chart.yaml +++ b/charts/longhorn/103.1.1+up1.4.4/Chart.yaml @@ -37,4 +37,4 @@ sources: - https://github.com/longhorn/longhorn-ui - https://github.com/longhorn/longhorn-tests - https://github.com/longhorn/backing-image-manager -version: 1.4.4 +version: 103.1.1+up1.4.4 diff --git a/packages/longhorn/longhorn-1.4/charts/README.md b/charts/longhorn/103.1.1+up1.4.4/README.md old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/README.md rename to charts/longhorn/103.1.1+up1.4.4/README.md diff --git a/packages/longhorn/longhorn-1.4/charts/app-readme.md b/charts/longhorn/103.1.1+up1.4.4/app-readme.md old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/app-readme.md rename to charts/longhorn/103.1.1+up1.4.4/app-readme.md diff --git a/packages/longhorn/longhorn-1.4/charts/questions.yaml b/charts/longhorn/103.1.1+up1.4.4/questions.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/questions.yaml rename to charts/longhorn/103.1.1+up1.4.4/questions.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/NOTES.txt b/charts/longhorn/103.1.1+up1.4.4/templates/NOTES.txt old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/NOTES.txt rename to charts/longhorn/103.1.1+up1.4.4/templates/NOTES.txt diff --git a/packages/longhorn/longhorn-1.4/charts/templates/_helpers.tpl b/charts/longhorn/103.1.1+up1.4.4/templates/_helpers.tpl old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/_helpers.tpl rename to charts/longhorn/103.1.1+up1.4.4/templates/_helpers.tpl diff --git a/packages/longhorn/longhorn-1.4/charts/templates/clusterrole.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/clusterrole.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/clusterrole.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/clusterrole.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/clusterrolebinding.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/clusterrolebinding.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/clusterrolebinding.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/clusterrolebinding.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/daemonset-sa.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/daemonset-sa.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/daemonset-sa.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/daemonset-sa.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/default-setting.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/default-setting.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/default-setting.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/default-setting.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/deployment-driver.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/deployment-driver.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/deployment-driver.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/deployment-driver.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/deployment-recovery-backend.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/deployment-recovery-backend.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/deployment-recovery-backend.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/deployment-recovery-backend.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/deployment-ui.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/deployment-ui.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/deployment-ui.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/deployment-ui.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/deployment-webhook.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/deployment-webhook.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/deployment-webhook.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/deployment-webhook.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/ingress.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/ingress.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/ingress.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/ingress.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/postupgrade-job.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/postupgrade-job.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/postupgrade-job.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/postupgrade-job.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/psp.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/psp.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/psp.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/psp.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/registry-secret.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/registry-secret.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/registry-secret.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/registry-secret.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/serviceaccount.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/serviceaccount.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/serviceaccount.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/serviceaccount.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/services.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/services.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/services.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/services.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/storageclass.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/storageclass.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/storageclass.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/storageclass.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/tls-secrets.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/tls-secrets.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/tls-secrets.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/tls-secrets.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/uninstall-job.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/uninstall-job.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/uninstall-job.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/uninstall-job.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/userroles.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/userroles.yaml similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/userroles.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/userroles.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/validate-install-crd.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/validate-install-crd.yaml similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/validate-install-crd.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/validate-install-crd.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/templates/validate-psp-install.yaml b/charts/longhorn/103.1.1+up1.4.4/templates/validate-psp-install.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/templates/validate-psp-install.yaml rename to charts/longhorn/103.1.1+up1.4.4/templates/validate-psp-install.yaml diff --git a/packages/longhorn/longhorn-1.4/charts/values.yaml b/charts/longhorn/103.1.1+up1.4.4/values.yaml old mode 100755 new mode 100644 similarity index 100% rename from packages/longhorn/longhorn-1.4/charts/values.yaml rename to charts/longhorn/103.1.1+up1.4.4/values.yaml diff --git a/index.yaml b/index.yaml index 90190ddb31..4301792c63 100755 --- a/index.yaml +++ b/index.yaml @@ -2565,6 +2565,50 @@ entries: urls: - assets/longhorn/longhorn-103.2.0+up1.5.1.tgz version: 103.2.0+up1.5.1 + - annotations: + catalog.cattle.io/auto-install: longhorn-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Longhorn + catalog.cattle.io/kube-version: '>= 1.21.0-0' + catalog.cattle.io/namespace: longhorn-system + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: longhorn.io/v1beta1 + catalog.cattle.io/rancher-version: '>= 2.8.0-0 < 2.9.0-0' + catalog.cattle.io/release-name: longhorn + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/upstream-version: 1.4.4 + apiVersion: v1 + appVersion: v1.4.4 + created: "2023-11-03T14:20:14.179754922+08:00" + description: Longhorn is a distributed block storage system for Kubernetes. + digest: d2d2d329989d9ae122e1dd47b1d1d91e90a3e042e882dc0e50ab3f642475c224 + home: https://github.com/longhorn/longhorn + icon: https://raw.githubusercontent.com/cncf/artwork/master/projects/longhorn/icon/color/longhorn-icon-color.png + keywords: + - longhorn + - storage + - distributed + - block + - device + - iscsi + - nfs + kubeVersion: '>=1.21.0-0' + maintainers: + - email: maintainers@longhorn.io + name: Longhorn maintainers + name: longhorn + sources: + - https://github.com/longhorn/longhorn + - https://github.com/longhorn/longhorn-engine + - https://github.com/longhorn/longhorn-instance-manager + - https://github.com/longhorn/longhorn-share-manager + - https://github.com/longhorn/longhorn-manager + - https://github.com/longhorn/longhorn-ui + - https://github.com/longhorn/longhorn-tests + - https://github.com/longhorn/backing-image-manager + urls: + - assets/longhorn/longhorn-103.1.1+up1.4.4.tgz + version: 103.1.1+up1.4.4 - annotations: catalog.cattle.io/auto-install: longhorn-crd=match catalog.cattle.io/certified: rancher @@ -4081,6 +4125,21 @@ entries: urls: - assets/longhorn-crd/longhorn-crd-103.2.0+up1.5.1.tgz version: 103.2.0+up1.5.1 + - annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: longhorn-system + catalog.cattle.io/release-name: longhorn-crd + apiVersion: v1 + appVersion: v1.4.4 + created: "2023-11-03T14:30:47.292314931+08:00" + description: Installs the CRDs for longhorn. + digest: 0341588d1541093e8ab9e425ee28c431fe0a13a2efa00b52d589e149528c2240 + name: longhorn-crd + type: application + urls: + - assets/longhorn-crd/longhorn-crd-103.1.1+up1.4.4.tgz + version: 103.1.1+up1.4.4 - annotations: catalog.cattle.io/certified: rancher catalog.cattle.io/hidden: "true" diff --git a/release.yaml b/release.yaml index 8b13789179..f7660d6880 100644 --- a/release.yaml +++ b/release.yaml @@ -1 +1,4 @@ - +longhorn: + - 103.1.1+up1.4.4 +longhorn-crd: + - 103.1.1+up1.4.4 From 441192b949f237c36335271efc75727aa6461beb Mon Sep 17 00:00:00 2001 From: Venkata Krishna Rohit Sakala Date: Thu, 21 Sep 2023 15:07:54 -0700 Subject: [PATCH 4/5] make forward-port prometheus-federator 0.3.0+up0.3.3 --- .../prometheus-federator-0.3.0+up0.3.3.tgz | Bin 0 -> 20037 bytes .../0.3.0+up0.3.3/Chart.yaml | 20 ++ .../0.3.0+up0.3.3/README.md | 120 +++++++++ .../0.3.0+up0.3.3/app-README.md | 10 + .../charts/helmProjectOperator/Chart.yaml | 15 ++ .../charts/helmProjectOperator/README.md | 77 ++++++ .../charts/helmProjectOperator/app-readme.md | 20 ++ .../charts/helmProjectOperator/questions.yaml | 43 ++++ .../helmProjectOperator/templates/NOTES.txt | 2 + .../templates/_helpers.tpl | 66 +++++ .../templates/cleanup.yaml | 82 +++++++ .../templates/clusterrole.yaml | 57 +++++ .../templates/configmap.yaml | 14 ++ .../templates/deployment.yaml | 126 ++++++++++ .../helmProjectOperator/templates/psp.yaml | 68 ++++++ .../helmProjectOperator/templates/rbac.yaml | 32 +++ .../system-namespaces-configmap.yaml | 62 +++++ .../templates/validate-psp-install.yaml | 7 + .../charts/helmProjectOperator/values.yaml | 228 ++++++++++++++++++ .../0.3.0+up0.3.3/questions.yaml | 37 +++ .../0.3.0+up0.3.3/templates/NOTES.txt | 3 + .../0.3.0+up0.3.3/templates/_helpers.tpl | 66 +++++ .../0.3.0+up0.3.3/values.yaml | 94 ++++++++ index.yaml | 24 ++ release.yaml | 2 + 25 files changed, 1275 insertions(+) create mode 100644 assets/prometheus-federator/prometheus-federator-0.3.0+up0.3.3.tgz create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/Chart.yaml create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/README.md create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/app-README.md create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/Chart.yaml create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/README.md create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/app-readme.md create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/questions.yaml create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/NOTES.txt create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/_helpers.tpl create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/cleanup.yaml create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/clusterrole.yaml create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/configmap.yaml create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/deployment.yaml create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/psp.yaml create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/rbac.yaml create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/system-namespaces-configmap.yaml create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/validate-psp-install.yaml create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/values.yaml create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/questions.yaml create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/templates/NOTES.txt create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/templates/_helpers.tpl create mode 100644 charts/prometheus-federator/0.3.0+up0.3.3/values.yaml diff --git a/assets/prometheus-federator/prometheus-federator-0.3.0+up0.3.3.tgz b/assets/prometheus-federator/prometheus-federator-0.3.0+up0.3.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..abef0673b9049feb5d7c54ef1a6ad80c4f4cff67 GIT binary patch literal 20037 zcmV)sK$yQDiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYMciT3)Fgl<4EAYzQCziKK-;#8{O543Wi97YQO|s&&bN0%f zRY4>qVN4Mm0F#DG~%8;JM#QDHcr6S*7AB z%b28E$fKuoDs{4=i~LU?;&Xg_eEj&~L-_aj__+P=$)mH=Kb<^$`0&fer;ksMAO6W) zdHCg@$nghx`<73oG?jlke(!g+7xxGG(4r8UYAy;jiU>)mrn#6UDb+e>2^U8xlbTOC z%SJ@fBAqkYdt1g;nbTEV&;=Wjm;O|dZ{4ZteeAlLuz1O&;-VOl2j6^6PLh+y$#Hy4 zz9J{d=_B*?LGPpHO{%0R8xg!1FGRt$ki3}1YNa$=^gbulh~&Je-X7lYA`>^N_pXx3 z1=lLx@op)WJYy=JE#-*JS-z-$OiC&K!BTCmrUtQ&mM1hX=k%m^C=28!P)?J_$s>Tt zY4YWIkVuv@s+iq`KGcqA$$xeuFHa*{mj2J<Jxn}> z8EIXr(b18lH_43ac{QmNlc^{)D|C{Ig#+hNFP&RZUK~-Sm{v!*m zApnp6y(-PWXVIs|?A<=Sa9Ps4V(No2fGza@(W8fFZTkQCCR4KC2{frzo1`VnTDvq;$^AH$qBP;uk_J zDH;(iD+bq)3)ls|Vl%F^p+JKHK8g$cflgR%$P&x5CP_EHUS@D_npa9Qd71gAaqO4w zE`Pm}x&1CDH1(6b=uECu<$1t-$-7_eI`;O+RmoC5U6E2`B;e5X4X@a=rhbNr%7PhS z8cr-yb(D%CWu;a}rO0`@Ix0mLE0$K0>s4&8?a{KNasdSTLk_E0nuZeVzU*7ol;&HCR*ir48z5x@oXj3z6hstJe} zM9swwsY*gsKl3I)?9KzJsForFYT!jSE9ksfGAVhsP9(6yJ{5(QBF~vbI1ttmC1tU zTC-}dJfvDkiyuUt@_&H=W}{n7x}l)OU0mi*|Cs~t^KIWu@?|BRj57OF?} zp-tB5PKjEkqyK&UzmFq_)Hq@&%5M^`Bb>`@M3$&n0MD!P{3Q_d5xJZ`7y6}SiWM4B z`#eKUgXV0x%ZL!_$2pfXBtUqUp0Oz}7`d7AbWWa&%mVnO$P^yn zbk4HMpra^~-3TU}xmm?Cc2tOr#fs%DH81xB(OkZQt3m>o$AMpqoS|rgGFPR{k=Ap# zpBRFxccj^#eWsznbOb}hw8&!ZU*#uNUkVBIH8axDHCqkf0`UkrB7=*6Ry23l0f|Or z0ROs+Y&vDB9+AOwab*EHh(P(X1lr3udnD&ZyLMEOkOf=7iF$N={3G{Q)3O?oljGy# zg}o%%KP#sC9z5xNaADr>7_cFl?5z%F0b;>R$(KB5Gj^d;np4y{r!-g0zm#S#ihT7- z2>mV38RADE$wh=@Y*NkY$KcN&3=tcB94+SFpI-gX3+G4AU%bA!O7vU(KJVC~{~e!x zdEC89 zS==vH^8Cdqc>@yv`YYVXrzH6qt%PWoak^5k$;CfbN?zzG8T^|%8r*9DR%D&;(H{95 z+UChsRhB~PD0=tq=pb@etxAO&1I;ooQGWGaJf$U_@SJPTRATf^t3D@x;o5$Mae8bX z-lKxk$`yk!3+%BNsT(UJhaNhg3-dYsAtbC6A8bD_>l}W;RQ*>V!{iSArS4^-)Xg06fm z?|z|p8{W9>uEs(04~0SnE+x3T@5I58c?w1*Tpb=7bAqf7h){(FxXG$MNF=8dmM1&z zJ3J!UwB9bs%G}Iu@b3dM_+{YEfJ3v}fU3s-rUs2O(zoU0FRVZt#DprI;`G#x)5Dtw zW-wZoE!0M&$l}RrWW+xY;VUMWJZ1e4fs|RKtoewQ^m^|1?zX|ZciUiNM8)n1C6ziB z)6tG1DoUpjY|Jjt)H?{zS+s&DTaxT8`DZ0GtaHvJ*L2pt%|R}>vxOO{SS&;VP7DRV z5-t9VF7n=KE3k)tBdnzKVBq@?8Cfcv;6`$lyt0{23H_k5H>qq|kzO!Xw#6}}bqxXs7@UY<> z0EYwe8aFjw%y4Qbb)XNzI^J5t(+%SSVObV&Abt64^toSFl{7890bEDLttyvzkO z)C+l4~x=FAc-S9%hTlX5i5=YKbXg#q0{ml8SGn006y~60wlN( zVAJr7wh2*dv7xWf>ou`TMaeRR$ST@sOK&roz0Un1zOJL4zS+ROSxfKKb2 zi3Y-5@I}$(6#Sf2GH3NgSEt)~QII!wjDA=TVQ6{ERlgby^bQ#|Zl{R8DS z>xT2pdFb7QC#VIZaE3;r(%6Yzd60ClC%KC?1(te+GV4FHXzXm4+iTtIjQqTm|O za9|+dih%wVcxJEg9(T(34Qdd@7d)pjUy)p-*DNEMxGDTSHQh*4rohG~^qhH5I@lFp zMmW`B6!l=>164y{Au^T|_O^tlVSH!fu@eu(8aMm>@wyd{hcG>GS{hn|)0~>66)RCN z!rrE=)PDERvjs<6Azccd83i3(jD&d~BQP=cmZcD)W8M#L=zUeCb3)NE{Z}zja8ISZ zfP*ZAi$V<(a#?t1_hBFj_3k$!VBQy=pfEgP-5xeVZp`oNB0RVvVByk&NVrE-+WetX zNlO+OPH8HIQsHzI*=I{u7|r%M(|}$OD8L!I=FupDtlz!a-FT@4!gNR`m0q`1UTD23 zm{f-~9X4WwxnKZ<=dMyry2t~!AUjekqNsx_$mNmT}hz*fYRP%`5NNOtOngy}RHCWyIDx6z6sUs*B2fTK5T z%1ovfrU=)H<5N*+zX8BwUmN@43=S_Yo)6oK!c=}(qiF{hUaWX%&28FONS2$ z*AG-pBN4DQku)!!3IlR8FU7P$8=$V{q&W*WY!Y*H&K~I=mW}HIuq~V{wi#+q1E`In zXgnTA5XghZml4E5SYyzjyGa9hAbQ`QxO+h>xbh9QA6k#%vdYkRaiZw&0)) z(fYa0WcJ%2z|992p?g9BMFfWY$H&oyBjJL^SFmylV8a!tLx5Vg@c zSWhs95uBsB5~N^ASL_|IsIeVk-1H8arO}cbGQea=yX_`XN8g*f6)?K))1cFU6k=`P zaGX#|YQ#hOnGDT%!0}O{?b%`&)S#sqrz>vPUCTPw-7t19};zLvEa>w?%sm$=HSd zAXYx_9Gb7Stk}3d@Mb5pa4wi$OMp3&ZfgorbcAxdSfiYPcHaIyiK2$s$PxV(B7-!) z28jL2tPv(Lh@DcI93+ZSna=%+#xCC0Vu3TX0AKXc*hcUOsL{`-hgK4YgV%zsWDBuG zre4P!3)x`Zu-|sJE`}2{WQTt}(6L9l=CL^Q2H$M2w??_{!uLJ2y7)QVJgBA?Z8wct zRCzvbI-0pA5PaP4w%wq|Y?@A)S2JGxE~Zil$T^iCZ!#c4UlxXLAx9-oU;adnY<#JW zS9S3LT>=wD&meEo948Uj#6T(4RcR&H8F*d|A8DO+Bi9p3a>FeMU5E;Uyfc>Z6vRwB zkM&Ht;$6^}W0hpAAlI*{D@#FVA&sjD+B$cXQPAPT1>vV4DniFxG z+P9-Z0BASmqB2U9nev>^j94NwUQkeK99>5)KC`}OXU&^2rM9|XG~rtfe_+h9ee2rD#~EMK{OGP12B4`km!Jjd7M{F>!_ zE<|QP#){NHk4bc6A$kFN!XCQtC6&?~P{N}dK+erBKs&c_e8GlK%mizqw4{QdRGSGaCDUM$t(^UKD}pNqfKHp_SfY_fQ}{EHr=&%b;%`XH!X+MAvtdZ^@Xm&pDsP$Uu*nw3YTS2P8ate zg_SrHrf5&O_u$bP<%L25`^NFKAhQn=kERrujTAWxe2+ajLeeFV9-*Bwrj?!-@ZWzO z$q7x5_JRRp7;s4T6=OCMcIvWCd4XBq5bZV*RqZ)3GcgNw2Fbq}G6TutcB%gfnn#~; z`Y<^D@Wf0Cqoqwl!QJzM2_rl+#nSyUkl>fzdMUI%^RqRZhMQkdJmy56y{4Tv5## z<6-8ii>9>^0S<5~q-3fTMdl=taZngyT<2b**rckm%sKNnS4GBT-ipkqO%s+@c*@x1 zib&g@fT4h~_INPV1J#<3S>z#M+^F_Onfn>=T+~Jx^!D)r6kn2y)G@?2<4u{2V|+Ck z+ZY9BL2<-1pradOTY5vSCb^hk9g?l156M0ndul6_3p;fLM+fkCzq#%90U1$Deh}i? zE5dlXexxNF;jEwACN;O!4zGK80BNirJxP0%Z0cG7n3;+BO}di;mjkif;BC~-J!ewRhG<> zO-i_pxUUCooDL)E2a=;X6@gG<-LbVg5Znr0DMxYwOn}7}`Ks-=wL*Q%t*0AvOAZc< z{C9u(u?a{>o!_!oH#R3caAt;!1O{V3$jB>^vp;jtED=z4fsO0Pq{=d1bypWygC{n) z05}5>6ItoGCmx0f#9kT$Ij3bd*dUe~(hfpJ#x&)*LRn)Nf)%~BWS^R?8Fsw3WS?bR z4^d)!#b_48hWcNPtT)_gLk|uTuO)8Y(^|fR#n-A;o27w;1JvbK>N?6fKtvtI=F`9$ z)<}p7%hZe^;=)xpu;dM$SfD4Z7hM8LXz0u+1sa#!QP?Jt2D)xj-mg)9YPhM{W(^ja zMxcn8fT7>2$M8y<81cHJAr-<51Lt2goZ%4il{545s>sboBEJXBFTx#ayaR->nVAw2 zp%C?QGuwu%3!W>TkGQ)5Ev2zayUhWz*HkkBv4MQ7eV$X3=Jq|a_-{gGvJK=I9Vyjox;9qZ{Rk$ z3M(JMvBW3pX`6jkMiZwbW5p`xN{3WS0i|#4Z0LVlGU?IFZMU~?W=v^`#s(uSf=%d& zg+s%tY5-tO5ILiT4Yp{$bwLZXeex*;MoMJiaKOQqD-qgge6f)8bm9(3YfiL-G^!CB z*ld^lZbnU`EWfY*e|LTqL_@AIb$UClvsFM0OPhB+=`)d-wKoM!WHQKNv;ZEbn!KjL z!J4*+uFCGkwTWYhQ!od2xK}~|f4q)d}PX+VT;K8xulip3}6eRPj zIg)E|xK-h8Y|tHo8QcViVD<-U)jieCrGVDscWCaTSRBe@D zK=dkvW`s_}k_~%F+FIb+Dp5e#pybKn7D>7Y^#NF}eVTB6H=n^o=0toadQ~X{gO5K|)He>7%~WP?t&;qbMflO*5-c z$7m-vytzJ8u@fGQG6aUyUNy3PC)!+xj;48q($f0YI(i{sc=%t%#0>$P*Vg(ZpUn(K zqGW7d?ZW=YhxYHPoJpFo@u6X-_EO<)vi=e4IJLTaF=L>b1m=toSXO|2`l@S&sLBi& zf??t?jz{+nf}$u0aStnJSqlsgAYQe`2sYLo-N2%R1KOxA2(est^3Gy$;_qau`IOqu=z<1sXSndR~V*x~=cZD@BiL0Ff13Dvg2R z7(@ldW2`9;W6LCapkNvrgw-1<%zvHu+zfOMV&|QRQoF1m$OBm&pX3a!pbf5Uc$dSr zz=N?K-x|gkh*#giXoL6N6V=SaC@hC8*%^3wVPOfLn8t%*$>fdX;17U<4<|J^1~uie z>QaDUb6S?HPzZn)%*plcSB+Q? zmQ@FDFNrIck5@Fp$&J{u#xUB*`?z!1e=FJrh?0FMhPsplYQrl9vKG?(@Cv5HF$qS? zt{&C)Tq$16a%VAc(cVK;Ar`#g3z~O^FB;w^n|!i^Pq8)_trx`uJCO}85Q@2KtC$Uy zibW|oS`)bR&a(ugE2cs+u$l;&VSQ5czqYg%$D z=6l4uI#Hnx19Xa@5zcxHrcvCqt8O!If%->rw+8Ml7XS#y?^Ucke}Yn z8Gr^{7*gcy6`PK0HQoH*j)>Ju6_$!c8!`=7STkOzXVNwZ_32%f^o0OP(t5;KY^oAa zYaQpf&j3dK4LHqxhA?Ed>jpOD&)yMwIeWaGW4{PH8E{6ysD^?XI>VPNUpdnkwf?71 zN%HR_BSwS+9~grP{34$A#78Ew@QXF=A8KOAw>pXoxUuS8qX{@$S#ZKKmjz zTF;1l1;})3FzMEy&^P05-uHNecb)9_#Q`)z3i{-Mp;u+Cx6}oQf(5ax^Gy4qhpZLd z5X6ndz(*Ig*w=3mYy^RlI3skmc#$g5l=!r*2iY8koL{qm`39+Yt(iL!rluP!5MQms z#bdIw%ie`IcE+!BMl!oE{(OU^2i6x`s0z&li%pV+HGbH+p?Yk1Ef7K=MIev%3l$CF zsg^LTpn9GyX^}b|twTOqO)GA}ly=*=dK=xwacus-?sFU6_Hw*t?eg(0f#PitY5OA3 z9DHET4h7n44Q*{Tc9eRbw92cC5{-@fwUN`>Z+fDQy4IIrkFhu?G{>i{*ZXqrvKCMh z@-28-346PsWzG&kXPR3BX%_Z12{K5=F8(>OmD0y#zW|mA+LpZ$C?;3FEefDb_tC=^ zRKiDqlN=lrLLVI12;r1!R#5QWt=r=rEZS;(3qDhCzz%WE_eI!XMy|~@;9_ek1N%!( zo!7Jo+m8$rBU-dG7n%hEA$-({9T-ErcBPx0iMQlMJB7IGg0c(IhH{|4u~VV|gWU1H zLTFTpfufoN6tbU|3)vcVjh-;x;u!EgYj6cau*s%Cm&5){KHw%$7{(|4NoJJUoTuDS zTI*$|swb<78udTV4~gNpG6P$WnA&{=EOFboena{MYR|*Rx8!e5wS8rr(s zXRw~|9j4Hh=~0aMZ;&JY+X@o5p+2K9dn{;|l5v%qQ%amL7bKE47}r$D?aa#wezmF) zT3p@zFq4E_F$);vW4&>->o#OgWyT7Y^|)R(>_!=s3TjjoI!_^$zy{pIL+0=1udl^5 za|Q4*_Ri;Wy5Vfy1#IL%Y-pA)X>DeSfCv7Bv$v_K@XCa(b4WJS0c(^W6q0NKo zKD()>4MZc^JL4M^bd3zKYpw%6}JF-u})jYxV{u)3S^|U=7GE_1{k) ze)+iF|L@`B&;1`h%4diAkG}Zhx)O}*1AqV}AhexFr`29#B63ZP9|Z-VYkppTg3j>&jKq0$XGrkwBB! zMG3KiL*2HU$%BZ119n6vLgb7V5y9NkQD^$QfTyWjYyl14yk!FRrfo0Z!D?8~X1!(O zrB4lL?^fGS(<%`OIVd@Vrr0blo2%gLuAocPAXCj32*m)3_^tU2Jv5ZIEiEPcXT>GU z>gIYDJV8d@Ht6i^t*&8&E|b8hvatObB-xg&bP(F2drACEARdM!}wn@7aWdM%V7 zuI59)hhY!R-^cjWej%t& zn^H}45mwnrjczsNhCo;qtoOEzt1_pn7<;_;cV_E*4%BYn(&@oBUxQ6QIgXFXR|KTD zaU(B*!48qu=!e;y~N$w}1UQvG{a zkKdvU(Xu(#r#{%9KRbQyO8+uMuQzo!)j;;&aSx8LQ2rnXi6 zo@*T03X(A3Zu*gZ{HRp+QCufHZoAIodz1}#aI39f;yl!lAm5IRXq7_)S1L_)Bh#TS zEMmuQyxza=+}GD$|3=tz_w)6CsK&2$C?93=S~zIK!h+S7syq*571zWcrf~%!Xe&~R zFkVU3H@srg5!n#ToQfi4rB+8JxayBek;Sf=NNlg|(Xym+0dmZT99FS36^rsikGfK+ z&i$b73$brY+ye>7BdrFRwrj;pDGXkNp=CrK96Y$&D0a_-QAXrJ2KFrXTRox!ZarjA z>)_rIWmRatVD*rZgJK>30|^(_AD~z&nRbK;%11%qUs>fjR2g5;863>LJtDVkg{JBn zHUvvg#bzs9%{O^uMvP&&xRiWJHH%#*?W4%<3jF-C%JVCn)`-0MyZMd81VycZ9l(r* zQy)Z8vrguSWYuD^>M4y0^*ftUqd_F(hYSxjFZ!tw3K>R0TJyz?EN^X`RC?L zvZy&{z;^S7b!JWLA%H?t8QBs2hL zkiRDvyfA>VVJADqY1)Y=Ej+}E-2_?kvaq=S6h;kPB}KH!>`*zfI;+KlWfgscxf z4YJAF;OMC#gdsHemdhz&i&C!+n`Ja@b)&{xH%4Uo(Kg=Y>(1MEx^RO|;qZDU&Trss zeLvfl>;pKR8v|NhnISwYLRPiuYI4-ZfnSQSi2?6WLh&_dyG8^#CPa+y}Dh?$Km7n-#>Tq z`y4-86X)CDK#gCHdUHQ<^R=c>NU#+L`u%@2ljj=XG3+8+@B;lH7!I=J>?YYw!njYG zos%tE+lOibOr~}m5tx>Dl@OSlbq++`m{~xB^`XZ>4U}det>qJIB4=pxW` z&-(PZI&ijKu|m{_#wH6hun4DPbuwZpvn~5y2l>K#*330Hp0peEjEZ6738S1i4NWzK zJo8y2|G`de5755LI@X0H@35dEd>~v;DKb}VInc=1OfuD2WYNBYYRc0LNAOTX)uBF~822umh3^SA*#XM|)XK1aftKX4#A;`bkZ_m>U zo8R)xg{+aEE)z5CrLM!&T^FT^V->cz0`%;0BUFk(S-~V)yHd+|tYTl+Yz))CyXhc! z+!wU$N?bTzZOL&CKr35e283oai@A;duztl+*C1hwI5U)3p} z-EG-$fZFCFcW7EwI&WCbM{VDLko7{TI8lSp7D{ppjT%NVZ_6>5<`QEZkFPUjnj+`~ zH`7*?3v+4h9NdT@PrhMxt8Ks3o4K&1o90GDP2G72pw@dF?12cERCfjQ`e}b<`jXyqgF$WpzlX-UY4xPc}mrYoJ0{&b?o<83dOaMs}b2) zwpsRvU9L zcFncuW48Yg6NI*hgZcEq;yUML$s~C0>}?JVUna-#q~dvYdVG3%e0K5(SB?QuqwZ_> z$a$U1Oy#UvutN9QpV#O9!*ruMCHh>>YW3wA_GH6456?wr8=^qhUW{9xv#iQNtW2UE zL_3h%W);uaQ6VxGV-yG%#Zk1k7nw^9-fD(?EppfW7k0K%wh(~1n`p`l)C9M^+M5s? zV$l%{5Yr-ywSN&zrv4M*v1jmztwv;U%~peGj}R|>jmY5QpB2sV>e4F6@Go9s(KD+@9}A8Qe}a>Sl(xrN7@*fu;P^62>ZM{cjBWi=uv$H&JDxZwIGwI4a@d1Mhq z-cEOAEso~!v$xt$04UZ=ms@|~3met4*B(h}iL8Y)|K3cB(tVG~`42z9_qy&;_`Jct z7e&5$C4~N#=gblzd!*HxnaQd+SAW4U9kc%W;!ldn;2LDEF@p#+$Kn(tYg~j|qKr+d znVS#%@dI13d_<0;2#V%bu#y0c`;b`8S&pV=hr1`q$^S;H{aPvpr2K9;wUffM0jsQ9 z)Ws~6h<3rPou+Un?6Q+;V;JsIfMa-`x>$zSTX=>S`!S7>Ehm4sv)_06o9DlY0qbyJ zI~EYq#sfTw=Da@_g?T6(F2IB12v~5FvNUN+KXvJ$FKr6bPh;9t=2N?Ow|?&G|Mz~| zqvtPPUtA^nt={^ME%9H+r(Yho{QnQnPR~C3|3Aj(-8=F{fV-M1j9~Koaxj8Pm`lpTs#CSR?G$ zFvC_AqQUPUvHl>R`{VyFbC#D(szjGtLV>p6|HI?cHvT_4J^7sf^HDx~&KMEBdl$RD z(*tC{zgYeK7w5GekoebM;YL0sn~J8lLT3D9rR0U4lEJ^JqrtrfU`5vX9_^98VSpyN zs>)JG9Yycn9UVmOYF)}R&k4;kZ?yo441_(1^8`sx9U2ADcD=*Pfox{f$r~ZBb3rpz zx7V!YRM0PlV&RCd}z zjW_l0dcPkK_MR;HVgwGLDbJZ+mF(+R@Fp9DAmasW+OvW7NyDKGeVb?=U~pVudjDuJ#(x#!yVa>S33j> zZB*peNX? zpyeTQk?0P~wX%SjCLt{{(=of*+wjJ1cQuZhe<+kHa4o^zeP<31jWxxiSr`j17MX9i z=UUQPUp3xk8y)Qk3V24cX}w{RVFmEPzYoaZmw`J44%2P}!W#dZ8Z=Hx-=>qlSZ5HK zP{mUhkFewX@NS$PEz1^uTNhG1Sv3ntS}zvqeaL02_tu>3x!(ty58l1o1|1_Vc1I|w z)UlY3c9m06M2%o?c6zAZMR?F67d+aMXK%?rE1_Yfb0)c_v-WKccEO!3^hm{GAqoh) zQ1C0!;=kx3@4bOEk3@LHKeasa`SWo<_m}^zeQ<|VfGzR=$6r2t+?D@7=YM^aPnZ)l zp*o!(El(nJk^NOnq6O15qna8eAKhekBA2l}B=%{Wt#xACPX!YWx}wR=XYOS zheuk9)iNj7)9(6Px@>gK7%eaa+|`A^aQ)QT-F?Oe@nPOH%iLzf-4uq}m)eBDT0Pw? z-$+ueS;H7f?ZY9z{yJKFo2IjHM}t@;=g}Di&*C$+zX!yw`dK4>&>|ZJmtt}>;l+`f zH?GBLF%a$0-|p9FX*+e^wCnvEx^6ycyy@cqetGfq^~JMazPtG0 z$6wB0UjFj)#jC5!7tg;A*4zxkC%^vM9OLratFOO!@ZZIQ=4GQpHTsQyvG44sGZt1m zBXPdNI#>qtCvjYec+P0Xq$08S8e0y7tu%?V2jl^Xrzbtb&);Xx;ozq|gYzkQ^M-u!Zr*zGe|=59 zc(?ZS|4RP;cPQjqbOv-Y~p@7a%UR136+mv^&xigP-mA9YzzUAJTG?6&=jlSAb%0$t+hF8|GzB=)+LD@qe z=~|XoKU*`qlhYnuvgf_)T+YpD+=2hD9dN_k>J4AVg|QrQtg4A$tZP90`{u-r*oyFQ ze+aI7^Do`ybM7O>v_^FH=J@Ty9Yvn@8jt@S^TPh0lC$0sL`KHL93#>XB; zLjYU;csZeIVw17@e)8of+)d!9sYP_SiB@_pB>xw79=ZMxblx}%V^|~aeJzcR`i<{v zC+rV#Xf1n9(F%uv>(^iRMbjd-`FI1cK|~F#l~s-<&uPi)=-s#_Bs4-C;rEVO(tq`6 z$_twFe_^4Kz8f&sDaY~$A84mB(ehJlbniaS7?j_L$%C9L z_Z5nofAS3u%W3|BHW*W9{}3CzlWg$$^FROd9`c_{G2Q9`-dg|j%eMUY=+Wo+?~n7@ zbGd`A4ttgnT42*{R%p5Msh&RL@{la(H6zfti?*7ev0_Pku#FjP9U=R^IzQBiwvyrdn+_@A@0N1gcpFF*JH_&A@CAkwl_wLt&O@9+DP#oq@#B=?I-aSjd$Ob$6=P<*Z0E4aHzV)D2%%6fW2d(;1IhCmGPA4?CT?@Q(#4Q z9N)1G1(mQO^?g#4I+MWLVtWXD#m>-cx#PI&y*JGPDw+4d6-(MzGGG5@`{1tc2}>V$ zU+A_8Fnawv+kE(V!7!8hhu{Sn91h4nElUhm?`^XO6nf!xyc;*hP`u)N6&(=?(@%Z|8ZfttL zKVdZ?z@J?zeA|j zRfM)n2mKu-*RQ~}7mi!a?KdnQ?)-)|a?{CN|r)l zQn79QC)q+QS!PYyknC!c(P>cW6K&y4HV22YuG30alDX;gDalp^UGS7tWd^k+q5Y~Y z1e)~h<9Fw;o?Se@eEyeTUc7vL`QrK2FIV54|8n~1@h{(Bn| zt7_3c@;Hy5&wu#o;_8wWjd+zPf#U6sZDOmBthbeYu z*osW-nc#+4#LoQii22jC=47Q|{bToe50~7hwzL0UKlk_llxmmU??3HO|LLUN|NZpg z+2hagpC9G3*@?^p^buOcqB6C~^jo9z)P%qUVJ9m6T^jWi^8R;aN z_pquBkGG~9P+jLq=_|fD7fRc10L_b+FP}Cqyu5r?U#LrL0$ldUOKh3yrD703AU>nA z738H5x}E}bxVd3@jBLZ-Hs??-*mK%2v%);D+py*z&;7GJ z9L0bC;IRJ@M10`fOyF%BK$Bb}pK5E14rumTiv1~K>)NS4eydxv`#(7buwRR7w{A0c zcH6nF)#5sj*K=*#XZMKNr9JZe#p{bvv`5atVq!bL)-A=7PbDhdpa0wF zo4Q<8ye=bX3;%y~{HVqM&rVJrpMK{5ALG+@q^xVSwu^izOfUI| zGLX29fxEo>*pXv{=G}Ef+o0k7+-1xYB1LdSNT(dN;5=d#l)27|*+jL!Q!eD8?X zfm#!4h;aDuv6bM{{;Lwj|MWQ4-w$N(5xu%)^1F@JJD;6i@#gPsmn`Z4_NecNl|a^W z1JIG&iQn;drvMGxOk@YW^MjNX_?>c=b;SV&gKj|pFJh^FYwqAZAu{NzA#jgs5FgM$ z@Vni%``HfO!}8k>sY#lw(IU3?rb{DW0EAx&PBY_t*cRsXEmxE>#)ZR&o9I;4S*! z@#Eum{_mHMPmVw9e;?zsXZik2qo}u^3vpeQWUz_I56B=evkb_<{BJlU$IwD#?=CSV zdx40(-}I)Qzr3_8MDKOe(NDTN6^ZV*k@uwrc;lr`Y!x+PG;!##yyktb*#e%M+THb9yqyAoqg< z_i~$ri^1h}f#UtKO{#T5)>l0HW;{GNNJw-F%TbI$6&sq)JJWpn>aYa}XeU@{v7ivS zUmMlUZ7wkb!GeTBU~L)?qnm=s2Z~JCoaR$kxh7l&lAbLqhygBU9b`fk^vbro0rMtL zUx5LDh81q%Vq@yKIlgaMd-Wu+efx>}#`#ACHPw2m0MxUDy!!L`(>_Q|7@Spi-7 z8(`@!J21AJv!q46P{V-y+_w&7b0%hd$qG1DVro@^CrHqptDqj=%pcpKW$pc2bIj|?PcGoU{{kQU(Oxhh zB)O>}*;mZvz(C7B!=HFD6>@>Ir4vzU;<7NzOw2-uiNX6l@;B?9pl+%E8AX@HIt2H3 z;Uf@(2L7qKkp;LrY%&Wsng*Y8JZqGUjefX1Nx)3uqTI(%%#Rj$h<8|WAA0L);YL}x zVtCM~aST~P0fi`?0cX5%jP^{y6=uU2G0_GH3?7>|wqa_$Yqk%5l8VLA&;er|C_P5~ zXB3@x(ahXxJ1-Y0G^C_t-(fpCmWCloaSkMNHZ2{2XUv%(H**2qoLf$)HFP+r3=Yff zJ?3>;@F>D2f_c6|;7*v*;0DBsVa2sQWxeENAu^Vm)lOCWVGbQs3`ESFP|}3yD!L+r zOL9XCO=$QED>e?u#4e;NGA482cxf)G%um9s1q-1mZsbJUNM*?l*j?0Y&NlEQ&u8#?yU z%7buFUwOO?Mq*Uu3$F-YhBBihKZ3so?%X&L8#b6$~wP53bwmssiA|ETxlWcjFG1~RqobQSHQCQ!bVA} z?!gW=K-nRdoOT9$>6azRK1*f^`9T!3xsb&$Ay@Tz?Ijtz%)eiN+#k3}rDhs~;rf-; z0&Qzq!Rf@6_2U$->dn~lgrh$FYFx998Q1e_0-R&h6GO+Dp9aU&lY01^ql#d`=RJLmz6lZTy3Q+L#w@T z`~5=^)qMKSr=&eIS)xpN~T4TrfZZftS zGJ8wS(TU;D&D+{L-q9o%lcNRY#ZljgL$ZG}XOab-gk76qC#dQG&h9t2t?n(uC?-D$ zaqY=gov3WsdDhXal3KHJfhzsV0xuf6(i(}_HOaPT3FIm=5qjQplniwr|FdEW(pv0A zn8c0itZXD0iv}TN_U%iks&%i7OO|Sb;e|W6YxZ@*3o2I*&1&O$6rI~vAst*-&W0oz zBN?B9oUW0?@F`DiW=@YJ+&J#J5Z8(rL1`)^h&o#os}b%lSH5q#)si|x2LVP10i;e& zi3Cufw_rT9?aqrpR|3Ck?)d$cb zoA(uHYuWSQz%T*c4I%Q(`z!oS5L!iW<1XPcaH0Jz2YHb|<$;3KS>~(m z>a4m9p>TM>El)RiEh|0ucx;d|_G&apCOfObpdo@@m@!Rxu25YZhG4Z~%{~O*4G;Zj zPRiM58P~%@Smi55vluqi|7ygw;Z7TRaF9gb*y;_N_q0~gVevED$Gpo$%$)e56jHmx zAHoB}4IT01Q!?IO?QCowvEqK_$PE@c2J_T<Q@ zVA;SG?jJjDWw46oc7M1xUswnXi2=FBwu2-W#f(`ZfudQx2(mw6DXr}E-Nnmiz^VLk zvT{XO0;Xkp-y9KAPuuLX!Z6p26|0;p9a1d?l)kmIHN`2|l1YzVZo9Q0bSZd7GN!ad z`51KRjB5JC!a=fBEEcRV%K-pb%Wux04odetEofoo@2MG>ORI`OPvl}|APf^g^Rk7U zrxSNT+5&P<5z?q!Y+$fm^1B%|qqF3FY_@l2Maa@ao!l{?wh^e!sGjs0u+2i7(j9UX zIy7xO^c(Dth z^3)qOdId=%&UCq7>l|TJdY`+&76$4vJ4b77=0Y(;aSXEpRhjQbadwE^8#_j5`J0b& z2dUm1!!=0Sim)3yC$OL3klf4}lB4cPZZ2iLxw6L69RNHP;A8+iqzgunH-MTciX|&h zo~LT7h_dfff9OOk*|3*pZ3`p?$_3fjhg-cGRu^;=w+7BF5CKgG4&*DZ?(cgDABF<{ zFp65vzV?P3)-<4F6X|jPSQE54w+k@rg3&jf^E@Ni89J~*010%r_1F^{c57Y9!sz_x zO%K@q!rG6)cvI9W;+uAY2l%*pRr&r@kfEb$Ufs4Ny|s>B2pBH?S21xz!0xp*I>~1< zLz5^fuf)dMj1TSKw#3-@&@fDUsc<)$O=Fk_W9=%o2pDbJXhL9F5%uY-uHujN!jMhd zWSOR9U5|sp-a=%2>g{QPst*EGYm8uH-60ChG&rCwc_1WKMKj+6ZV8P=y{s@yNc0Uk zIJgATNDE!}I_W1Vh5*&4Tu5ARk2TP96H#z2z#Mm_X^JKmB^HcGMwKWIea25iZ0EIc z+HHMTA}M-=f5`LVQfUHJ;J^Ty&RKfx>|-1Z;*vd3Fbxgj=#3PR24w|gKpH%)Tog~p zT-+Ekd*~&RnRD!5J`*xKFyn)1vB8%O^K#f0IG5MsTf-oR1q-(u&hWl_qMCUa#o>@C zJF6)#EG)sX(0I^y-B!L(o)-OH7^@Hk2sVc>bQM~QJ0*l)&9{O};qy4RCj z%oI|-R$8J#7Fczd^oP)}t3$aaFoiQCfN9MZrPYay@@e4_MSm80Zv9b;1&@I<*&@F) zlZCSKcKlbh{>4;Eb38&~8rIWs=d!NygX)(xXc}2Qiq4gXx0l5gOL4d&d?Cm!>->i7 zZ3!NCI6zH#yCY|#fQ{Z<)CcW*T|gdCCHoLolWn-Gk?4n4K-f!}1iNHcUuuUJSjQL| z1zcIaA!-o|UhoCYJHr=^kO1~JWu@LF@}ss1trx}vQxJeggK%!0T9VNW{o#_M!GKF2 z;Q)HOgjho`j0l;9iKs2b$6QA}&dSNDwUf7k0lHhmy{#aC!X4#1u!JDD0-@TF4yOwm z>8!^l8pTc1={5@{h@@!OLXosC&MFEtmLh0ZB?BwGd;iv@AVa;GGh0H6k+BW-AJ^(Q zL_F98<5KVfHJ^nw6Vq^oHRm<(?AUzzG_Re@dOOx}p8<>p9I%@E3}NVOH&(~cKUWJq z@E!K}632cKb~51qfl(oRz!0Y;%U902MXk%{6U9{}k!P}ol-o!kw3Gk}x#fSVxG(SU z0x&~fiF154=!1cHlTtg_dt1oZZE1~y<}AN<2>?kHT~1rrVUPb-44d*?JJfRdC}S7Q z;jqdv%b$QYnaN>x6nw25(bR40RVoUMupu-gC@>s2D`6SCg$g!wGN)FX)mqXWk~f=D z6vc>5T}ObeT5gCiY!@c&G+H$M`s;Z2=i|EhtWkSLIn*1z`QB|D)`9e1mG^JL(;es)eA?C;$PH*TIlpEBr>YB&cS6=Izps&O1;Ug6< z;5#SQoo7p0q->2{(MOBG(@}H_mZ#gs)!XPcj$`xxb)VblcHFF;J-#L9Hgv40UuO8O zivzJW*L^T0h-pQCTy?YJx*5_tXFwc$f`rp^P1_`a?AP``YuV@tZt2=3hCOyvJMWAY z=BPGb@76+P&AAiuEqE9Sd%K`z&JIDFm|NSS!vbpowJ=d+Y{Pk2f;lN_O!f=lC7_4c z8-bK?rQLf0MY@k3wxAL|0wng}pb+}t06d1GNU3H81;3Tq&m9cAYJ3a6L2t(mamdAB zZL+4;MhkEyH6`MGqZM24pi#ShCDUPI_-oUb1d3L$M-bocc&}o*Dic`MW(V0Vc>%Oy zqi@AdS{v*~ADU_#W8fispP_%C1cDXF9E_0Ow%nrDjA;bp@fKUS_rrpF69Om_)R|qp zhEX!u+cM`VH+0YXK>KHGYposPT(Lr=p!6Aw!|6VlD|30xWjsp6(C^? zjk4`y5~Q&9zpgE3B5V6^;BEhHncHnBV=L?iY-J|%N5)lZ4h``MuXX5igFhI-U@MPs z!br2-ar=JkCQq)I1p=~*-UxgH==EJZ=9Ev{K%%e-Q^ zm1E$3I9n0eX2aR&A7{@CRqci*|D;R&RHLyo+zY;47o!&1y1hIxk&Ip&dExSkoFpfY zeFI~n498ysw?;z=OS+m(Ci-I&*MQLMb~C=D2^3KvKv46nR{^u@?P}B9QM}kNjIJp( zHyi+8PLoscU}ac#098k}Bze{t$INfE`(V9jgJ@E|Lj%_LFB&{;@xI&Htoj8JokhV7uplZpr^XJ3DUI|35oD zd-ys3^J9G0=6}Af^P8;$0<~YOKaw}S&)q!)dPsy32v?$#jv#?@Lw*8Z2&hXUBst5I z zV}_k%PSWvjysK#!`g)XIu+N74_LTyP8GtVlx)dGrSg@(QOk;r zlj#5ZfBs+a#DeV%!d)&KDfCtbJhZ*Aa2Ra*RY!kQr9{es8)G78u_7<8UIwCOcWZb- zrgUk*Ijx-mxIi3!v}wIqO223FQ*bvqAclvx_q4ha0p4W|RRqv8*!297eYQk3)V#n) z$oU{Tzt*8SPF$m^kTX}!pzYeEWFn2kN0c@~UcgY@d{vd&Xw7LQfl)Vom8#Z8i~!_Z zm$hHA>%4^YJBVuXu#(Q*HDj76U8UDPk_={cNJ^W;A_{{nFh!^18T2&x02;QafdS(- zR;;j=Y7+fa3h)jAh1hhWwFrZ392(@gJwZK(EKIrN+NW=OBH*YP2GrZYJ|}0oDfv>V z@J2)sODh0D!5ji%n^!MYo1C3z);9J&^o+Vej)sA*UMZt$zTC1GUZZib;HVDkKfNx6q6;D+>pf)SxYb@;z-$u41jPB;F3xu2gB@$48G0jEYS9Z^u=>92O8Lh zHv_cm?{KDkkd%`@(c{{v<2CgpL zhzu@p>OFHacj;BhMr0y{VMGzZk_RI)s3{Xtc{OMNwcjvMztz6WybZckbv&wZ9v%Arr#D4cFrFb5KGQ(YCu3<^p1dlh&l&|I$;8X%zjfq*g0RV|J={a{s43V0EK)r^Z)<= literal 0 HcmV?d00001 diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/Chart.yaml b/charts/prometheus-federator/0.3.0+up0.3.3/Chart.yaml new file mode 100644 index 0000000000..3b9594cc8f --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/Chart.yaml @@ -0,0 +1,20 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Prometheus Federator + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.25.0-0' + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/os: linux,windows + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: helm.cattle.io.projecthelmchart/v1alpha1 + catalog.cattle.io/rancher-version: '>= 2.6.5-0 < 2.7.0-0' + catalog.cattle.io/release-name: prometheus-federator +apiVersion: v2 +appVersion: 0.3.2 +dependencies: +- condition: helmProjectOperator.enabled + name: helmProjectOperator + repository: file://./charts/helmProjectOperator +description: Prometheus Federator +icon: https://raw.githubusercontent.com/rancher/prometheus-federator/main/assets/logos/prometheus-federator.svg +name: prometheus-federator +version: 0.3.0+up0.3.3 diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/README.md b/charts/prometheus-federator/0.3.0+up0.3.3/README.md new file mode 100644 index 0000000000..7da4edfc22 --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/README.md @@ -0,0 +1,120 @@ +# Prometheus Federator + +This chart is deploys a Helm Project Operator (based on the [rancher/helm-project-operator](https://github.com/rancher/helm-project-operator)), an operator that manages deploying Helm charts each containing a Project Monitoring Stack, where each stack contains: +- [Prometheus](https://prometheus.io/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator)) +- [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator)) +- [Grafana](https://github.com/helm/charts/tree/master/stable/grafana) (deployed via an embedded Helm chart) +- Default PrometheusRules and Grafana dashboards based on the collection of community-curated resources from [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) +- Default ServiceMonitors that watch the deployed resources + +> **Important Note: Prometheus Federator is designed to be deployed alongside an existing Prometheus Operator deployment in a cluster that has already installed the Prometheus Operator CRDs.** + +By default, the chart is configured and intended to be deployed alongside [rancher-monitoring](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting/), which deploys Prometheus Operator alongside a Cluster Prometheus that each Project Monitoring Stack is configured to federate namespace-scoped metrics from by default. + +## Pre-Installation: Using Prometheus Federator with Rancher and rancher-monitoring + +If you are running your cluster on [Rancher](https://rancher.com/) and already have [rancher-monitoring](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting/) deployed onto your cluster, Prometheus Federator's default configuration should already be configured to work with your existing Cluster Monitoring Stack; however, here are some notes on how we recommend you configure rancher-monitoring to optimize the security and usability of Prometheus Federator in your cluster: + +### Ensure the cattle-monitoring-system namespace is placed into the System Project (or a similarly locked down Project that has access to other Projects in the cluster) + +Prometheus Operator's security model expects that the namespace it is deployed into (`cattle-monitoring-system`) has limited access for anyone except Cluster Admins to avoid privilege escalation via execing into Pods (such as the Jobs executing Helm operations). In addition, deploying Prometheus Federator and all Project Prometheus stacks into the System Project ensures that the each Project Prometheus is able to reach out to scrape workloads across all Projects (even if Network Policies are defined via Project Network Isolation) but has limited access for Project Owners, Project Members, and other users to be able to access data they shouldn't have access to (i.e. being allowed to exec into pods, set up the ability to scrape namespaces outside of a given Project, etc.). + +### Configure rancher-monitoring to only watch for resources created by the Helm chart itself + +Since each Project Monitoring Stack will watch the other namespaces and collect additional custom workload metrics or dashboards already, it's recommended to configure the following settings on all selectors to ensure that the Cluster Prometheus Stack only monitors resources created by the Helm Chart itself: + +``` +matchLabels: + release: "rancher-monitoring" +``` + +The following selector fields are recommended to have this value: +- `.Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector` +- `.Values.prometheus.prometheusSpec.serviceMonitorSelector` +- `.Values.prometheus.prometheusSpec.podMonitorSelector` +- `.Values.prometheus.prometheusSpec.ruleSelector` +- `.Values.prometheus.prometheusSpec.probeSelector` + +Once this setting is turned on, you can always create ServiceMonitors or PodMonitors that are picked up by the Cluster Prometheus by adding the label `release: "rancher-monitoring"` to them (in which case they will be ignored by Project Monitoring Stacks automatically by default, even if the namespace in which those ServiceMonitors or PodMonitors reside in are not system namespaces). + +> Note: If you don't want to allow users to be able to create ServiceMonitors and PodMonitors that aggregate into the Cluster Prometheus in Project namespaces, you can additionally set the namespaceSelectors on the chart to only target system namespaces (which must contain `cattle-monitoring-system` and `cattle-dashboards`, where resources are deployed into by default by rancher-monitoring; you will also need to monitor the `default` namespace to get apiserver metrics or create a custom ServiceMonitor to scrape apiserver metrics from the Service residing in the default namespace) to limit your Cluster Prometheus from picking up other Prometheus Operator CRs; in that case, it would be recommended to turn `.Values.prometheus.prometheusSpec.ignoreNamespaceSelectors=true` to allow you to define ServiceMonitors that can monitor non-system namespaces from within a system namespace. + +In addition, if you modified the default `.Values.grafana.sidecar.*.searchNamespace` values on the Grafana Helm subchart for Monitoring V2, it is also recommended to remove the overrides or ensure that your defaults are scoped to only system namespaces for the following values: +- `.Values.grafana.sidecar.dashboards.searchNamespace` (default `cattle-dashboards`) +- `.Values.grafana.sidecar.datasources.searchNamespace` (default `null`, which means it uses the release namespace `cattle-monitoring-system`) +- `.Values.grafana.sidecar.plugins.searchNamespace` (default `null`, which means it uses the release namespace `cattle-monitoring-system`) +- `.Values.grafana.sidecar.notifiers.searchNamespace` (default `null`, which means it uses the release namespace `cattle-monitoring-system`) + +### Increase the CPU / memory limits of the Cluster Prometheus + +Depending on a cluster's setup, it's generally recommended to give a large amount of dedicated memory to the Cluster Prometheus to avoid restarts due to out-of-memory errors (OOMKilled), usually caused by churn created in the cluster that causes a large number of high cardinality metrics to be generated and ingested by Prometheus within one block of time; this is one of the reasons why the default Rancher Monitoring stack expects around 4GB of RAM to be able to operate in a normal-sized cluster. However, when introducing Project Monitoring Stacks that are all sending `/federate` requests to the same Cluster Prometheus and are reliant on the Cluster Prometheus being "up" to federate that system data on their namespaces, it's even more important that the Cluster Prometheus has an ample amount of CPU / memory assigned to it to prevent an outage that can cause data gaps across all Project Prometheis in the cluster. + +> Note: There are no specific recommendations on how much memory the Cluster Prometheus should be configured with since it depends entirely on the user's setup (namely the likelihood of encountering a high churn rate and the scale of metrics that could be generated at that time); it generally varies per setup. + +## How does the operator work? + +1. On deploying this chart, users can create ProjectHelmCharts CRs with `spec.helmApiVersion` set to `monitoring.cattle.io/v1alpha1` (also known as "Project Monitors" in the Rancher UI) in a **Project Registration Namespace (`cattle-project-`)**. +2. On seeing each ProjectHelmChartCR, the operator will automatically deploy a Project Prometheus stack on the Project Owner's behalf in the **Project Release Namespace (`cattle-project--monitoring`)** based on a HelmChart CR and a HelmRelease CR automatically created by the ProjectHelmChart controller in the **Operator / System Namespace**. +3. RBAC will automatically be assigned in the Project Release Namespace to allow users to view the Prometheus, Alertmanager, and Grafana UIs of the Project Monitoring Stack deployed; this will be based on RBAC defined on the Project Registration Namespace against the [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) (see below for more information about configuring RBAC). + +### What is a Project? + +In Prometheus Federator, a Project is a group of namespaces that can be identified by a `metav1.LabelSelector`; by default, the label used to identify projects is `field.cattle.io/projectId`, the label used to identify namespaces that are contained within a given [Rancher](https://rancher.com/) Project. + +### Configuring the Helm release created by a ProjectHelmChart + +The `spec.values` of this ProjectHelmChart resources will correspond to the `values.yaml` override to be supplied to the underlying Helm chart deployed by the operator on the user's behalf; to see the underlying chart's `values.yaml` spec, either: +- View to the chart's definition located at [`rancher/prometheus-federator` under `charts/rancher-project-monitoring`](https://github.com/rancher/prometheus-federator/blob/main/charts/rancher-project-monitoring) (where the chart version will be tied to the version of this operator) +- Look for the ConfigMap named `monitoring.cattle.io.v1alpha1` that is automatically created in each Project Registration Namespace, which will contain both the `values.yaml` and `questions.yaml` that was used to configure the chart (which was embedded directly into the `prometheus-federator` binary). + +### Namespaces + +As a Project Operator based on [rancher/helm-project-operator](https://github.com/rancher/helm-project-operator), Prometheus Federator has three different classifications of namespaces that the operator looks out for: +1. **Operator / System Namespace**: this is the namespace that the operator is deployed into (e.g. `cattle-monitoring-system`). This namespace will contain all HelmCharts and HelmReleases for all ProjectHelmCharts watched by this operator. **Only Cluster Admins should have access to this namespace.** +2. **Project Registration Namespace (`cattle-project-`)**: this is the set of namespaces that the operator watches for ProjectHelmCharts within. The RoleBindings and ClusterRoleBindings that apply to this namespace will also be the source of truth for the auto-assigned RBAC created in the Project Release Namespace (see more details below). **Project Owners (admin), Project Members (edit), and Read-Only Members (view) should have access to this namespace**. +> Note: Project Registration Namespaces will be auto-generated by the operator and imported into the Project it is tied to if `.Values.global.cattle.projectLabel` is provided (which is set to `field.cattle.io/projectId` by default); this indicates that a Project Registration Namespace should be created by the operator if at least one namespace is observed with that label. The operator will not let these namespaces be deleted unless either all namespaces with that label are gone (e.g. this is the last namespace in that project, in which case the namespace will be marked with the label `"helm.cattle.io/helm-project-operator-orphaned": "true"`, which signals that it can be deleted) or it is no longer watching that project (because the project ID was provided under `.Values.helmProjectOperator.otherSystemProjectLabelValues`, which serves as a denylist for Projects). These namespaces will also never be auto-deleted to avoid destroying user data; it is recommended that users clean up these namespaces manually if desired on creating or deleting a project +> Note: if `.Values.global.cattle.projectLabel` is not provided, the Operator / System Namespace will also be the Project Registration Namespace +3. **Project Release Namespace (`cattle-project--monitoring`)**: this is the set of namespaces that the operator deploys Project Monitoring Stacks within on behalf of a ProjectHelmChart; the operator will also automatically assign RBAC to Roles created in this namespace by the Project Monitoring Stack based on bindings found in the Project Registration Namespace. **Only Cluster Admins should have access to this namespace; Project Owners (admin), Project Members (edit), and Read-Only Members (view) will be assigned limited access to this namespace by the deployed Helm Chart and Prometheus Federator.** +> Note: Project Release Namespaces are automatically deployed and imported into the project whose ID is specified under `.Values.helmProjectOperator.projectReleaseNamespaces.labelValue` (which defaults to the value of `.Values.global.cattle.systemProjectId` if not specified) whenever a ProjectHelmChart is specified in a Project Registration Namespace +> Note: Project Release Namespaces follow the same orphaning conventions as Project Registration Namespaces (see note above) +> Note: if `.Values.projectReleaseNamespaces.enabled` is false, the Project Release Namespace will be the same as the Project Registration Namespace + +### Helm Resources (HelmChart, HelmRelease) + +On deploying a ProjectHelmChart, the Prometheus Federator will automatically create and manage two child custom resources that manage the underlying Helm resources in turn: +- A HelmChart CR (managed via an embedded [k3s-io/helm-contoller](https://github.com/k3s-io/helm-controller) in the operator): this custom resource automatically creates a Job in the same namespace that triggers a `helm install`, `helm upgrade`, or `helm uninstall` depending on the change applied to the HelmChart CR; this CR is automatically updated on changes to the ProjectHelmChart (e.g. modifying the values.yaml) or changes to the underlying Project definition (e.g. adding or removing namespaces from a project). +> **Important Note: If a ProjectHelmChart is not deploying or updating the underlying Project Monitoring Stack for some reason, the Job created by this resource in the Operator / System namespace should be the first place you check to see if there's something wrong with the Helm operation; however, this is generally only accessible by a Cluster Admin.** +- A HelmRelease CR (managed via an embedded [rancher/helm-locker](https://github.com/rancher/helm-locker) in the operator): this custom resource automatically locks a deployed Helm release in place and automatically overwrites updates to underlying resources unless the change happens via a Helm operation (`helm install`, `helm upgrade`, or `helm uninstall` performed by the HelmChart CR). +> Note: HelmRelease CRs emit Kubernetes Events that detect when an underlying Helm release is being modified and locks it back to place; to view these events, you can use `kubectl describe helmrelease -n `; you can also view the logs on this operator to see when changes are detected and which resources were attempted to be modified + +Both of these resources are created for all Helm charts in the Operator / System namespaces to avoid escalation of privileges to underprivileged users. + +### RBAC + +As described in the section on namespaces above, Prometheus Federator expects that Project Owners, Project Members, and other users in the cluster with Project-level permissions (e.g. permissions in a certain set of namespaces identified by a single label selector) have minimal permissions in any namespaces except the Project Registration Namespace (which is imported into the project by default) and those that already comprise their projects. Therefore, in order to allow Project Owners to assign specific chart permissions to other users in their Project namespaces, the Helm Project Operator will automatically watch the following bindings: +- ClusterRoleBindings +- RoleBindings in the Project Release Namespace + +On observing a change to one of those types of bindings, the Helm Project Operator will check whether the `roleRef` that the the binding points to matches a ClusterRole with the name provided under `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.admin`, `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.edit`, or `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.view`; by default, these roleRefs correspond will correspond to `admin`, `edit`, and `view` respectively, which are the [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles). + +> Note: for Rancher RBAC users, these [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) directly correlate to the `Project Owner`, `Project Member`, and `Read-Only` default Project Role Templates. + +If the `roleRef` matches, the Helm Project Operator will filter the `subjects` of the binding for all Users and Groups and use that to automatically construct a RoleBinding for each Role in the Project Release Namespace with the same name as the role and the following labels: +- `helm.cattle.io/project-helm-chart-role: {{ .Release.Name }}` +- `helm.cattle.io/project-helm-chart-role-aggregate-from: ` + +By default, the `rancher-project-monitoring` (the underlying chart deployed by Prometheus Federator) creates three default Roles per Project Release Namespace that provide `admin`, `edit`, and `view` users to permissions to view the Prometheus, Alertmanager, and Grafana UIs of the Project Monitoring Stack to provide least privilege; however, if a Cluster Admin would like to assign additional permissions to certain users, they can either directly assign RoleBindings in the Project Release Namespace to certain users or created Roles with the above two labels on them to allow Project Owners to control assigning those RBAC roles to users in their Project Registration namespaces. + +### Advanced Helm Project Operator Configuration + +|Value|Configuration| +|---|---------------------------| +|`helmProjectOperator.valuesOverride`| Allows an Operator to override values that are set on each ProjectHelmChart deployment on an operator-level; user-provided options (specified on the `spec.values` of the ProjectHelmChart) are automatically overridden if operator-level values are provided. For an exmaple, see how the default value overrides `federate.targets` (note: when overriding list values like `federate.targets`, user-provided list values will **not** be concatenated) | +|`helmProjectOperator.projectReleaseNamespaces.labelValues`| The value of the Project that all Project Release Namespaces should be auto-imported into (via label and annotation). Not recommended to be overridden on a Rancher setup. | +|`helmProjectOperator.otherSystemProjectLabelValues`| Other namespaces that the operator should treat as a system namespace that should not be monitored. By default, all namespaces that match `global.cattle.systemProjectId` will not be matched. `cattle-monitoring-system`, `cattle-dashboards`, and `kube-system` are explicitly marked as system namespaces as well, regardless of label or annotation. | +|`helmProjectOperator.releaseRoleBindings.aggregate`| Whether to automatically create RBAC resources in Project Release namespaces +|`helmProjectOperator.releaseRoleBindings.clusterRoleRefs.`| ClusterRoles to reference to discover subjects to create RoleBindings for in the Project Release Namespace for all corresponding Project Release Roles. See RBAC above for more information | +|`helmProjectOperator.hardenedNamespaces.enabled`| Whether to automatically patch the default ServiceAccount with `automountServiceAccountToken: false` and create a default NetworkPolicy in all managed namespaces in the cluster; the default values ensure that the creation of the namespace does not break a CIS 1.16 hardened scan | +|`helmProjectOperator.hardenedNamespaces.configuration`| The configuration to be supplied to the default ServiceAccount or auto-generated NetworkPolicy on managing a namespace | +|`helmProjectOperator.helmController.enabled`| Whether to enable an embedded k3s-io/helm-controller instance within the Helm Project Operator. Should be disabled for RKE2/K3s clusters before v1.23.14 / v1.24.8 / v1.25.4 since RKE2/K3s clusters already run Helm Controller at a cluster-wide level to manage internal Kubernetes components | +|`helmProjectOperator.helmLocker.enabled`| Whether to enable an embedded rancher/helm-locker instance within the Helm Project Operator. | diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/app-README.md b/charts/prometheus-federator/0.3.0+up0.3.3/app-README.md new file mode 100644 index 0000000000..cba87c6a85 --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/app-README.md @@ -0,0 +1,10 @@ +# Prometheus Federator + +This chart deploys an operator that manages Project Monitoring Stacks composed of the following set of resources that are scoped to project namespaces: +- [Prometheus](https://prometheus.io/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator)) +- [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator)) +- [Grafana](https://github.com/helm/charts/tree/master/stable/grafana) (deployed via an embedded Helm chart) +- Default PrometheusRules and Grafana dashboards based on the collection of community-curated resources from [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) +- Default ServiceMonitors that watch the deployed Prometheus, Grafana, and Alertmanager + +Since this Project Monitoring Stack deploys Prometheus Operator CRs, an existing Prometheus Operator instance must already be deployed in the cluster for Prometheus Federator to successfully be able to deploy Project Monitoring Stacks. It is recommended to use [`rancher-monitoring`](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting/) for this. For more information on how the chart works or advanced configurations, please read the `README.md`. diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/Chart.yaml b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/Chart.yaml new file mode 100644 index 0000000000..f72ae6b20b --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Helm Project Operator + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.25.0-0' + catalog.cattle.io/namespace: cattle-helm-system + catalog.cattle.io/os: linux,windows + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: helm.cattle.io.projecthelmchart/v1alpha1 + catalog.cattle.io/rancher-version: '>= 2.6.5-0 < 2.7.0-0' + catalog.cattle.io/release-name: helm-project-operator +apiVersion: v2 +appVersion: 0.2.1 +description: Helm Project Operator +name: helmProjectOperator +version: 0.2.1 diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/README.md b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/README.md new file mode 100644 index 0000000000..fc1d39e81b --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/README.md @@ -0,0 +1,77 @@ +# Helm Project Operator + +## How does the operator work? + +1. On deploying a Helm Project Operator, users can create ProjectHelmCharts CRs with `spec.helmApiVersion` set to `dummy.cattle.io/v1alpha1` in a **Project Registration Namespace (`cattle-project-`)**. +2. On seeing each ProjectHelmChartCR, the operator will automatically deploy the embedded Helm chart on the Project Owner's behalf in the **Project Release Namespace (`cattle-project--dummy`)** based on a HelmChart CR and a HelmRelease CR automatically created by the ProjectHelmChart controller in the **Operator / System Namespace**. +3. RBAC will automatically be assigned in the Project Release Namespace to allow users to based on Role created in the Project Release Namespace with a given set of labels; this will be based on RBAC defined on the Project Registration Namespace against the [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) (see below for more information about configuring RBAC). + +### What is a Project? + +In Helm Project Operator, a Project is a group of namespaces that can be identified by a `metav1.LabelSelector`; by default, the label used to identify projects is `field.cattle.io/projectId`, the label used to identify namespaces that are contained within a given [Rancher](https://rancher.com/) Project. + +### What is a ProjectHelmChart? + +A ProjectHelmChart is an instance of a (project-scoped) Helm chart deployed on behalf of a user who has permissions to create ProjectHelmChart resources in a Project Registration namespace. + +Generally, the best way to think about the ProjectHelmChart model is by comparing it to two other models: +1. Managed Kubernetes providers (EKS, GKE, AKS, etc.): in this model, a user has the ability to say "I want a Kubernetes cluster" but the underlying cloud provider is responsible for provisioning the infrastructure and offering **limited view and access** of the underlying resources created on their behalf; similarly, Helm Project Operator allows a Project Owner to say "I want this Helm chart deployed", but the underlying Operator is responsible for "provisioning" (deploying) the Helm chart and offering **limited view and access** of the underlying Kubernetes resources created on their behalf (based on configuring "least-privilege" Kubernetes RBAC for the Project Owners / Members in the newly created Project Release Namespace). +2. Dynamically-provisioned Persistent Volumes: in this model, a single resource (PersistentVolume) exists that allows you to specify a Storage Class that actually implements provisioning the underlying storage via a Storage Class Provisioner (e.g. Longhorn). Similarly, the ProjectHelmChart exists that allows you to specify a `spec.helmApiVersion` ("storage class") that actually implements deploying the underlying Helm chart via a Helm Project Operator (e.g. [`rancher/prometheus-federator`](https://github.com/rancher/prometheus-federator)). + +### Configuring the Helm release created by a ProjectHelmChart + +The `spec.values` of this ProjectHelmChart resources will correspond to the `values.yaml` override to be supplied to the underlying Helm chart deployed by the operator on the user's behalf; to see the underlying chart's `values.yaml` spec, either: +- View to the chart's definition located at [`rancher/helm-project-operator` under `charts/example-chart`](https://github.com/rancher/helm-project-operator/blob/main/charts/example-chart) (where the chart version will be tied to the version of this operator) +- Look for the ConfigMap named `dummy.cattle.io.v1alpha1` that is automatically created in each Project Registration Namespace, which will contain both the `values.yaml` and `questions.yaml` that was used to configure the chart (which was embedded directly into the `helm-project-operator` binary). + +### Namespaces + +All Helm Project Operators have three different classifications of namespaces that the operator looks out for: +1. **Operator / System Namespace**: this is the namespace that the operator is deployed into (e.g. `cattle-helm-system`). This namespace will contain all HelmCharts and HelmReleases for all ProjectHelmCharts watched by this operator. **Only Cluster Admins should have access to this namespace.** +2. **Project Registration Namespace (`cattle-project-`)**: this is the set of namespaces that the operator watches for ProjectHelmCharts within. The RoleBindings and ClusterRoleBindings that apply to this namespace will also be the source of truth for the auto-assigned RBAC created in the Project Release Namespace (see more details below). **Project Owners (admin), Project Members (edit), and Read-Only Members (view) should have access to this namespace**. +> Note: Project Registration Namespaces will be auto-generated by the operator and imported into the Project it is tied to if `.Values.global.cattle.projectLabel` is provided (which is set to `field.cattle.io/projectId` by default); this indicates that a Project Registration Namespace should be created by the operator if at least one namespace is observed with that label. The operator will not let these namespaces be deleted unless either all namespaces with that label are gone (e.g. this is the last namespace in that project, in which case the namespace will be marked with the label `"helm.cattle.io/helm-project-operator-orphaned": "true"`, which signals that it can be deleted) or it is no longer watching that project (because the project ID was provided under `.Values.helmProjectOperator.otherSystemProjectLabelValues`, which serves as a denylist for Projects). These namespaces will also never be auto-deleted to avoid destroying user data; it is recommended that users clean up these namespaces manually if desired on creating or deleting a project +> Note: if `.Values.global.cattle.projectLabel` is not provided, the Operator / System Namespace will also be the Project Registration Namespace +3. **Project Release Namespace (`cattle-project--dummy`)**: this is the set of namespaces that the operator deploys Helm charts within on behalf of a ProjectHelmChart; the operator will also automatically assign RBAC to Roles created in this namespace by the Helm charts based on bindings found in the Project Registration Namespace. **Only Cluster Admins should have access to this namespace; Project Owners (admin), Project Members (edit), and Read-Only Members (view) will be assigned limited access to this namespace by the deployed Helm Chart and Helm Project Operator.** +> Note: Project Release Namespaces are automatically deployed and imported into the project whose ID is specified under `.Values.helmProjectOperator.projectReleaseNamespaces.labelValue` (which defaults to the value of `.Values.global.cattle.systemProjectId` if not specified) whenever a ProjectHelmChart is specified in a Project Registration Namespace +> Note: Project Release Namespaces follow the same orphaning conventions as Project Registration Namespaces (see note above) +> Note: if `.Values.projectReleaseNamespaces.enabled` is false, the Project Release Namespace will be the same as the Project Registration Namespace + +### Helm Resources (HelmChart, HelmRelease) + +On deploying a ProjectHelmChart, the Helm Project Operator will automatically create and manage two child custom resources that manage the underlying Helm resources in turn: +- A HelmChart CR (managed via an embedded [k3s-io/helm-contoller](https://github.com/k3s-io/helm-controller) in the operator): this custom resource automatically creates a Job in the same namespace that triggers a `helm install`, `helm upgrade`, or `helm uninstall` depending on the change applied to the HelmChart CR; this CR is automatically updated on changes to the ProjectHelmChart (e.g. modifying the values.yaml) or changes to the underlying Project definition (e.g. adding or removing namespaces from a project). +> **Important Note: If a ProjectHelmChart is not deploying or updating the underlying Project Monitoring Stack for some reason, the Job created by this resource in the Operator / System namespace should be the first place you check to see if there's something wrong with the Helm operation; however, this is generally only accessible by a Cluster Admin.** +- A HelmRelease CR (managed via an embedded [rancher/helm-locker](https://github.com/rancher/helm-locker) in the operator): this custom resource automatically locks a deployed Helm release in place and automatically overwrites updates to underlying resources unless the change happens via a Helm operation (`helm install`, `helm upgrade`, or `helm uninstall` performed by the HelmChart CR). +> Note: HelmRelease CRs emit Kubernetes Events that detect when an underlying Helm release is being modified and locks it back to place; to view these events, you can use `kubectl describe helmrelease -n `; you can also view the logs on this operator to see when changes are detected and which resources were attempted to be modified + +Both of these resources are created for all Helm charts in the Operator / System namespaces to avoid escalation of privileges to underprivileged users. + +### RBAC + +As described in the section on namespaces above, Helm Project Operator expects that Project Owners, Project Members, and other users in the cluster with Project-level permissions (e.g. permissions in a certain set of namespaces identified by a single label selector) have minimal permissions in any namespaces except the Project Registration Namespace (which is imported into the project by default) and those that already comprise their projects. Therefore, in order to allow Project Owners to assign specific chart permissions to other users in their Project namespaces, the Helm Project Operator will automatically watch the following bindings: +- ClusterRoleBindings +- RoleBindings in the Project Release Namespace + +On observing a change to one of those types of bindings, the Helm Project Operator will check whether the `roleRef` that the the binding points to matches a ClusterRole with the name provided under `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.admin`, `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.edit`, or `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.view`; by default, these roleRefs correspond will correspond to `admin`, `edit`, and `view` respectively, which are the [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles). + +> Note: for Rancher RBAC users, these [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) directly correlate to the `Project Owner`, `Project Member`, and `Read-Only` default Project Role Templates. + +If the `roleRef` matches, the Helm Project Operator will filter the `subjects` of the binding for all Users and Groups and use that to automatically construct a RoleBinding for each Role in the Project Release Namespace with the same name as the role and the following labels: +- `helm.cattle.io/project-helm-chart-role: {{ .Release.Name }}` +- `helm.cattle.io/project-helm-chart-role-aggregate-from: ` + +By default, the `example-chart` (the underlying chart deployed by Helm Project Operator) does not create any default roles; however, if a Cluster Admin would like to assign additional permissions to certain users, they can either directly assign RoleBindings in the Project Release Namespace to certain users or created Roles with the above two labels on them to allow Project Owners to control assigning those RBAC roles to users in their Project Registration namespaces. + +### Advanced Helm Project Operator Configuration + +|Value|Configuration| +|---|---------------------------| +|`valuesOverride`| Allows an Operator to override values that are set on each ProjectHelmChart deployment on an operator-level; user-provided options (specified on the `spec.values` of the ProjectHelmChart) are automatically overridden if operator-level values are provided. For an exmaple, see how the default value overrides `federate.targets` (note: when overriding list values like `federate.targets`, user-provided list values will **not** be concatenated) | +|`projectReleaseNamespaces.labelValues`| The value of the Project that all Project Release Namespaces should be auto-imported into (via label and annotation). Not recommended to be overridden on a Rancher setup. | +|`otherSystemProjectLabelValues`| Other namespaces that the operator should treat as a system namespace that should not be monitored. By default, all namespaces that match `global.cattle.systemProjectId` will not be matched. `kube-system` is explicitly marked as a system namespace as well, regardless of label or annotation. | +|`releaseRoleBindings.aggregate`| Whether to automatically create RBAC resources in Project Release namespaces +|`releaseRoleBindings.clusterRoleRefs.`| ClusterRoles to reference to discover subjects to create RoleBindings for in the Project Release Namespace for all corresponding Project Release Roles. See RBAC above for more information | +|`hardenedNamespaces.enabled`| Whether to automatically patch the default ServiceAccount with `automountServiceAccountToken: false` and create a default NetworkPolicy in all managed namespaces in the cluster; the default values ensure that the creation of the namespace does not break a CIS 1.16 hardened scan | +|`hardenedNamespaces.configuration`| The configuration to be supplied to the default ServiceAccount or auto-generated NetworkPolicy on managing a namespace | +|`helmController.enabled`| Whether to enable an embedded k3s-io/helm-controller instance within the Helm Project Operator. Should be disabled for RKE2 clusters since RKE2 clusters already run Helm Controller to manage internal Kubernetes components | +|`helmLocker.enabled`| Whether to enable an embedded rancher/helm-locker instance within the Helm Project Operator. | diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/app-readme.md b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/app-readme.md new file mode 100644 index 0000000000..fd551467d3 --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/app-readme.md @@ -0,0 +1,20 @@ +# Helm Project Operator + +This chart installs the example [Helm Project Operator](https://github.com/rancher/helm-project-operator) onto your cluster. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + ​ +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. +​ +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. \ No newline at end of file diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/questions.yaml b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/questions.yaml new file mode 100644 index 0000000000..054361a7a4 --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/questions.yaml @@ -0,0 +1,43 @@ +questions: +- variable: global.cattle.psp.enabled + default: "false" + description: "Flag to enable or disable the installation of PodSecurityPolicies by this chart in the target cluster. If the cluster is running Kubernetes 1.25+, you must update this value to false." + label: "Enable PodSecurityPolicies" + type: boolean + group: "Security Settings" +- variable: helmController.enabled + label: Enable Embedded Helm Controller + description: 'Note: If you are running this chart in an RKE2 cluster, this should be disabled.' + type: boolean + group: Helm Controller +- variable: helmLocker.enabled + label: Enable Embedded Helm Locker + type: boolean + group: Helm Locker +- variable: projectReleaseNamespaces.labelValue + label: Project Release Namespace Project ID + description: By default, the System Project is selected. This can be overriden to a different Project (e.g. p-xxxxx) + type: string + required: false + group: Namespaces +- variable: releaseRoleBindings.clusterRoleRefs.admin + label: Admin ClusterRole + description: By default, admin selects Project Owners. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: admin + required: false + group: RBAC +- variable: releaseRoleBindings.clusterRoleRefs.edit + label: Edit ClusterRole + description: By default, edit selects Project Members. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: edit + required: false + group: RBAC +- variable: releaseRoleBindings.clusterRoleRefs.view + label: View ClusterRole + description: By default, view selects Read-Only users. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: view + required: false + group: RBAC diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/NOTES.txt b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/NOTES.txt new file mode 100644 index 0000000000..32baeebcbe --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/NOTES.txt @@ -0,0 +1,2 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "helm-project-operator.namespace" . }} get pods -l "release={{ $.Release.Name }}" diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/_helpers.tpl b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/_helpers.tpl new file mode 100644 index 0000000000..97dd6b368b --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/_helpers.tpl @@ -0,0 +1,66 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Helm Project Operator + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "helm-project-operator.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "helm-project-operator.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "helm-project-operator.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "helm-project-operator.labels" -}} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "helm-project-operator.name" . }} +chart: {{ template "helm-project-operator.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end -}} diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/cleanup.yaml b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/cleanup.yaml new file mode 100644 index 0000000000..98675642d0 --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/cleanup.yaml @@ -0,0 +1,82 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "helm-project-operator.name" . }}-cleanup + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +spec: + template: + metadata: + name: {{ template "helm-project-operator.name" . }}-cleanup + labels: {{ include "helm-project-operator.labels" . | nindent 8 }} + app: {{ template "helm-project-operator.name" . }} + spec: + serviceAccountName: {{ template "helm-project-operator.name" . }} +{{- if .Values.cleanup.securityContext }} + securityContext: {{ toYaml .Values.cleanup.securityContext | nindent 8 }} +{{- end }} + initContainers: + - name: add-cleanup-annotations + image: {{ template "system_default_registry" . }}{{ .Values.cleanup.image.repository }}:{{ .Values.cleanup.image.tag }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + command: + - /bin/sh + - -c + - > + echo "Labeling all ProjectHelmCharts with helm.cattle.io/helm-project-operator-cleanup=true"; + EXPECTED_HELM_API_VERSION={{ .Values.helmApiVersion }}; + IFS=$'\n'; + for namespace in $(kubectl get namespaces -l helm.cattle.io/helm-project-operated=true --no-headers -o=custom-columns=NAME:.metadata.name); do + for projectHelmChartAndHelmApiVersion in $(kubectl get projecthelmcharts -n ${namespace} --no-headers -o=custom-columns=NAME:.metadata.name,HELMAPIVERSION:.spec.helmApiVersion); do + projectHelmChartAndHelmApiVersion=$(echo ${projectHelmChartAndHelmApiVersion} | xargs); + projectHelmChart=$(echo ${projectHelmChartAndHelmApiVersion} | cut -d' ' -f1); + helmApiVersion=$(echo ${projectHelmChartAndHelmApiVersion} | cut -d' ' -f2); + if [[ ${helmApiVersion} != ${EXPECTED_HELM_API_VERSION} ]]; then + echo "Skipping marking ${namespace}/${projectHelmChart} with cleanup annotation since spec.helmApiVersion: ${helmApiVersion} is not ${EXPECTED_HELM_API_VERSION}"; + continue; + fi; + kubectl label projecthelmcharts -n ${namespace} ${projectHelmChart} helm.cattle.io/helm-project-operator-cleanup=true --overwrite; + done; + done; +{{- if .Values.cleanup.resources }} + resources: {{ toYaml .Values.cleanup.resources | nindent 12 }} +{{- end }} +{{- if .Values.cleanup.containerSecurityContext }} + securityContext: {{ toYaml .Values.cleanup.containerSecurityContext | nindent 12 }} +{{- end }} + containers: + - name: ensure-subresources-deleted + image: {{ template "system_default_registry" . }}{{ .Values.cleanup.image.repository }}:{{ .Values.cleanup.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + SYSTEM_NAMESPACE={{ .Release.Namespace }} + EXPECTED_HELM_API_VERSION={{ .Values.helmApiVersion }}; + HELM_API_VERSION_TRUNCATED=$(echo ${EXPECTED_HELM_API_VERSION} | cut -d'/' -f0); + echo "Ensuring HelmCharts and HelmReleases are deleted from ${SYSTEM_NAMESPACE}..."; + while [[ "$(kubectl get helmcharts,helmreleases -l helm.cattle.io/helm-api-version=${HELM_API_VERSION_TRUNCATED} -n ${SYSTEM_NAMESPACE} 2>&1)" != "No resources found in ${SYSTEM_NAMESPACE} namespace." ]]; do + echo "waiting for HelmCharts and HelmReleases to be deleted from ${SYSTEM_NAMESPACE}... sleeping 3 seconds"; + sleep 3; + done; + echo "Successfully deleted all HelmCharts and HelmReleases in ${SYSTEM_NAMESPACE}!"; +{{- if .Values.cleanup.resources }} + resources: {{ toYaml .Values.cleanup.resources | nindent 12 }} +{{- end }} +{{- if .Values.cleanup.containerSecurityContext }} + securityContext: {{ toYaml .Values.cleanup.containerSecurityContext | nindent 12 }} +{{- end }} + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.cleanup.nodeSelector }} + {{- toYaml .Values.cleanup.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.cleanup.tolerations }} + {{- toYaml .Values.cleanup.tolerations | nindent 8 }} + {{- end }} diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/clusterrole.yaml b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/clusterrole.yaml new file mode 100644 index 0000000000..60ed263ba8 --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/clusterrole.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "helm-project-operator.name" . }}-admin + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" + {{- end }} +rules: +- apiGroups: + - helm.cattle.io + resources: + - projecthelmcharts + - projecthelmcharts/finalizers + - projecthelmcharts/status + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "helm-project-operator.name" . }}-edit + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-edit: "true" + {{- end }} +rules: +- apiGroups: + - helm.cattle.io + resources: + - projecthelmcharts + - projecthelmcharts/status + verbs: + - 'get' + - 'list' + - 'watch' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "helm-project-operator.name" . }}-view + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-view: "true" + {{- end }} +rules: +- apiGroups: + - helm.cattle.io + resources: + - projecthelmcharts + - projecthelmcharts/status + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/configmap.yaml b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/configmap.yaml new file mode 100644 index 0000000000..d4def157dc --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/configmap.yaml @@ -0,0 +1,14 @@ +## Note: If you add another entry to this ConfigMap, make sure a corresponding env var is set +## in the deployment of the operator to ensure that a Helm upgrade will force the operator +## to reload the values in the ConfigMap and redeploy +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "helm-project-operator.name" . }}-config + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} +data: + hardened.yaml: |- +{{ .Values.hardenedNamespaces.configuration | toYaml | indent 4 }} + values.yaml: |- +{{ .Values.valuesOverride | toYaml | indent 4 }} diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/deployment.yaml b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/deployment.yaml new file mode 100644 index 0000000000..33b81e72e6 --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/deployment.yaml @@ -0,0 +1,126 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "helm-project-operator.name" . }} + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +spec: + {{- if .Values.replicas }} + replicas: {{ .Values.replicas }} + {{- end }} + selector: + matchLabels: + app: {{ template "helm-project-operator.name" . }} + release: {{ $.Release.Name | quote }} + template: + metadata: + labels: {{ include "helm-project-operator.labels" . | nindent 8 }} + app: {{ template "helm-project-operator.name" . }} + spec: + containers: + - name: {{ template "helm-project-operator.name" . }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + args: + - {{ template "helm-project-operator.name" . }} + - --namespace={{ template "helm-project-operator.namespace" . }} + - --controller-name={{ template "helm-project-operator.name" . }} + - --values-override-file=/etc/helmprojectoperator/config/values.yaml +{{- if .Values.global.cattle.systemDefaultRegistry }} + - --system-default-registry={{ .Values.global.cattle.systemDefaultRegistry }} +{{- end }} +{{- if .Values.global.cattle.url }} + - --cattle-url={{ .Values.global.cattle.url }} +{{- end }} +{{- if .Values.global.cattle.projectLabel }} + - --project-label={{ .Values.global.cattle.projectLabel }} +{{- end }} +{{- if not .Values.projectReleaseNamespaces.enabled }} + - --system-project-label-values={{ join "," (append .Values.otherSystemProjectLabelValues .Values.global.cattle.systemProjectId) }} +{{- else if and (ne (len .Values.global.cattle.systemProjectId) 0) (ne (len .Values.projectReleaseNamespaces.labelValue) 0) (ne .Values.projectReleaseNamespaces.labelValue .Values.global.cattle.systemProjectId) }} + - --system-project-label-values={{ join "," (append .Values.otherSystemProjectLabelValues .Values.global.cattle.systemProjectId) }} +{{- else if len .Values.otherSystemProjectLabelValues }} + - --system-project-label-values={{ join "," .Values.otherSystemProjectLabelValues }} +{{- end }} +{{- if .Values.projectReleaseNamespaces.enabled }} +{{- if .Values.projectReleaseNamespaces.labelValue }} + - --project-release-label-value={{ .Values.projectReleaseNamespaces.labelValue }} +{{- else if .Values.global.cattle.systemProjectId }} + - --project-release-label-value={{ .Values.global.cattle.systemProjectId }} +{{- end }} +{{- end }} +{{- if .Values.global.cattle.clusterId }} + - --cluster-id={{ .Values.global.cattle.clusterId }} +{{- end }} +{{- if .Values.releaseRoleBindings.aggregate }} +{{- if .Values.releaseRoleBindings.clusterRoleRefs }} +{{- if .Values.releaseRoleBindings.clusterRoleRefs.admin }} + - --admin-cluster-role={{ .Values.releaseRoleBindings.clusterRoleRefs.admin }} +{{- end }} +{{- if .Values.releaseRoleBindings.clusterRoleRefs.edit }} + - --edit-cluster-role={{ .Values.releaseRoleBindings.clusterRoleRefs.edit }} +{{- end }} +{{- if .Values.releaseRoleBindings.clusterRoleRefs.view }} + - --view-cluster-role={{ .Values.releaseRoleBindings.clusterRoleRefs.view }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.hardenedNamespaces.enabled }} + - --hardening-options-file=/etc/helmprojectoperator/config/hardening.yaml +{{- else }} + - --disable-hardening +{{- end }} +{{- if .Values.debug }} + - --debug + - --debug-level={{ .Values.debugLevel }} +{{- end }} +{{- if not .Values.helmController.enabled }} + - --disable-embedded-helm-controller +{{- else }} + - --helm-job-image={{ template "system_default_registry" . }}{{ .Values.helmController.job.image.repository }}:{{ .Values.helmController.job.image.tag }} +{{- end }} +{{- if not .Values.helmLocker.enabled }} + - --disable-embedded-helm-locker +{{- end }} +{{- if .Values.additionalArgs }} +{{- toYaml .Values.additionalArgs | nindent 10 }} +{{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ## Note: The below two values only exist to force Helm to upgrade the deployment on + ## a change to the contents of the ConfigMap during an upgrade. Neither serve + ## any practical purpose and can be removed and replaced with a configmap reloader + ## in a future change if dynamic updates are required. + - name: HARDENING_OPTIONS_SHA_256_HASH + value: {{ .Values.hardenedNamespaces.configuration | toYaml | sha256sum }} + - name: VALUES_OVERRIDE_SHA_256_HASH + value: {{ .Values.valuesOverride | toYaml | sha256sum }} +{{- if .Values.resources }} + resources: {{ toYaml .Values.resources | nindent 12 }} +{{- end }} +{{- if .Values.containerSecurityContext }} + securityContext: {{ toYaml .Values.containerSecurityContext | nindent 12 }} +{{- end }} + volumeMounts: + - name: config + mountPath: "/etc/helmprojectoperator/config" + serviceAccountName: {{ template "helm-project-operator.name" . }} +{{- if .Values.securityContext }} + securityContext: {{ toYaml .Values.securityContext | nindent 8 }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{- toYaml .Values.nodeSelector | nindent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{- toYaml .Values.tolerations | nindent 8 }} +{{- end }} + volumes: + - name: config + configMap: + name: {{ template "helm-project-operator.name" . }}-config diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/psp.yaml b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/psp.yaml new file mode 100644 index 0000000000..73dcc4560e --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/psp.yaml @@ -0,0 +1,68 @@ +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "helm-project-operator.name" . }}-psp + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +{{- if .Values.global.rbac.pspAnnotations }} + annotations: {{ toYaml .Values.global.rbac.pspAnnotations | nindent 4 }} +{{- end }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "helm-project-operator.name" . }}-psp + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +rules: +{{- if semverCompare "> 1.15.0-0" .Capabilities.KubeVersion.GitVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "helm-project-operator.name" . }}-psp +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "helm-project-operator.name" . }}-psp + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "helm-project-operator.name" . }}-psp +subjects: + - kind: ServiceAccount + name: {{ template "helm-project-operator.name" . }} + namespace: {{ template "helm-project-operator.namespace" . }} +{{- end }} diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/rbac.yaml b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/rbac.yaml new file mode 100644 index 0000000000..b1c4092029 --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/rbac.yaml @@ -0,0 +1,32 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "helm-project-operator.name" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: "cluster-admin" # see note below +subjects: +- kind: ServiceAccount + name: {{ template "helm-project-operator.name" . }} + namespace: {{ template "helm-project-operator.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "helm-project-operator.name" . }} + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | nindent 2 }} +{{- end }} +# --- +# NOTE: +# As of now, due to the fact that the k3s-io/helm-controller can only deploy jobs that are cluster-bound to the cluster-admin +# ClusterRole, the only way for this operator to be able to perform that binding is if it is also bound to the cluster-admin ClusterRole. +# +# As a result, this ClusterRoleBinding will be left as a work-in-progress until changes are made in k3s-io/helm-controller to allow us to grant +# only scoped down permissions to the Job that is deployed. diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/system-namespaces-configmap.yaml b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/system-namespaces-configmap.yaml new file mode 100644 index 0000000000..f4c85254e8 --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/system-namespaces-configmap.yaml @@ -0,0 +1,62 @@ +{{- if .Values.systemNamespacesConfigMap.create }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "helm-project-operator.name" . }}-system-namespaces + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} +data: + system-namespaces.json: |- + { +{{- if .Values.projectReleaseNamespaces.enabled }} +{{- if .Values.projectReleaseNamespaces.labelValue }} + "projectReleaseLabelValue": {{ .Values.projectReleaseNamespaces.labelValue | quote }}, +{{- else if .Values.global.cattle.systemProjectId }} + "projectReleaseLabelValue": {{ .Values.global.cattle.systemProjectId | quote }}, +{{- else }} + "projectReleaseLabelValue": "", +{{- end }} +{{- else }} + "projectReleaseLabelValue": "", +{{- end }} +{{- if not .Values.projectReleaseNamespaces.enabled }} + "systemProjectLabelValues": {{ append .Values.otherSystemProjectLabelValues .Values.global.cattle.systemProjectId | toJson }} +{{- else if and (ne (len .Values.global.cattle.systemProjectId) 0) (ne (len .Values.projectReleaseNamespaces.labelValue) 0) (ne .Values.projectReleaseNamespaces.labelValue .Values.global.cattle.systemProjectId) }} + "systemProjectLabelValues": {{ append .Values.otherSystemProjectLabelValues .Values.global.cattle.systemProjectId | toJson }} +{{- else if len .Values.otherSystemProjectLabelValues }} + "systemProjectLabelValues": {{ .Values.otherSystemProjectLabelValues | toJson }} +{{- else }} + "systemProjectLabelValues": [] +{{- end }} + } +--- +{{- if (and .Values.systemNamespacesConfigMap.rbac.enabled .Values.systemNamespacesConfigMap.rbac.subjects) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "helm-project-operator.name" . }}-system-namespaces + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "{{ template "helm-project-operator.name" . }}-system-namespaces" + verbs: + - 'get' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "helm-project-operator.name" . }}-system-namespaces + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "helm-project-operator.name" . }}-system-namespaces +subjects: {{ .Values.systemNamespacesConfigMap.rbac.subjects | toYaml | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/validate-psp-install.yaml b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/values.yaml b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/values.yaml new file mode 100644 index 0000000000..63fae45af8 --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/charts/helmProjectOperator/values.yaml @@ -0,0 +1,228 @@ +# Default values for helm-project-operator. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Helm Project Operator Configuration + +global: + cattle: + clusterId: "" + psp: + enabled: false + projectLabel: field.cattle.io/projectId + systemDefaultRegistry: "" + systemProjectId: "" + url: "" + rbac: + ## Create RBAC resources for ServiceAccounts and users + ## + create: true + + userRoles: + ## Create default user ClusterRoles to allow users to interact with ProjectHelmCharts + create: true + ## Aggregate default user ClusterRoles into default k8s ClusterRoles + aggregateToDefaultRoles: true + + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + +helmApiVersion: dummy.cattle.io/v1alpha1 + +## valuesOverride overrides values that are set on each ProjectHelmChart deployment on an operator-level +## User-provided values will be overwritten based on the values provided here +valuesOverride: {} + +## projectReleaseNamespaces are auto-generated namespaces that are created to host Helm Releases +## managed by this operator on behalf of a ProjectHelmChart +projectReleaseNamespaces: + ## Enabled determines whether Project Release Namespaces should be created. If false, the underlying + ## Helm release will be deployed in the Project Registration Namespace + enabled: true + ## labelValue is the value of the Project that the projectReleaseNamespace should be created within + ## If empty, this will be set to the value of global.cattle.systemProjectId + ## If global.cattle.systemProjectId is also empty, project release namespaces will be disabled + labelValue: "" + +## otherSystemProjectLabelValues are project labels that identify namespaces as those that should be treated as system projects +## i.e. they will be entirely ignored by the operator +## By default, the global.cattle.systemProjectId will be in this list +otherSystemProjectLabelValues: [] + +## releaseRoleBindings configures RoleBindings automatically created by the Helm Project Operator +## in Project Release Namespaces where underlying Helm charts are deployed +releaseRoleBindings: + ## aggregate enables creating these RoleBindings off aggregating RoleBindings in the + ## Project Registration Namespace or ClusterRoleBindings that bind users to the ClusterRoles + ## specified under clusterRoleRefs + aggregate: true + + ## clusterRoleRefs are the ClusterRoles whose RoleBinding or ClusterRoleBindings should determine + ## the RoleBindings created in the Project Release Namespace + ## + ## By default, these are set to create RoleBindings based on the RoleBindings / ClusterRoleBindings + ## attached to the default K8s user-facing ClusterRoles of admin, edit, and view. + ## ref: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles + ## + clusterRoleRefs: + admin: admin + edit: edit + view: view + +hardenedNamespaces: + # Whether to automatically manage the configuration of the default ServiceAccount and + # auto-create a NetworkPolicy for each namespace created by this operator + enabled: true + + configuration: + # Values to be applied to each default ServiceAccount created in a managed namespace + serviceAccountSpec: + secrets: [] + imagePullSecrets: [] + automountServiceAccountToken: false + # Values to be applied to each default generated NetworkPolicy created in a managed namespace + networkPolicySpec: + podSelector: {} + egress: [] + ingress: [] + policyTypes: ["Ingress", "Egress"] + +## systemNamespacesConfigMap is a ConfigMap created to allow users to see valid entries +## for registering a ProjectHelmChart for a given Project on the Rancher Dashboard UI. +## It does not need to be enabled for a non-Rancher use case. +systemNamespacesConfigMap: + ## Create indicates whether the system namespaces configmap should be created + ## This is a required value for integration with Rancher Dashboard + create: true + + ## RBAC provides options around the RBAC created to allow users to be able to view + ## the systemNamespacesConfigMap; if not specified, only users with the ability to + ## view ConfigMaps in the namespace where this chart is deployed will be able to + ## properly view the system namespaces on the Rancher Dashboard UI + rbac: + ## enabled indicates that we should deploy a RoleBinding and Role to view this ConfigMap + enabled: true + ## subjects are the subjects that should be bound to this default RoleBinding + ## By default, we allow anyone who is authenticated to the system to be able to view + ## this ConfigMap in the deployment namespace + subjects: + - kind: Group + name: system:authenticated + +nameOverride: "" + +namespaceOverride: "" + +replicas: 1 + +image: + repository: rancher/helm-project-operator + tag: v0.2.1 + pullPolicy: IfNotPresent + +helmController: + # Note: should be disabled for RKE2 clusters since they already run Helm Controller to manage internal Kubernetes components + enabled: true + + job: + image: + repository: rancher/klipper-helm + tag: v0.7.0-build20220315 + +helmLocker: + enabled: true + +# Additional arguments to be passed into the Helm Project Operator image +additionalArgs: [] + +## Define which Nodes the Pods are scheduled on. +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## Tolerations for use with node taints +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] +# - key: "key" +# operator: "Equal" +# value: "value" +# effect: "NoSchedule" + +resources: {} + # limits: + # memory: 500Mi + # cpu: 1000m + # requests: + # memory: 100Mi + # cpu: 100m + +containerSecurityContext: {} + # allowPrivilegeEscalation: false + # capabilities: + # drop: + # - ALL + # privileged: false + # readOnlyRootFilesystem: true + +securityContext: {} + # runAsGroup: 1000 + # runAsUser: 1000 + # supplementalGroups: + # - 1000 + +debug: false +debugLevel: 0 + +cleanup: + image: + repository: rancher/shell + tag: v0.1.19 + pullPolicy: IfNotPresent + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + containerSecurityContext: {} + # allowPrivilegeEscalation: false + # capabilities: + # drop: + # - ALL + # privileged: false + # readOnlyRootFilesystem: true + + securityContext: + runAsNonRoot: false + runAsUser: 0 + + resources: {} + # limits: + # memory: 500Mi + # cpu: 1000m + # requests: + # memory: 100Mi + # cpu: 100m diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/questions.yaml b/charts/prometheus-federator/0.3.0+up0.3.3/questions.yaml new file mode 100644 index 0000000000..981e1c0676 --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/questions.yaml @@ -0,0 +1,37 @@ +questions: +- variable: helmProjectOperator.helmController.enabled + label: Enable Embedded Helm Controller + description: 'Note: If you are running Prometheus Federator in an RKE2 / K3s cluster before v1.23.14 / v1.24.8 / v1.25.4, this should be disabled.' + type: boolean + group: Helm Controller +- variable: helmProjectOperator.helmLocker.enabled + label: Enable Embedded Helm Locker + type: boolean + group: Helm Locker +- variable: helmProjectOperator.projectReleaseNamespaces.labelValue + label: Project Release Namespace Project ID + description: By default, the System Project is selected. This can be overriden to a different Project (e.g. p-xxxxx) + type: string + required: false + group: Namespaces +- variable: helmProjectOperator.releaseRoleBindings.clusterRoleRefs.admin + label: Admin ClusterRole + description: By default, admin selects Project Owners. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: admin + required: false + group: RBAC +- variable: helmProjectOperator.releaseRoleBindings.clusterRoleRefs.edit + label: Edit ClusterRole + description: By default, edit selects Project Members. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: edit + required: false + group: RBAC +- variable: helmProjectOperator.releaseRoleBindings.clusterRoleRefs.view + label: View ClusterRole + description: By default, view selects Read-Only users. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: view + required: false + group: RBAC \ No newline at end of file diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/templates/NOTES.txt b/charts/prometheus-federator/0.3.0+up0.3.3/templates/NOTES.txt new file mode 100644 index 0000000000..f551f36613 --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/templates/NOTES.txt @@ -0,0 +1,3 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "prometheus-federator.namespace" . }} get pods -l "release={{ $.Release.Name }}" + diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/templates/_helpers.tpl b/charts/prometheus-federator/0.3.0+up0.3.3/templates/_helpers.tpl new file mode 100644 index 0000000000..15ea4e5c88 --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/templates/_helpers.tpl @@ -0,0 +1,66 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Helm Project Operator + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "prometheus-federator.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-federator.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "prometheus-federator.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "prometheus-federator.labels" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "prometheus-federator.name" . }} +chart: {{ template "prometheus-federator.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end }} diff --git a/charts/prometheus-federator/0.3.0+up0.3.3/values.yaml b/charts/prometheus-federator/0.3.0+up0.3.3/values.yaml new file mode 100644 index 0000000000..699e6c9ba6 --- /dev/null +++ b/charts/prometheus-federator/0.3.0+up0.3.3/values.yaml @@ -0,0 +1,94 @@ +# Default values for helm-project-operator. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Prometheus Federator Configuration + +global: + cattle: + psp: + enabled: true + systemDefaultRegistry: "" + projectLabel: field.cattle.io/projectId + clusterId: "" + systemProjectId: "" + url: "" + rbac: + pspEnabled: true + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + +helmProjectOperator: + enabled: true + + # ensures that all resources created by subchart show up as prometheus-federator + helmApiVersion: monitoring.cattle.io/v1alpha1 + + nameOverride: prometheus-federator + + helmController: + # Note: should be disabled for RKE2 clusters since they already run Helm Controller to manage internal Kubernetes components + enabled: true + + helmLocker: + enabled: true + + ## valuesOverride overrides values that are set on each Project Prometheus Stack Helm Chart deployment on an operator level + ## all values provided here will override any user-provided values automatically + valuesOverride: + + federate: + # Change this to point at all Prometheuses you want all your Project Prometheus Stacks to federate from + # By default, this matches the default deployment of Rancher Monitoring + targets: + - rancher-monitoring-prometheus.cattle-monitoring-system.svc:9090 + + image: + repository: rancher/prometheus-federator + tag: v0.3.2 + pullPolicy: IfNotPresent + + # Additional arguments to be passed into the Prometheus Federator image + additionalArgs: [] + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + resources: {} + # limits: + # memory: 500Mi + # cpu: 1000m + # requests: + # memory: 100Mi + # cpu: 100m + + securityContext: {} + # allowPrivilegeEscalation: false + # readOnlyRootFilesystem: true + + debug: false + debugLevel: 0 \ No newline at end of file diff --git a/index.yaml b/index.yaml index 4301792c63..d0db2fe910 100755 --- a/index.yaml +++ b/index.yaml @@ -5599,6 +5599,30 @@ entries: urls: - assets/prometheus-federator/prometheus-federator-1.0.0+up0.1.1.tgz version: 1.0.0+up0.1.1 + - annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Prometheus Federator + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.25.0-0' + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/os: linux,windows + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: helm.cattle.io.projecthelmchart/v1alpha1 + catalog.cattle.io/rancher-version: '>= 2.6.5-0 < 2.7.0-0' + catalog.cattle.io/release-name: prometheus-federator + apiVersion: v2 + appVersion: 0.3.2 + created: "2023-09-21T15:07:20.483796-07:00" + dependencies: + - condition: helmProjectOperator.enabled + name: helmProjectOperator + repository: file://./charts/helmProjectOperator + description: Prometheus Federator + digest: 47127ade201ff9cdcb6db5f90c9396a9470addb6d37c1f0a657817bbf6433572 + icon: https://raw.githubusercontent.com/rancher/prometheus-federator/main/assets/logos/prometheus-federator.svg + name: prometheus-federator + urls: + - assets/prometheus-federator/prometheus-federator-0.3.0+up0.3.3.tgz + version: 0.3.0+up0.3.3 - annotations: catalog.cattle.io/certified: rancher catalog.cattle.io/display-name: Prometheus Federator diff --git a/release.yaml b/release.yaml index f7660d6880..9d6e1cb0d1 100644 --- a/release.yaml +++ b/release.yaml @@ -2,3 +2,5 @@ longhorn: - 103.1.1+up1.4.4 longhorn-crd: - 103.1.1+up1.4.4 +prometheus-federator: + - 0.3.0+up0.3.3 From 76e8a78b24d9ea91ea3d2895717c921b747f7ad8 Mon Sep 17 00:00:00 2001 From: Venkata Krishna Rohit Sakala Date: Thu, 21 Sep 2023 15:20:03 -0700 Subject: [PATCH 5/5] make forward-port project-monitoring 0.3.0+up0.3.3 --- ...ncher-project-monitoring-0.3.0+up0.3.3.tgz | Bin 0 -> 124827 bytes .../0.3.0+up0.3.3/Chart.yaml | 32 + .../0.3.0+up0.3.3/README.md | 29 + .../0.3.0+up0.3.3/app-README.md | 10 + .../0.3.0+up0.3.3/charts/grafana/.helmignore | 23 + .../0.3.0+up0.3.3/charts/grafana/Chart.yaml | 30 + .../0.3.0+up0.3.3/charts/grafana/README.md | 571 ++++ .../grafana/dashboards/custom-dashboard.json | 1 + .../charts/grafana/templates/NOTES.txt | 54 + .../charts/grafana/templates/_helpers.tpl | 214 ++ .../charts/grafana/templates/_pod.tpl | 885 ++++++ .../charts/grafana/templates/clusterrole.yaml | 25 + .../grafana/templates/clusterrolebinding.yaml | 24 + .../configmap-dashboard-provider.yaml | 29 + .../charts/grafana/templates/configmap.yaml | 117 + .../templates/dashboards-json-configmap.yaml | 35 + .../charts/grafana/templates/deployment.yaml | 50 + .../grafana/templates/headless-service.yaml | 22 + .../charts/grafana/templates/hpa.yaml | 21 + .../templates/image-renderer-deployment.yaml | 123 + .../image-renderer-network-policy.yaml | 73 + .../templates/image-renderer-service.yaml | 33 + .../charts/grafana/templates/ingress.yaml | 78 + .../grafana/templates/networkpolicy.yaml | 52 + .../grafana/templates/nginx-config.yaml | 94 + .../templates/poddisruptionbudget.yaml | 22 + .../grafana/templates/podsecuritypolicy.yaml | 45 + .../charts/grafana/templates/pvc.yaml | 35 + .../charts/grafana/templates/role.yaml | 32 + .../charts/grafana/templates/rolebinding.yaml | 25 + .../charts/grafana/templates/secret-env.yaml | 14 + .../charts/grafana/templates/secret.yaml | 26 + .../charts/grafana/templates/service.yaml | 55 + .../grafana/templates/serviceaccount.yaml | 14 + .../grafana/templates/servicemonitor.yaml | 58 + .../charts/grafana/templates/statefulset.yaml | 56 + .../templates/tests/test-configmap.yaml | 17 + .../tests/test-podsecuritypolicy.yaml | 29 + .../grafana/templates/tests/test-role.yaml | 14 + .../templates/tests/test-rolebinding.yaml | 17 + .../templates/tests/test-serviceaccount.yaml | 9 + .../charts/grafana/templates/tests/test.yaml | 51 + .../0.3.0+up0.3.3/charts/grafana/values.yaml | 1062 +++++++ .../federate/federate-scrape-config.yaml | 13 + .../rancher/pods/rancher-pod-containers.json | 636 ++++ .../files/rancher/pods/rancher-pod.json | 636 ++++ .../workloads/rancher-workload-pods.json | 652 ++++ .../rancher/workloads/rancher-workload.json | 652 ++++ .../0.3.0+up0.3.3/questions.yaml | 128 + .../0.3.0+up0.3.3/templates/NOTES.txt | 4 + .../0.3.0+up0.3.3/templates/_helpers.tpl | 252 ++ .../templates/alertmanager/alertmanager.yaml | 165 + .../templates/alertmanager/extrasecret.yaml | 20 + .../templates/alertmanager/ingress.yaml | 77 + .../alertmanager/podDisruptionBudget.yaml | 21 + .../templates/alertmanager/psp-role.yaml | 21 + .../alertmanager/psp-rolebinding.yaml | 18 + .../templates/alertmanager/psp.yaml | 45 + .../templates/alertmanager/secret.yaml | 33 + .../templates/alertmanager/service.yaml | 53 + .../alertmanager/serviceaccount.yaml | 20 + .../alertmanager/servicemonitor.yaml | 55 + .../templates/dashboard-roles.yaml | 179 ++ .../templates/dashboard-values-configmap.yaml | 61 + .../dashboards/rancher/pods-dashboards.yaml | 17 + .../rancher/workload-dashboards.yaml | 17 + .../grafana/configmap-dashboards.yaml | 24 + .../grafana/configmaps-datasources.yaml | 46 + .../alertmanager-overview.yaml | 616 ++++ .../dashboards-1.14/cluster-total.yaml | 1646 ++++++++++ .../dashboards-1.14/grafana-overview.yaml | 635 ++++ .../k8s-resources-namespace.yaml | 2770 +++++++++++++++++ .../dashboards-1.14/k8s-resources-node.yaml | 963 ++++++ .../dashboards-1.14/k8s-resources-pod.yaml | 2442 +++++++++++++++ .../k8s-resources-project.yaml | 2480 +++++++++++++++ .../k8s-resources-workload.yaml | 1997 ++++++++++++ .../k8s-resources-workloads-namespace.yaml | 2162 +++++++++++++ .../dashboards-1.14/namespace-by-pod.yaml | 1438 +++++++++ .../namespace-by-workload.yaml | 1710 ++++++++++ .../persistentvolumesusage.yaml | 561 ++++ .../grafana/dashboards-1.14/pod-total.yaml | 1202 +++++++ .../prometheus-remote-write.yaml | 1674 ++++++++++ .../grafana/dashboards-1.14/prometheus.yaml | 1235 ++++++++ .../dashboards-1.14/workload-total.yaml | 1412 +++++++++ .../templates/prometheus/_rules.tpl | 8 + .../templates/prometheus/clusterrole.yaml | 25 + .../prometheus/clusterrolebinding.yaml | 18 + .../templates/prometheus/extrasecret.yaml | 20 + .../templates/prometheus/federate.yaml | 10 + .../templates/prometheus/ingress.yaml | 77 + .../templates/prometheus/nginx-config.yaml | 68 + .../prometheus/podDisruptionBudget.yaml | 21 + .../templates/prometheus/prometheus.yaml | 319 ++ .../templates/prometheus/psp-clusterrole.yaml | 20 + .../prometheus/psp-clusterrolebinding.yaml | 18 + .../templates/prometheus/psp.yaml | 52 + .../rules-1.14/alertmanager.rules.yaml | 217 ++ .../prometheus/rules-1.14/general.rules.yaml | 98 + .../rules-1.14/kubernetes-apps.yaml | 375 +++ .../rules-1.14/kubernetes-storage.yaml | 160 + .../prometheus/rules-1.14/prometheus.yaml | 448 +++ .../templates/prometheus/service.yaml | 56 + .../templates/prometheus/serviceaccount.yaml | 20 + .../templates/prometheus/servicemonitor.yaml | 52 + .../rancher-monitoring/hardened.yaml | 129 + .../templates/validate-install-crd.yaml | 21 + .../templates/validate-psp-install.yaml | 7 + .../0.3.0+up0.3.3/values.yaml | 1551 +++++++++ index.yaml | 36 + release.yaml | 2 + 110 files changed, 36822 insertions(+) create mode 100644 assets/rancher-project-monitoring/rancher-project-monitoring-0.3.0+up0.3.3.tgz create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/Chart.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/README.md create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/app-README.md create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/.helmignore create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/Chart.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/README.md create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/dashboards/custom-dashboard.json create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/NOTES.txt create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/_helpers.tpl create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/_pod.tpl create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/clusterrole.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/configmap-dashboard-provider.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/configmap.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/dashboards-json-configmap.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/deployment.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/headless-service.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/hpa.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/image-renderer-deployment.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/image-renderer-network-policy.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/image-renderer-service.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/ingress.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/networkpolicy.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/nginx-config.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/poddisruptionbudget.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/pvc.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/role.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/rolebinding.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/secret-env.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/secret.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/service.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/serviceaccount.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/servicemonitor.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/statefulset.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-configmap.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-role.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-rolebinding.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-serviceaccount.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/values.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/files/federate/federate-scrape-config.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/files/rancher/pods/rancher-pod-containers.json create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/files/rancher/pods/rancher-pod.json create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/files/rancher/workloads/rancher-workload-pods.json create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/files/rancher/workloads/rancher-workload.json create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/questions.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/NOTES.txt create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/_helpers.tpl create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/alertmanager.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/extrasecret.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/ingress.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/podDisruptionBudget.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/psp-role.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/psp-rolebinding.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/psp.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/secret.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/service.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/serviceaccount.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/servicemonitor.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/dashboard-roles.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/dashboard-values-configmap.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/dashboards/rancher/pods-dashboards.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/dashboards/rancher/workload-dashboards.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/configmap-dashboards.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/configmaps-datasources.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/cluster-total.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/grafana-overview.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-project.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/pod-total.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/prometheus.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/workload-total.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/_rules.tpl create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/clusterrole.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/clusterrolebinding.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/extrasecret.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/federate.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/ingress.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/nginx-config.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/podDisruptionBudget.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/prometheus.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/psp-clusterrole.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/psp.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/general.rules.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/prometheus.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/service.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/serviceaccount.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/servicemonitor.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/rancher-monitoring/hardened.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/validate-install-crd.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/validate-psp-install.yaml create mode 100644 charts/rancher-project-monitoring/0.3.0+up0.3.3/values.yaml diff --git a/assets/rancher-project-monitoring/rancher-project-monitoring-0.3.0+up0.3.3.tgz b/assets/rancher-project-monitoring/rancher-project-monitoring-0.3.0+up0.3.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..4c85f491268afac1f1baeca345134c7ad21b6c6a GIT binary patch literal 124827 zcmV(^K-Iq=iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0POwge%!W_FpTH3t^)s-XKZ=Yn-<&YnauMjwv*@?+tFC^Iddk- zqo7H2H;&i@1AtnN?fLz$<-N4`O5Ps|3jsFCW^413<_B9%0)?$ms4CRLaT?7C?`E9+ zokX%bXDOA8({$Q@KEqu0uJAnh=4Q`eFc>`9+k^iHgF*TKo!yRqpTkh zF6o4lc!UhDR2GY=$P#?jP4S$J(2>R_^x9!lWu+O7W0HDvO-uW70h*Tu_#d z(8GWK4h?(5C%r*;fc^~)dprB;&xe&&Gs5RoiY^l)l+ZN4+`gb`%q~P_X}b}Ewsv|? zdi&7SPVZ@bOPnMG3t|Rezoj;CM*m{rpYLqoEOS2xz5Q={!;P3^B#lWLQ6fegT@g_IiBDY_ehGz35hol((Re|8whcdF`?SJ8leeI$f)1%^&_>7Mc*g#$#6q77rS0qN~6#MUg zhVC%FF=J|yXHsTj)bCHJoaN(Q#OD3X749-qSfgw4VS`3$nr&r{FM1k4E(niUDoH9! zu)IG_S%qP9ta{$ZLJ;W}IZfD6}?e0D4W$AR|j9gtX9s>e-o#}dWw=u^w zm6)c4LoqVPG#Mex&uJR>=oLw)nE!8)3!?hxaP0uTd*$aiof0B@0?#>KEdL`Ras#29 zCzVsEs?PH9w@){gZXJ1*krBdKme2_BY2&=uI=j8WzvP+ve|O_kQ`9f(X*mDSagq~p z3k*Qx`9Iuy`egrU`TT#nyR$z3@8j7-FUSPvi9~w2(S&hik-?*ILgMJGx3P&%W>g?5 z5JrDHc>St7VSJ9IIx^IOqYAtrQGz)^=a^Goi?Jk zv9XEXYDBDH9nf=@PUtk}dfRVoZmL3Xc3>bgh^8n@uqN|$eTRYlj%f3)QD{IPN$TK);fV}K{ML#w0}-Gnx<- zae}2n88vj8PB{_6HcgiDekk&lI>GM5C9M!}loNun1+;TR`^ZEqm7xryGAIFE>$s{-qF6G%iXQy_4by|JH!Mwt=n z^d5H%2s7hq8iI-0KxWIs6hqO2Lhh+7@5@E1lBpGIW*jF5Ak;0mMm4v(&j7=fR`0z< zUI2zmy=e#Wpt6rgoJ{B?fZ=&MX6y{Td;3cC&^uwsg<`gSdNZM1NR(kN6>_ULrlP8c z1GLda6^GYxjIMqudKEl!FDB=%fL*y4RDe)ZJM!2R&;#Av+?Xb8 zj1wcX%7kdl%|xdEK*)$mBQ(K@&HfeXgmBcN`5^RXi^5&Kv-lWC z1}ELY_V)XO=fzP8>wQf4IgQ9c6tO&&dfk*9Z6 zMi2|yyTCG72~&fY>|fPRrV_ESApBfyG282+(-;mM8s!P*i0dvE z?Jw^udPI8D9wIT7TYw65f-tOtR{InOm+R~J%I_{Hk5oHwk{3-soRMVis}T*$u3mSc zYFBl)i>FggrdX1$q8krK=;6Pq_uI(z>)#)4t63=Hj6)ghBTdl(vs6Z@=#6-+OByVQ zW(6uprbnyb^(t`c4O4qU^T=Al&=@i%Gki`NN17|h1u4_N3S8QTG7d#OY2}f3+)qfX z3v3nLh;XWiN@V0Y65B>Gis@uRxB?;wupwT90$~AdYH9#X3eTn467ig-+c4i+D|t@I z1p?i#5I_mb^HJh^P2UdU*z$CeX<}eLx`jSrmURGf;?bB)WMyW|0Y1)1L?>4$W3kWY z6$RdsNueAocx2G3vq8;VCk$^=yS6PO=&9h0U1rKa0O)!#6y0-ulaBstdIGW_l)>KgTK#{f{U zbk{rZ9q>>Ut17TzNCU9!o12zVM~b>dz=B@P6cOQYOvEuV<*nu+y>jQWr>{SjI64!G z#+$j;z!ew7q-|pxrOXFs*&Lbkl*EXs6UfO8PBul)Jza}rC}T902umQuMHh$w4a@~j z^%qcb)%R4oo)F+XH_Gf3G&fggKM?`d_9!l!IKDBsYm*=x1IyEOTJ=IT;AzLqN#}pm zVp30>N26~C-wq7U9!DH!>Xep*pJTOq?Vlsm8SV?M^aylyhG!|X1RG_v_4;Aa7Ru9f z+V?B=mB<(98?4|6a}hRTuy=7G7sdXAih4Doe@68gTXb2Y4;-2d#N#{#1lFvD-=!Wp zVnWbyLadq;GrseQ>o_4)!B7eB?xW6g%ajfths|hGt z*Ntx&CLA;#J}y&%=zN|_g;YqLa%Si;a2mAS5+fsQ%$=qU<1JNd|oT= z2G5pw*e5ay4{p5?`;U8ao+Xz5J;Fk>G_6RDQ>FbxlKDeno7J2SFpyK463}=pfoTpq zQ948IFt#_W^5N=XlAx40s-1Bq?hzMsSMuIt2(G?u#ipzvm%s5c{@s$Rq`dmRP zLCGLyno7pxoFpux>GafC>ulx%DBFy5WtMa&c?xR1r9`rk=&BVs6_UG&YYix6l1|Kl zD*895d}^)+0f@{J%?a3(s^a*uYXjHInrVg`=i+8yv&1!<3O_)3wK8xuti%gci=9S+ zR4hN5;d6p80<49$k;o?#dg(-lgyERPJ#a!F{fVSKXE>MbjwB;Yx=IQZ%vRL47w%y}79tH&`1@GmLC< zYSvG{gpbgNt|MCS-*?sO(v+Z%NN{x4Gj)2BT*?mWe*EZ_EfA*T7dxHjiKJOVI;e+& z3SC6f7=To^SOdgw^zko0zWw>#5$d8BL}HqV-yXYfKYOx0PB5Pft=^+T9|;=IB&X9U z;fj?+{uK7YKG6x9O7tipDeC>GYU?Y--d{kZ*g^w;WRIbQxe^^c_RatWii1EKKsjby zlK2QyE_^r${lEYJ{|7yO{~q<;s8i$P$I)Z-yZQ;uqK_YcFTqyxSuw?>S9F4UFE1tG zDNf$KeHC>56;00+Mgt{yZ3Mo@YIWX%DZuZIEeI3*v>v3Y4%f3fWED>Py6?fe#u#&`4> z`D3jp*#O>??|UMj&oM8JLccS}w4`*)cIHF~JPl|4wSHI8PDSaK?Rb{EL0d08i{8cd zYHGD?)7;aa<~#}7d|Ox+j2@$ZD>oWe`!Et2|?Jb$B zbMKMnXO0sZ5!7)Tqk}pYGdrkbaHz9YFjLjK(GQIK*J;{+;Tb53#f;@ijEq9c zH(3Lv9A`;FIGCktBOQ{&w!rfQ3!%ntT2V}BwPI|XuxWp1urufn_WFaJKAmTros&69 zrRYiq5{7LRXS7SxZULaH`WmRrRPjhP-er?6PHk1wjdJfsNJ(aw8+EF{z${C{06Q>{ z{0f@{hH&bcKA8hHMqqMiO#O{&(sac9I$~Uo(6hm_p@aSt6H;g^NiEi$)0n{Jv9%JR z!Y-w;Skd=WGgZ-CfapfLp^7(Vb4*jPez%PsVL5Xbs}sfxOARpB>%!GfhULuS0*LG| z9v}2^Mxq{w{o1z6@0O}vbW3%r@0wo?_Y|a)tBjyWF_DDNX-Wi|u?sgb(3Gt?CPu#y zZ`tPV7752hO2WIU6J2Oop`#*V`jx!Oh($v?k`X#o0*LcK(KoR)auam&>bRsJSkSK@ z`UtMZ7;z#U{>*WjdqyC!y}J)BD$G*JBwQ$cAEx(-zuaA8e%Y0Y9g#&p_p^k1M%1#% zS9uQ;3`@6N==IgFQQbCe1J3R~|DK>4U}@W{pXPi%JhDn?y?fO5UTDhRh)9MK@b1yK zcEyH(J^$_C+g-QH-d!qO%&2XDd4=EzRmA1t8x{j)dg!;7nwKNAJNS0s8hb?rT=E?rDFUc4QAzyL zSfCmE9D5xF#x`y-!1l^&_Y30Me1(z=g?~AXN#B4G{Y_$It}e}-t(67n9q4KTh4JKS z#1Y?)RgKx-7UeH)w`k=f&gG1x60X@|w;Qs8&f8c1>J@>kZi;*4!Tr zO{XDKfFtdhdqzm+h#4nSaydc|6~R7z6k{@HsUY%0VF>aeWAO*ab$n2y<%5t|l1ZKj zBDZ?~^3Xxk_iWZ3;g{1>RqyolH;)5fI@4xW?Nop2IpsC8xiHi@k@&PY_f9hwpQ_1% zIry!ZB9dX~(Fpx(c@_Tequ%4K2c?6?IOr%)X(5)u^DL}5D4CFACztS{)ai+MDol0-7+M0Hz<%;$iA_ zLO6{BV<0d)$vDgK6dPlWg}cOz#UeZ-SZv`2BJANDPr*#JWW!RNW~xA8kg(`5GN+s~ zPGaAU5to5kSCT<*XRkLfufz-ohlVlT(=MBdVJF>7BRu1|Z)c1v?q>DETGt0L!}=C% z&hk`RuTD90d}dJhNX;AAwcB4)08R6rEtBms*UkgwW%qxrY0{hGbNzfP%`@IqBElCns#RJ)UtN+M(7QbhpG2P zTS)p=E7eAf{Jfxp|6AA*Z9*+(a)8?+gsRUJ`_jby)zxXHXhI zJ8rMov!Gi_w*0$bsroRrE0(aS zt~fH~9cTqp&sCMFY3RJMfM^MuzEX8ZaNZY35d83Y1~b5V6memRDEE0 zif}NzAnJ_QDz`a4)0UPVB)6gSG*#S_y*vaHFMFm3UFaG95$r|92o;**6FMg-wq!3< zWz4iAx1x^-xQYqmj^=@qXSg$%J=6{Vd!HH(_J8M{Ej z6tLDUuyw{)i)VzT@Z!$ps6rJb`lfrmc^q&9){ES3QFP9dd`_Mxn9fhk?S8S#M8UBl zuex_zF?9igY#Hs``I^O``02bQIQ}cAlKh-Tr1P6s(9ui31q2qsB+%R<6m)%m@FVs6 zs#y~2g=<{J7t7K~sX<}X zI{ci(+H;mOCN&fC;>|J8v>wv;ipcj`4V78%crpscsr6-U;t^J4 zB)kxHhnHqD-+%`;=mlA@?{0p06pXm4% zhm{q*3rp}(l(dT{a6(*x%Vx&m*wtulR)_07pCvd_7~*Ml?OEz>W4;fD&9x32Gi7L+V~$fv;4rlJ zzd_ebmu8s94PBxFJvROx@wGZs)n-SNZxigoxQzPCVa<+HR{EfClr8Zun6$FK;vkju zz*GwK1c#Fn#Nr-Y(Gk5#CCgaCrdR(#u6(_BP|+I~70%^Xd%YmZwte;YHkx3X9Jc3$Hk~JZjC%uqDHnCIhg8bOrz=a5F|G9`sF2w@=&-4$=(U!^MD)a)(CH# zPVzcioLRC%y7(X6K-Hx#Esq=5}<-tPnln!dqTM-Z4Su=t@`xWxE# zwx`K83#Ax(*~Jin(gF=u%WEr8G{YQq&d62A4WpQe&PyeJ6klK>MyLb-7q1B@eMhMC zhMCnRofd=F=#NCk2?m{jkX#Mp8k6C|xAYaw0^e;5zpLSQJ@}=G(dCYz*z=-PFimrO zd3;7LtOwerR}uEF{ZUH-5P}=7cd5XViV4-8YcJRvCheTLf^a1eYC1G}0m7S0fdK1p zmW8Gmz82Mw!GmTjiG^B%t`?T9V>rT6YpYFM851>A+RhTK}Y@9 zRpE*jqZO%V5`b{(2{8G11()V?d`2p?n1=Co#vv$SEF0tKY}=ilRO&b^>c}2bB@|4t zBK4LLMTbXjecmu>Jil$&A{%=sX!YIUHh@zfrc4N$c^<*uwWHRP6r(6}$Ic$b1kMqN zrhm70L4~b_TOm(aQ>=w#YpuUx zs+}%Cce_eobf713KcA0l5ZSR;2l};?_@zcLB?3ihHUek8>_2{)kjhkBw>L&6B_JbWjOmslD-W zLm18Ibe3FssvN5u4H(F<`EZi0-g?CkTqL0NE76NQn&VdSzN|Tfh-# zLR7>_7u8Lvr64#mdizP`Sj#i=nx=G~&rzDs$Al|HwyJ2gY+8|nQA#d6Ls%uc7N%B; zC%l3Vv0Op~7dAmu3Xd_0I2JQCQm~U&q{3;`K%Nye)*f{=tq#;6?Xoik!E2XHQDBlKaz9&8iFMh|ethYx=K;m6(ox*BH>qO(_nAAWjqaK`@Y zZ->cG-}a&_aWZf<6me%MA&DTS=8M<={a-u(z4yn1C&%OeJe*wRul_UcUCvqmXfWT2 zdJsS69Vl(%QcP;j=9S^&bz*{k5t>bS*9l?b13#QO#1GZ4M+<>s5UL4>46|->Av_f7 zPymOq@|~g!)NDFWQ#8k!kC}@8Xau4uHU7*cyPi=WxbvRP`Jm_;C?k?2jB4UUR3LJ; zI8CX>6Nry!8FPggr~0frWqf+dnG&A^MFM+g^dpCG+|Sko~lmz@kCU~9!^<$ zHD|e42)XKU_q7f60NtCSwHooFxTJn-R3Kn}Js*?UOG)8{0JUcoMAe3Ha4%e-YL>dM zFE6u%ac4^d$Mz|JRg80CR9aZs9ZR*Prlv>QM)EDuB+uXnoE|zT(F-iG)hFA4lkOFA zRv&Pu2~JZs%|;R4X@(V-LsA$1)-Lp)~3;^t0JUUsr7(Wt&WjcQZAYXd%&C9)~buj zM09ZkC!0+YGY=-yTs!9_m&z#gTT(1rgpIDQ*w{_SOQH>L4Zu@Z z$^ZNpqi-E{tfPJd+E!4;B37q8Dk||9qC*(&ySRD_gHP2Qb;zQXR}s#mck~^^{t$lY z*rWIjQ{xDiExkZjfkU8%WEsi1m?7gm;l;Zv8ZOVsW|8722oa)lZ+KR$5K*x2TCwgS z#-($+2&w0V87ck*Pe|=9a%jCpKwJ=B)JF|?Vf4pAwGkTbi&p5ccw#TbhdSIW&0aln z{s7LtAi#m+3nUz%>6s2aX|CiU%rS7wO7Jk9q^a$C5%<=5qX3Qx%2JcCIZdv7ZZ9E{1WZ+_H->0+ zgZ13htnOiKOvr*Lho0VNx`0;#Ju<0rT*UY+*=ivV0NddrWB1+a(sbJ2^xy}1BfOPf z7?{6pVTe{11S6;svdT{N4w&}?5E2c^3Eq0#5QeiS_^OrtF^-t|LSicQRgR@{>h0eJ zOI3eI8^slkOvLdS&Hh3-om>TNyV*DQO2C~5Dna!+$VB%3y^`WmIYFI&5`IinaUOM0 z@2xgkbwLIo7^=C#vynQ7UPLp^WY`rJ`B(=UH*xC0p3Zp986#E*nSN2}HCyQ7fd2uV zwy)I-h*UeD*k=RBFx=cD_Sv8ScMl%1`-6giY3<&qZx*yJxV6JYLdP8QtFF&>f}6z< zyezsn$!Kb&5`Cf~oVqCAg+hy_rTM(kRgFWdjKv<=bjs}y(A93364JY#@UK*$oHx8K z>Q3FC+Pe&~$>d*)&yoAH4;A&lJw;8;7jx0m3F0d0K*Bb**Qt#!tz#>>cpY%SUtaY9 zM6XyJwdHCGNjya#KXxqdRNUWG-`)(|agv3tc4h zOkTa9MXH=KBKiX@4&|nq)41F9Oz8p5B$+RI7c5{S0`?##Beb{s%zLLLC$IPlPS8b~ z)q4Nu0+J)j7R2+nFCdGO5wpN-%IB?}!axIB$@Ny(3H1b?OSG0IEBus=FRJLF*K|4q zd9bicHH4b*9%oEKC?_hAAkygte08HIn0ERGtQE&Sjq1qHPv)Eyg5(p0kCKQMrYx20E|2Fg=Tbd;Vncd{9M1Ce~EU6JGL&|V4@^uZ=8a^9zUk#t_`Xdx28?Vmp zKokW0&4T}mWat7L;Y{xzQ1K9vu{5?jop4^nItv9U9JF`2B%U3Z8+|Ink%KDy~(YCS938Hn|;6 zAi_l+%@7t^T>ytDor6$9Ny%xN3TKY)rG2hO#)_n69$G;nHQE0oC;SRB4`;N$sR&0G zhM<_{P;(ayzdSjgYZ$@{G7g!5a{R%U#^z7JF(!534}A`39UUjgBs5ng7><3LXLKm7 z<_9*Go+RTOr~$h$V-td<$!#|#Ch!40H=t~3;W$FghTzMR7n&=mwJ9JV=nEhdwLA&% z#vT)nY4y+*ZdN=qQtiCs)jKdDTG#~rgK&lzM+u13Go0!-1{d{G+tbHM6mxueO#eZ& zi@S4ZHyh~(Jskr1n@L;UsVVXqYm4w^2JM!(2DV#JrxsM-!)@E6fQhm5ZO_)lc(+AQ zLl*x!;ciRbyf)bFPH*>FZLr&uB0NVJJqscLL39sMx!dkML`ZdvkcJ>3bx}grVM0ES zFd=1=`BMB=&0A>KSe<;(YH0;Y<$dvjUWY2LpbB2DB+f!-BbKHll9iE>oHY&>>P(Y5 z04Mm=7Li{4;Hos8zO-Ek7CTT-+i0l9Sjc;;ua*ik*eVN~O{1~1!SKB5i^eyb6*8}1 zV=*7Uy%jL__|C-$$FWweUir)lvWYW<)(UvxK&&6QyG7K7l9jiEuNL*X=49=5FUYv3 ze{?ySGfBD^oI-dxow*rJ5}; z7>VHmwaRP)72>E%uwlx)^oP0-Ucm`=s@b0n=A#RiK9qXq@47e76M`4!;-w#s(_Q0x zE!pkP365q9zF~g&OMkGRV{pXlN09fXqitYyugXGL_2%gD9*5&%$Ga*f$cV9)gQzDs z(TZgg1s5;&wtT0|$;5==Hlwyd9ZZ0Go9WqHqv)2GjZovYY0#>^?OiEOW5h}54r+pg z%m_-!gG&I_Ul38SA z@A|BUj=9kAW@zv-)YB9O#wi6ullLEuXrdV2dapU2CvJP2;~g85TMIk$gLoQ$GILvR zZeYB~=Zp(9#sVmj7wSOG@r)#M&wJd3#?^=dZR12R?=r5K?zeeD7K6_RDpqeGZ)oyP z5b+a9=Gsf3Or}G?3*q!Vy-!R<#LfvX;#VF{(EE?uF3@|Dl%du2>g$NAe{D<)?~guP zoC{sCDp9R^BukMlG(*2B(v_es@86#s{W2DR?x>fsxC!kRN3fjExhglNaRsF2KKte&evUz0 z+^`$fw#{VHR8wn_JNttl>Gz$|g^c2B?f9*rt~CO9MsSQ0HZ`}I$0^RTFpaTcpWp8Z&TP%@?a02ln*hOj~0r5zgRGST^7mQgUT0JL9S^*;!n_XlZ(F*&l zIXm~-J4{s%0LvQNAQU1sjdQq}vNt=qr00 zRR)A<^GX$_IwF)-oat?%I|lC75lJP-2})QPzZV^=5opUDZ6RRHCLus_D($xW{q)qh zd7Pg9ZmH#*NZbV#RVb&=bkr=*StQ89d1d6f7NMZbe`<8KSgu)#=!5 zRu+>!Wg9UaG*3;Dr{&V0{XmzKJh4aYCAE&pIwcFyX=sA|R-NSJt4%!Vu?|bqPU_P$ zDcw|*Zz=5ivuC1PcZ0Fc|M7L^|5zvfSf2PJt8n#1H@Yfbr~N3i($`7*0ZGB&kUrIC zgUCqKXjguJDZW*&pi@q9%ZqEY;Fi-5kEY8E`&lk#wyT8^xpxr8A0_empB zRi!`)uH}OM3qw#Ag4_w@hm9(UydiI7%tXqgtPJ~!wEKw>c}@PTe|Fy5AtT)~>DBJw zx>>KD?k`k)x^J^y*-U)qdxg_jphmW-6yah4ri*X^hH?`5$NneJiU`KBUh+~MSoz5H z_Af=MM8EbXYEGiKkVUao8pZZ`6zdWx)@D+yOQk5Pt}^u`$ayvqW|*b1yFfMtpgD?R-9r@aY&{-VyZReQy95d`# z2zMAUCD^(5pHJuZ5@C43)=CC{*9f>)7jn&BVc%bh9G~=i`Ed%rfXZ+XIRtI>eBU*e z>9Wyc)QD&yl3V3I_rtk`0o}?W-GZQQ)nVO&z-~>U-EJP-tzL!RD(qW}>R~{*Qb@R_ zls-OntOyHN9T=`UG+c9VxH=MCQ@JiiTy2!NhB$Gxk>V<2#Tg~u*Ou|8A`?!HwQ^$` zoC7|lQqij)jt_vRInhB|k6#xWZvxgNpxq3`6nDMj0?gTPXdo^di*Ib3KoeS68Q3#c zy)*WLR@WFR-3@Jl^fB7L5lMDNQVmUy6xWZU%*ERr7(cO;aihbcVQOKc@gsmI`cU=s ziZfO5RBNZeX*7TY5#?DO>D*;|NdaoCt~M znL~QX3av6pA>n~;&^o7>;xyGB2ioP;co0@}qG(~!s|KLQZ>-T3^w%~j+;QbuWodgy zHbIQjDb<27+?9aksZKwy-8Z!hkTA$d6a&e{8 z@>CXF<6jv+GYS9bq^7_AF6g1x_>3TtbArZ9&Y&M}3iG)%c5k#LCy~W!WtcjYe?c-% zAgsZ;j`8%ko2#ziLwPdzJy+AcgZ${1Xx;Q!76UFVt?b3$b*4Ml+&XP5k z$tiA|ZeE!N2Vl`Y}20n=+EYowObi+|}^8^QZT-ym&{Ps-ROr z7Q6=FLUCP4QyZYWI6ZymB=qm!p`G6FNpGON^5=@Fh>65B5!*-*vJlUJMFKr)aIZ2N z8MuV(CT!Z*VLSftmbQ-CAA>^t6`fP*ELHV^qg7$*;P*1G!SND1Z**957XT=VNT$fI zo*3_3POp&Hj_rb4vW)O@s9MJRu#Mej@1a8(uyi)I!uU!%c^Zd5O_d?Wm^UcNz^Na) zgOkKI?u|*0rEmNu@>E9!vtidv0)8LK=_F_YF#f`Z67!BQb#`IJ`xsK$3l1L-MBUp+9tGH2_L*&iS4CJv8`w$ z$)qPx8;by4i3)J5+*X@elRrp|2vg0{F1TEV;32YzVld(OH5OEgNNPAKGYbayTY7mz zeDDV1@WF8-gm(>VSFtNC>shrKjhTu@a_AqB8*m&2V!Z%H^*Q6jcNEK*kcN}%7BMEa zO@t9iZB$xsvTam00atBmY6j*u)6PUshgLkt2?du1kKmQb=T2|uX-{j%QgUJQhT4gP z6Ss07RZUw$(y5$zi&LNz@t`@KS``g(R=k?V7-`qL=F>c})nX|O zAYALEfB>0E1(wvX#SkWwrOxln&*a(L*xcNJz7|Mg4J>~Ki2?*0v*h30_G3+2GtN|7 zo@r~}s(MBL&Ce$Cme|NqA@8j9D4iy{k-*;`G@cO&G?#GW%kWSnM zofd|B5tcF`Jzdir{BKP?SFOml*6&;k07>J8zc=2$??2wCAOka!n1716(fgb5cf{t3 zN$c)Cefsq2v)ylp8}9_%alU_V15=?+gp;J35+O<4L61N&+C!b=ci%t%>EP|j>6?Ss zFFUAn^78f3tAmr5oh|h7;|5&2EC*sT=XBg(y-A)ViWq7zkNakjc0~d(=G%jqNi_Nn z(KIHPNb%VAgUdbVeI23?NMs3>kcptvE5JB1?UmkfUyUF=`VQJNi)O!aU}1~)N|h`; z`@4s{wX>*XW~%7MqnJh#b>LL$*k6k5s|mzH7VT|qp^olQ$FhA;=tp5Y98q3F`dYus7W64mv1UkKTXeV`2iZ^nRq${0VR> zjD?vP>B-UEN(6zDjfJXcNnq>uvrFQwRvtZP^EvLu1O!-6hUu#k$HN#6P)b~w`wJb! z8Mq3^OdRR6Hnb|l(HV%JEG4NFXn_FKa!}?&as5Gsw9{>@M{7uJxuytHUTQY(1IknS z2l{~i&S;7{+nw@0fvGvC^O3&Nq6v{#8ToFm4%TQ!Ms_Iu$LQr{rg*2aYz14{lce*> zgkDUtG97Dv6=@B#0dU20h`vh z$w1kvZfqCx%CRmk;1j%%2{=FyRBeWQ0oOQ zFffJD&K^$v6NG+RjlNoCU&L!ydbtLt%#A3t^~AY_=!E}M*2&~6#jg9U0^uUJi}YYt|p{h&;a|Hzqy z@y-aR(h9K;7G}^ZS>AA9La&YEG}|cI-|#%CEU=7p7-aW(yete;?PoVM)|l6##$j&4 zH+AHswnoozB&rTVNX?o(Lm%A&jooU|gVdmEaA=nL(lLK@R)tr4NLDOS5Zm*{ixg1$ zmdn&o=Pnj1px|4rdU3tQY7MbgvjvxzI^LkipqmuQAY~fcl9*@%PzV{}Y2u5p7(!?J zo{u5x6Hpnk0-E@6Yv{fQyr!O_?$S~f^V+SFNWIdvbN0?$;n}SDp7pA2^e$(r zYqfgIW^AEc5x}PMe%ry&@8DX%!V4_bBZ9!O@{9D|$a+fjTLX7mT0LblN+6rn!(|XA3nkI_XO_xzcA>-;J7!!#dtfh?xr7MxtR3?{_q|kI_ zt@;xUt|#qk3#mtQb&a*A;us8NgLeODvnEy8qK=ySRaiO>t#=kh?zlpg=3ZT3a178o zp4!P*)XY;8L87|VC&zZ^$TaRx665I6Y7uQzZh4nr!DXBI>Olx^+=?PBi5ITd%%_xG_@8#YPxgHDKJOtP70zfH9}3i= zDOl2ViYorJFWk8VJzfxMv;vkfg!MQV4sm>ZYuOPbU#KL#`V_gihAcfgWc<4<=dT&4aPT? z0xN^@0jzLFlwf;`D7YgoT|t$L2UxAhzr4!G&l8~MbDaI6=j=B~7C8a)qK}|S>LO2F zVYJ66L?S4q=jQ!;txzd2r&^lbE+@7_cEL*#0jw*dt z*aNMYx(5)Zpz?}86$_-s%pO!KO?PEG_L9&{U;BC$m-ua(h07?z+?+6b3ku6K;cPx? zf>d*5=>o9AqIQeb@G=?Ld0Y|V2RknxP^zXmmFP)nn{ZA|VFJFfd{ODcb`ErY}W%ge?gF(suYj1zBx3>SfkEclsXQRM@F1<*GQ=mvs zr=ta9tMlQ8Vn#;6-O+}{aJ2&*lDANRSKbvF((UC3x|WUU1`1*&LM+5p)%k(`aKpFv zT6+DVPANlA_9~R|!N|Qt@4DlD=JyIkeQo8%CJlMf;hY-#;QfHO8*p@^vOuyzU5M7P zN_jEEy1g3(B5Bi7Z5Wca!}|E~e~TI3*?+QXWj)4DSvB+d&(bFwub%EN1S>ivj`Y5J z-3fVKgxp=|wT82($zBOVA#dZg5T=%O_d{~GhU{Y1>Qli_c-tC=SIZ1;;lk@$r(V^) zS#RlqQSO}|f*E=jW*$@h%IeOKCRjbdDSzwM9s*WPXW5CyLNg|;Fc|jK#LhGweq;on z<)8*ztL+^x(DcF(Qzp?PP;|~Sc~xUone$|;xrDJCK8&}vYSyF{GWz(jfA&oD4G7V% zE$kXmRd7c8=fi#>tM;$OIrR-riaxAHI~bt}PQWesT9MOKH_8%|d+6u5{w z-=6;3I?lqy!7}0SSAqp~N}#;79V;9yrcMBBr6m)-YRo3{TY2Ful-f00sZZL9T`TQ$ z$NV=d%ugB6%{l}b1ikJ+Ho%pq%7b+~v*o{dZU*cttm?n7-uZ^MioxiOU(nEo4xsm? zoxNCgL1`AO|Mm53t5dfNGS;2EAmi1iPSw7v1-G&PYNpV9zK!L!E#1yY3gdi30$o_< zReM@Mx}}~49NI=HH%RAZoZdD1Q;NIk$aNTu^BQD3fOhGsX1}d!|Gu_;^U|6aR>I43vhw`-(3Z^|iT&|R$(z~Glw8|F>RE>pl2Z zE}DxgF=0DMjc4#qq}=PTyw2j&yz)AQ&o!3*g3#$q zLQHcn%yaA(mGHU0Fn5V54Y;xlTW%j$J_V*&meBJut`%SdPvo1)u)fRmHE5q-{dSI1 zi)6DkKd*kA)wk4kB#nj_iT`keS2$ly82*(BfmUZ9#1NBD zkOQ4u(@=%#kE>{->cL-kRUkZ}&cF3sLx0U;v-7IIt zS9x#usqJ6>?9b}>yO))b_-z z@XHqCSZ)sDIe8QpQKRp4Sh;Img}KQWt!7?$-3>b7dWJK4XuLEP#2fo}7LZM$a8sLq=v2ej6^$~R+U$&nqAF<82^=OCL}=4yXPxhS8u{oQ~|j^exl9r5-;#ftD%?* z4ce6(uq8GDvUOJS_sCMiShqLy|0RW7`E9+t-3%vKu+T;}PjuelHl6frF+DC!yMI8Szgrr=zk=BHd8rNcVWRgS#AdZZ6;$lAwOQDL+ zy=m7KuaVkTOEXyE4!r0bTwvE4fBxL~Q%UdYaAz6SlZm~(OGG=<7{BI%HB7ye`ln|7 z;aOe&%UJw^3Z7?*Rro%Sr%UyK%k+Owhh_cWlfmATwfuJ<&yD22DpaU(8g0dLm7|_` zZ1>S(Q`dGbZ7~1;`rJ!P>}uxgbYxweAic6gf-mTcZ z3V1;WMPTSt;n-ho%~Mi_nxfvTn|njwSWkHk`Y*CBX9;Pq09;1@_jeZbe@}+%`=5Jx zR;B;OiinSKWW#dS*!zpjKMOP!f6EdQ918#k=W@n4{fEwn=0bL@>3bXcZY28mHV&0e zV(LYsS{`H3%dTP9asiSHwpgg7%2CD8us7WA4Z4F)RZk$y3(n||oaLDqpAC}0nN z3*7ojWr~1+wp`)uYaJ2*RBY|#0V*Y`JUBUz#R%x`FAsA;;H9mp^Xfqmmz(3&mwI7z zefmG9X-v~;i(_Ev{r}F>68+yF?mb=8|ND6Ei~hGJ`o%%a!Ar(sH!#E@Z%V41Ujf&i-};N-uSBJOQhj9d9FqOSGfECW;y-e z+1)Mk|2t3C`Csqn`4sd&4F6gvc#aJVr^5x_5#7Jr`%8a%GLapucrn{h{cbNaGfvNG zLZ&3Pxy2DOY12`}II&5uW5mH`-Ls{fyZ~ zdVzU-aCBHya(TD^0xdpIFrBNfGbZF49bU5NvU+)V^gMiVboiop!SnP$yc5Leav;C{ zoJyf{@3>^=lBtq9RxHHebQ2^H7U+T`iOx!{#IF&0_*Q{ENUt7(w&)d2^Gn+Rq_xv1 zkFXG6_YbKzSWI-1Hk~&3Aj=N;oN>f4M45$j{Kqc=R()G}X02rsBQ^O&U*|$XaXp`K z8|IW&c71Ui+qA_n)Tx5gbAESonvPK6ayQ49BlKi{e|LWap^4C(#(lQQ366hGldHFk z$scG!bjU2b!hgmy{3g%p=l@Ek-|#Fu|M#Enm-TMB@-!Gmp}3> z`~nMeU2J8Ozb#LN^#~$*C5)Zr8S0!5JE(Kq9;z=tZ28^RZLDm(3ZF>x@vwN!KjYHvGu5DaSEcEZo%|*qssT>~_M>R;AX}=Uium&IrktqGU6I2#Q02>Jfvheu0SFs!-aOxOUWhexG|wueXUXTk?^qjgCb zj=#qVPNQNX!gtGOV-dy;I&lmv<6F=VEOy8LTC(PiXv}Mm>vk=fOJ5DDA*!?)Q(EIo zY8z8L8tOc^xY7xi`-J1kghpDexqOl7uM_k2A?4mWR(J$h8BmV#fRIF1gc=&Ld3f}jCZ&5MMaXh zHvKmRZ-M^r?eCQHzwHlq25b6%AJ1*j|0*@dYB}SM13-Toh0?8ZAWC{#E$g)D>&bLH27e8X7klmOv7LBc-27eRJ}&4IN#l&^$gAM=+bB0K zk(iqumP&g964svE)KCc7->r3rWdIeP(k;6JNpxHuC=i0-ck8a$3ODYFh-RdiP5rCA zGBv1t{Z5Pt=7gWHGm`#53m59r>(0eAiW$wUqE5eat$;NBIgy-3Lf;naA8!c|G@4G$USDBK79bRZSbhb$6qX~ePqqr9 z;)V6AU@$8(ZaFHZ@5K#4M!%e%VjWrZ^z=7p-lX+YP-bLx)2B3c-#N(=9D$WrLxPos z=2!1}=zo8M&MQiMz3!LEYjyvpRQ&Xs;3`Mr<_c&Bq!{J_VtAK ztM#GlpmZS@KeF=~kYVF&>N{xyKi%nsDWc%;u(=|EZLIdY>|8&OWbxfkzF531-x_s_ z2E~1~*wVP-hBFad|FpbAyZLNfuhj-+9mf-#;@Wv}J2ApZ(oKnwB<`R`I#?p=nCcz( zSJB_CQi4ufV9Au+Lxb;237N-cTRfa-2yV}ckj@w0oT5|t4 z*m+Wp|FpONbZ>qCcOMS~Trf!R!Y(tU4~l6dQRpnKcue(4=2=2eXF?Mq`lfRKg7LG2 z;aKz^cgo)~reAkMcn+_&(SuWn_3)qvZq~gYRZGyGqs2w(*EkCc6X|_LACTfw|Bqm4 zQ5T>KRc>*tkETI=1-+BgxU*IHMN*ll6QWyd0#IrE{@lh#x-F5WyeB_%@ur@HR7*H^ zhIHxP--VjZ4_z&FIJZ0|tC;P86@WMu#m9GVUv);P)7dT}4S46c2c-BD44bS|o@YN- z1=koNtp-oceW^qKeY^*xoN%q7NRWZ2s;j^~_xsj#O0D1Ja)1ME8&xgkl*SkugOx@I zkyjW;k04H1jZsj}lda|w&gyAvYstFQwtzl`i)np;CiZLMSwqW^tdH;nC zHEzCYP>VaymNWmup7Znj-S4JbbR@3ZJe`C^IO&+2JQcIEY`5Oo*1L1T-tD|%&Pw}l zwVC*w=+-BhysD@{jr;9py}UUYtJC-_rD63778tMpxtYmnw&1R2bSj{iSe^Dzm#{kl zpgL+>S*T2MD>~gscN>5%65mzq;O~?5sa~lshKSscwz-J`YQ~xyb?@fXqfv!kNkXc4 zm^$9YqzWly;QsrTf4KG&uHR@>j^3|LD4 zcb}B#|I@+KHT}Pr=j*2b?oMib2&@l*(jDdcB>1K+{WlBLcXxJeIe+~uq5nI>VVVB# z5BAse|6U$LvlvH@ZsQEtS|~YTJR&bhmar>NH(>AOVS7R>SAWxb6nxkl?)4wT{cp#! z{xTJpo;a_lkQ-Ezxk-^~3=c9aXWJUcbCybyO6WuZH8eP|sVF#^lFPR`oWVhopw6TJ z+k4#KdenQo_3N+D#|Iq^9CaSN-}%^aHeh}iZSK9|ScKRZ?qKxH0zAX7mN9S&?bCy; z@(4a4$@4TqPj*evG8^8bTf}p{`Jo)1iPt|Bzvy#u;bi<&XQPSpQs9=F)DSH`el+G4 zFyEd@4IuIdBpK-MYVBSAROjW__FV7$7hQ!=#)@?*2e2&u-_GLszqh}R|93CXHP8Qo z92RImTCo7G{Qa5AnSOXUyC8n97F@2u483Y%R*i$&7FNsDN6p7*0lGLxJHyWBAJ*b6nVWks44xx9^0}*kYkwY8M&&y!Y zv@DL*wy%yuFdvEx|4S|ML|xU+?m%=_MC*l)m2L}dVuGW>XT*dIgZT1U`E^d?Fg{pA z+c}LFERR~%^Q{{4QtZRl6&LR=0o3PjDc3h~{fuRTAJ?y~8D}kf8%0Ej5wb3Q>Y>6o z9gdXbVI#sRX7h#PM*$(|+##jOgcl;}YKTcLVfy$9#}PSeFBSG!!ek*7f%B zyA=?$0eCNf==TUMrt-LM8&bc6xO~>8x}JTnnAjq1g<7lQf_^R`l-2bL;=V{M*Yd+B zd{*NB%X*o>8I7GQT0<@t#eru2fB)(3ekuR!?qIO9_W!w;M{U0TW|tDw%_OYqBo^-mXC8mc%=1c5ZBu~taMgb#`|aan0TjHe~xL|pW~EHh>#A& z3dQmZGNCEe#7tq-#tR09bI}Y`cr@*T8t(sx4pYQM%rmL%+g zrc;#A^lY?YhoOg1?svrI^E{>Us$V$6i@xN9=$@MwrBUg#xj)0v*~a6(^AUQWik(29 zRKv7_gOz35p5qLUX+k9>qW2&9n3zJnAE_&E$?O88pU(;BG$tM?SAyIl##;+>l`7gT z^hl8TIpNRQJj0xz&cA=BHhpi<9ds&2v{?S%zU_V6`?mXSb?H@Z74IoX{e4|n-dwTU zsobBPc1}-en&lGAEjQfj-?se{ocxhfP8!O12+|pO z;NfFJg$%1gb@%I|2Z+Q}s{PzCIsLs))0D`=xHCd&o+SQzjwq zgY^x&nmD0YJZA~xdaY_peq3zU0U2XnF{2B`)FygG(y3H4HXJM|N!o$5LXcX72j0)7 zN<>7QW(q-7ttjVQ?HfTPViUb%1QL>Q5+kZp+8=n^s0s9hCP|2Pl|_E!IHn|38=+|$ zrkut{>;?-1IK9(QAR$wd#tXZ2p&U-=G(8X})w{8}%K5ZqNwWYgOV9CT+mbYGS(4M~ zOdiv??byVOU9>GqWAaxT1I~xkrLwqW5+^M|YmaDK_*w{(&=ep@o;PhXRqVnHq6xPv z=wB8smJuG2RN`rih3h~F9JTRs)y4f~CEJi6PSTig!c|`< z36o8I6mF2#b5Ae<6mnfV*cCNP}s=&i`G+A2?)iYRttIi?6nvW Xxe5;bjpbO zgUde6)CoY5%M6x59({rln`FqELfA>GXb{@E2zjn5bSOD+M-@kS~ZlUoNdgRL4 zQrk2Y5~mT_YFKO6sR`qASo4mgb8@PewoOHw>*og?NyeLrxT~az`46tm_wRpy|Ni%n zA8nW6$M4`Cv=kO66JW182c5>kAFqX)a;Equmc%zeZ#APoa5i7Qqg2V`8J)D)d^)Cw z8a7}}(|(qY$y{=FiW%m>y)|zgU6%1BOaXF~|5RRO&{vwq^qj^yPU_dH=@77RfOl>h z@+H2ci!A*@v2h-q>0RC8z?&OVuNG+uVs=1CY6aCC&@E=%9ik0rhyn?v{4`DN6}}|5 zLf#5WlCpH!L_rOZ3|q~XF3zUkV-Q3d4tQ5uGhr=3Yd)-HVQ=M{2~RVu^^_KcTTf{{ z9j^RRvjl5tGprO!n;c7`#7vRHHR&WzK$}U^bDYrFN}rrWgr1XFF(F^N{JFc9KiBeS z4T!b;SzB@~e}+ZAAo)|#_$ndv)ROg6*k&usq`y2kJyozzt$6s`)>ka1qPJ$YZdG2r zG27T`q0Mo=UgtLJe5faDrDgv9&#acP>>*M}ver=q1=l)?s^^|{6fA2+R=#$5`W96b z>lsDY(NHYBtsvdYIAinuP!f)f6yI3zExV|&ObEt$V(5AG-fy9W)JKG*JcS{qM|5el1^x3?63q$<=3Dq+FR?2*1Dn^5NlmgZOJ>+6@lb8HRs;v zrZoDZH9`UXzMh_78+ohq-LIVo-p?j2f>&(&>(nTm0A>1u)J7pTOfXHZt0Y__MIBQ7 z0JFN*GBp%6Vuz12BlITVG>=Idqql3lQ@P+;?^N}y^-fC)uk}t}#Eh=0cdBlCt?&pd zg$j=ZOOi%ch)upy0DDQ-1@)+&>GRHQB7hREmlg&2)k(f>8QM(-n*r}u`~ zjs5?EG5X+~aK)vpg3`~|TEI{Xh+gE&xu&%au&&rz2k4;Rfevu3uex0g;Cep)e0FES zmCvo4pKVl3N=HO9GRGzYmqLs^|D(Lp9g8tP^W*GDJY6IoVN9$!y`*WUvP~10iKf$r zI7Q`wRc{_och;cgmE;okR1pZb={Nz>TPnVax)~xK)t{Db({^59iP3SBzSbTsou!)H z4$Exhrxv-&vEA;B6+ z&uTc*3Nu#2hgG%o%Mi$fCp$=%C`t=VD+1xwl$_^@q)VV#TwDt5=BNlAKh-=?m7~vg z5eu-+G{QsH8cAk3vKqFx4%sE1>bQ0PY5QG^^_E_)G##qea?oZ&>A9;ZlnZjAyHKqv zU+=iv-f>2Jzm5I21k`=(Hq(XmUi;Gb8g9N%a@*~}>8UDuFPp9oWr9stFEY0ff>zpY z%WtRdbjwv4sQS9Z|M|7kd;DwXW1wMNZ_L}+n8BOs(5RpeS`b<$S{c|$*+q9)R#kN* zGv7M3ATt`B!7XYDuvtE*Ca%`mboMKS#o?gx^=|d+;k@eYe&yR?_3Pb1b?GYQU1z8A z?Jy2$&Tr+pq;qoWZ@-)MxVmdks=HR*ajPHe=vZT>+}+>pBv?Wh>~1L!n<%x z)Te=K%>OWavb(o8EaiWDvOC;a=YP15rzZbHu(IpC59_=SpCs?Y7n+d6anL$Ch{WM?$Zk;%!iBhv@Z6x%D5;S5d*w4eA z({zj<4YtwFaJY?j_V>5ZuvRj);`V{JgZ>#c#{cy$4-5uY+#R%Fq`mwW;mF#ixA4@2 zcaYwqCVa!)#f$LG?8_hZ7#lF}qBBA}HSq&Fmkm1AD<>`ZPla-+Hl2b$QwtxztefBG zCK<-}IE}TT`EB3+ZZAWorLHhlOJiJLYKFJe<7KI*%SuW`>wQv%D$Bi2+HHex8@9o) zPRF)v8w}fTgNmrG%bZxs&MOVne5uchr)^$xY`j!V#PVYQO*m^Vwbb8bxt~km-_p?3 zQuC~q-YiyU>Sn8<*b5?}<~Ccn#1!UYl{Ph1*knCl>NegImzO42mnzabr|GedHd=L9 zwDx@IEOq5*qr3X$c7niE(T<+3hgS%Ev_YyakcJ3`p-C7#%qntH4 z(T2pN4?;E}9TUyh$v!ZQfJ!?3ZMu)^Drn0NMyno%(u}GuV zVo>SRCl!NgKdvYTRdF?IIcO~ht>qwBVl4-uuS^b-9H(MV%kOmeo~URcmm5((FH7Se!$_TUu7zxHfFqY&;%Z2VCL1 z>BRS|bfPAj$BjGje3eer#9z2^Ctj@5i5GX&iI=N%;^iH6;)hi_@xvW;;>T4w@#7tJ z;-^(Q@zZU0Vwp!{*o&g{4LHi{D0HgC&k0MVH3|53@K05>Z>-R{ws}Vd&t0p~sggaL zqw{?ZqB+6>{ey5;E%YuIj77v*gMSm8@jnq91C+WlnnAB~kY`E5D#cps(%Mt)<^uOs zay`x!^s;R@iijnQn>AX(V?|cYd(A*VF_xr|VgL5Gzx}QI`gQlk3-r@ZqxpO!8d=Ji zM0BoQdKyA40TQ`wX(>9ZTE8&%lOgN)) zOd9*2r&PyEnW-&sONW9(v;~{2HkmQk=y#Xs*M_0LO%zP?`M5dYu|+{8Ki)k8HtTyF zgf$M{Jq|X?b6*b*n!tT)BKX-MV52PmH6!4f2!1v=*r>|+dT`KQx>(YOj zvHL^6MkU^-LBH${(a$4%YYc2Ec7GVysKNdEFtAJj(F@M9Y<&(_7rQ?+Y}Da@eP~!g z2-ld{RP6pRvGF43>%qh~ZElRdPJCOQD(Ob(7WbcxpW7BFl3%I7?^bt&p9aC3)ZYtlD^1nDZbvS#_iiYvvsESGt*Vx6jv_5!;%8whjVUfVjhC z*vlBQuP_;Qs|~h#I_#Cs`ev$z(6Rsv_W5iQXE#S4a1}ncRL@If4TL^X-t6^suAVk~ zWd+_^KTucXv(^tREp$@_!Lzl3V67lnD+tyK0#kUcAZRH1XHyW|qaBOu;aWowJbx|? zK}$!`wUVH&*jh=jRuUAYKebf5RuX(tCBeemFDt{^LL7lgYm+<8u=d5L>#I-=)EcBz zC@sFCe0@bUil3;Ej9=Y9Hr*sqMSYryKUJoR>u-de_8Xyc8gDyQMMHs4m8qhs0GwtG z1@1df#oeZsXc4}$Y6skL_{xZ-DT!(#$FI{%pmqwKR_P_`0(X91*(KIl_Ix0JrsNXF zlwt{QyiO+Jt2tzSB-Hk)BpPPQ(`8f^SSN~T*p=%<5q^o!J89lp7P*-$va^CvKCBFr zzLrJSvdCH%S<51yku0*#bQb{mGYKR1)^;t8tfdiK@#|*4*V2gp+)CQIFWK#TcLlfa zDvRFF#P55#753V&yB-i)hDNSXTgUI+8`yOZcb9TFcW2n$qTuId*WISz61meA{9LTM zKaJt_@@S(?a;snNQ`lWM$+fQ5+W7i&HojhhfS-}=_2&|X&UJBZd>uSrtMT=9vZJhR zuIq|j%jUX4{C8znwzzIFRE>6 zLU|MWx)#d4(rcl7EtIc?^0iQ2UHX#=<@Xw0-1TuSl?Tt)E|ssCG;%GN*A-g}<_`K= zFkcJipZ|2ODwr?4{jywA>xujQxix>0<`%2iPgk0auFWmhnw_A`|Ht07cDIci?R@sH zKv{3s-VJrJ<(p2^p2XhUY15`jcHfiJ>``G7lCUNLE(qFj9rw4thgVS&2ttl6iPG>x zF4O?LU}k`4F!L~0tD`T8>?pqGP)I;zrz8!L9f<5eWCtQUYU#B}w^+s(tBPBN0MFF1 z=>eYEv*y+$*5;sbkYox*`n&fVxr;fV%1cpk5;kK)ptK)vt3=pgwoD zS?L(x3BkqVXJhOr-bOqj$May^qT4B7GwxudOKg!$mNn9+h z%6&>an)f!=Rdh)NBa(!+Pd`y+DX%DrA9z?&Hk$MBmPbRW^kS8~blxwqU)6Kl;7IZs zHnmt};P5B`%ZF9DB8gOL7c*~pN^AnVH8CYgJ|%wP$k)w;*V)s>Vv7W`7LNHOcwUnx zLz&}6E_vzI@zT{hFw2iTNo%^vreJHb$ga9^9>l-uC%?Lt6Y)|wY6j!;p*yTpfkLzx zzgYGmGrRenn|wIWMCXu^-aXoFFY>~>x>e)!eRs#2`&Rx^0T8U;BO-3IA$)%1Kd-!r zk`Jz;6BA@a;wjjw>LR?N!Oy~bcgvg_1IbqKRrzodj}p%q{wu%zs!JBWe4)1zx2o^D z01hc$t&)pE@;<$b<$;=f0W>t0>Q0FSyZJ6me zd-6f9*lH(e9VsEHdittZ<62$SB&D9Ztgia>VMi^kBuYI{p6$BRC-|>OcPg@5{*6S1H*~8B4Q&%d=e|fU~V%^=LR_b?KBY}sQ;g(B=IP&BL z4#$@q3pae9$)l+afJrdpZw0tVZS;?H|Aev;YNI!T|3;khu9p=h+vC=7czk$-I|oOX z&XC}fOXtvad&8s7>DgHaACj|jlj|Y`k?s=Q@7&$lh%gu3-#^>zLty5NiLit8McO`R zRB}PtsGS8%NZ8H>uI&~hW_MfNUVh%@*F;=Xa?`rSzBhl3PN&m3?)8$tJDtwV-$w`C z!;@#-Uaxm@d~kfw?L6xo9G@H=JVTvjynrZ_SUl@|@wH|jt|3RP= zWo;ibIwYYiw978E>w*j^qlxECG-~I9BgQ3i#+Z!=qRb9F?2td8pBUo8B?3vF|0u^a zL__KkG^8SwND%CzA@@9f6MGDgva9}1{ut>a%-*-deLrGU-nPfY^KEC0g$&zL5R%?` zc4FqK>|}SkJ;u(}&Nppj-=|JwHx_%J>@jD7PSk&aG7pGa=QzNZ)T5G;uob(xOsMq( zE#!0eIoDIZLyxj+(ILXjMSEP#xm?2VrAOSk^FuBiazO%*-%i}|KH3Y3e@(zIR%50igV|6Md3(oe{;t@PTMF)owsg0R06 zKM4PTeQ)lih14Yu7Oja-Zk1Z1&xkUYFo}9;ugRJ^>?hM(rYpM!T_(-a9^!TjwaJD- zFf4A(8zy4N>RM*kXPXKNuFlkRxu(_ zsK|ItlIkmJ&TIJ-7Wi6);u58(U{(Ay9_BoGI32%=yGa~fE{O%fRf4}op`^pxLNt=R zGH59%VKVoJT2VpV$RYo%d7ku{u;8QLfL2_NEmrCOA7vtU%vR? zM{D;te)-XQjwlNyW@(D>ZvLP6hfrckUVOCjUOrmd?|Y?9>uWV8+ZHt=JDcxo_;E4y zV9uR=`j3EpPX%S8>NRAsnu=faqJ%ZgU39==V)}r^#MBKnV=+!7&jgE! zbzMw^k-wKODB_taT}6b0A_ic%pt#q5z;s&{1(W#nrmP5@D3rE1D=@u2lZG;Ce;*#qQb#OE~5?KN9RFDL=+vdtV7+ zzKp2n4k#OP(^(B5@dC5otN8VzpV&~fX0(!G=Ct5Un*pmC&opf(-8OsAzfTu^nwM18 z-N$hWwKvDb2s8R$Eq6(sNfmh>R4k*TL_@;=S!t4|D0r1K2|L-ETb706t^ zsAxBDGUAg=E5jy}y?Ns%0XNH@MyaV?)6yMj8`Mmo7G*B^^nmfyFu4?>`CxWUYAuVQ ztRbpFv*V}$wR7^sr5S>(so+MlL$0BfE#!0eY*jB>6^Z{MSY~%9Fw7*bWeCe*i%bwrCM9)sjTWLtZJ#NR))q* z2ys@k!p-=&6b5O)!)3NUuJAnd>to&C;E~bOjj8Jr_AX^WsaA_*384DkeGS=GMwz9n zT$$8p=%msLdaXoK4W_M;spE_=;gqV2wW(gRo;+%bG)kprb2Luz36(w8BAR4N?0p^@ zSP0b^P(3XGeWuBvREkwc0%bz}WJczsa_#lxO*Mo}W<*Sn6z5DhkV1)N6b>Bjl6{mB zFXixXEM+hdq9NF(4ooHD`3#$|4cP&GXAv2B{BaRUh zPBxX-5%txO-z*;IP`}1gv)34oV{2*bYt(EU8u_eSy|w0>DB8;ZSmy8Kx_C?TMgtfN zFxIj$mUa)~@j;d%Y6T!`_IChTTL!YK?r#YwE2)yIkMkFJY)nJRM*{nU-y_T<^~heQ zb$0d~P55xlU|G7ty{pV~odRt4|K1%8;?4^af}9)IBm4X48`OOY#CCQj$LLL}wbavY z4b`Hwg_moOG&Dz@jU1Yz&f-J!YIQ@i=m><7|3$C=8`MSZg~vUOlzhQ;pJJq6UK0^d z873oqzKtQ0bTrZ5u_Vm7g>VUkOXhL@L04NzIKlZo#7e5)p>`(Qm{>{mYh2q<$&TWN z<|v5EY>NgI#L7M&lVo@u+1p$aI)VmTLhD432$7_TYD6BJ`PYOpZV|@E^`MnYM2mXw zCnTRUCtEb4she+GPn~MDzWE|5Q*(OX+-_r-a`#xLz(u%P(TBX&w`x{%T>)g9FUkHxrOjagjNF{luF5@PA)dq?Y~G6r0T70Zd; z!2o*|U*}nsW6C2F+v}Z*eA=~kQFY^lG=3^?3(o2(oz>KthcXV&(bX=2ad;!Rg)3jq zW3ue#L~sAhxe7}{8L95^CFz8xN3XBc?70WY$t_~fZT|5^{J&SHp)E){V~2K8VKGYs ztmXeb>>VC zGGK$nGp#>n@aGby;I$fT;HYc=?MWchg%Nv;m}JRF3&{HHOYqjCBnw@xScq|6ml z)5ca`w`f_LWSbv4!V`VcZ6Q{hd^;tnOSUWtE*6DJ8dc}3PDCajpiPr-;^|bI{^8d2 zSk4boLbXD&CSSCWOSTbThVN?`tS2q%GEECfqm0u+x;FE4B3YM#T1bjm>EG2+Zx`a` zUz9dTU@c?~n7L4mZRitQr^GY~u4TE}EO1S8Wu7d;A^I;7Rg7v$LzCpziSShFs;02K zHifkiG)QGFWQ|f<^F#Nx>Qh|HqRA$_PDGy#Taq)#iY=t;vSahZfYYIz&BwS)2F0X4 z2)H{a*cST5zXnba4WbZ_NRechm{&iH{6UhMcUScB;-j@&{O6-}zn^9B#mnpI{s;N8 zB2VQDD|>m4a@iLjEhmU-2;YiFo%r|UV^^uhm^3sc*C{nf{r>mZkx3ZdgnFVb-8Mfo zp9a^dOgR(KI_W$=)$_2|B>_%J>ht3{7*oAYLN$_KFv%4s1E_^1JtLK1k^Vp2dFPYeaY9$0P|HG8;kkU zVF$#gk_N=5Isoz23IpP+l?KFT&@}?YhYG}}Tkiql1H=c2PbmqAZ$@qn5Z?rT`NZeq zc5uVR6=kDANaW!1RuT;kE870y#YZb)9rf{ur2lz>?Y;B{0qtc+W6<8~WZYnr40H;mK{cPFpKwSYH1pJb zv&g8Ar!Qp*Yd+w#fkEx6FBuhr5YJW}5YMg^hIn?ZG{mzXJ)T|h-B}|)58>?QhO_IU z*~bvghG_Oo7m)Y07SZf=ie)`+;#czZzQ+yOzpr01qs*C-34;9a;ll^}^=td$0{!@7 z-}n2W5sVD5lti#auAAeEG%Q<_BK3G-Y!moav`Qk_nUxu?r<{CNtw9V(ueL>hk1{+i zaj2hub;KyFkcI;coNFPT(xt!h@Zv2CsBB#b>UqfJH_T|kP1)7YqVDs{g?LFtQ^M{) zzBE!(!+=D7n2c$z`YiGf~BxAimBg?dvw~Oxhfw>rnH_=>Pfw?FzrZgAy6&OsV z_?8%qQO3i18BDoAYMEy9_r;P5z+?=>n`kl@%`=(h&oQa7zzm}i-x72Au6gFNiW%k& zcfnRF#WPzmPgPs7skrUqqth{uKY@cu^>cc(I%PBRM%q7@4GtBtHb)eX%Ehl?b{m*!;(D3v za%VWO)JpM^BDGuJv{$W3p%j#j9$S+h6y)E&WYA>8(4fh3YMo~j(M;~FR3B|1z4Sc7 z$luE*gZ|_zorwknQ6D>!UX%V8QJulq_mO23R9wZ_FVplh76nkZCs#M4kX++jO%25c zjC>3?)^+5Y=4e3KD%%vwW)zo!H2QxqZgQkv9_D&dI7O&1tIBPbwJX6tuQ3r z*Gfau{YsMVt7ZjAx^HgMeJzK~iB15r?jh@5F9sYk)q=nwqn8H?5#W#k1<)M8A+toj zWmH4B4BRn_HjO43DY#>n#(oX-F>}{>O8S^4jw>+8JUv@_h(`qN-LyG!urnsx#*z+% zGMZy;V%y1brHP?fh;8enA-28r*ml|NLwH-=NkhZibHIlwPdBfce8(?EWimCQ6iF;gqPB^Cn|GrCJELjY*+?YnnId;MTKn`}DYH_vod zGTCHCb6%GOQFH)iv&m-j1DK8SVijid?gj^7HPzx(Bky94`ZER@vrBSpS|@%|vLfSTMAGWEpMcti{kKrKB*piwv&OhQnG6Zg_(O7~4h0 z)+j6v<>=xT$XKhTfsD051?!EiuZDtkoir4zuc}}j_*gX#Y(opy&vXJ%unq<5dNC+i zuNH)Y^=fG-Sl1kn`4!>RDWbQ;A@rJ{SD3k}%^KYEmgZe-NgiPXtJhxz%gZ9nE$6+! zEcB@a(*e_Ia2fkDP3J9^1o2@$>A2_(PUB!;I91|XVmM8#WnY2eoNI&4b_Toi48Td(tVjW1f_>M@s*bzBFRy}T{D(@ zC_&U5P=aXIq~R6|C5V-`^fV=i$p`*R>7b0r8!oN}B1JW6slXuF53Wwp|M%`-5cf04 zH}HSmhYZm*u0X6BbeXNhVbnLV@UYI*gA{wguy0lwV6CykxNl?ypsBgLX{ZH;f3qs{ zjLja#e_hJ=9)=6nz6vN$Z23p3fwdnqfpQ=y2hto+4m2eVt`fzpnYZl%c^6^ zVEI*GmISkA$PS>gPP&OuS$%N;ZBvcRL7SHE0YDpoHUMoZNdRrrVrvZAR+kh7sBLpm z+d@_T0NbkX4q#i+JHR%8ZJP?)=FT>2hg0X}VOMjL0kV$l3;;3!WB|xYNdU;xVrvYL zS6Dv+2zhf6@&ib{04G=99pL1m_b0%~0pMi7$?JfV)fWf!Hr2=+y&VF21N1i21)w)T zZ<)vjptn_)fdhQoeE7B)wJadG>bnC3SM&}D4iMa?BDlG;&D!bA{eHEthyBh?M$A1x z%z&6@x&Xush&dD40K~io1X=)`Hy=1Z;JXrlo~!Q;&~wo{pl3kOn~I**7Y8&q)yN#p z>DH})<^atBno~*wnwu6|V>Gu8e7FGTHXqJC>?>!0=&J7y5M9wbAUZ&Fn~Lb>&Ng~K zRO9nuU$l8hy7YN7TStXtj<|K}Z9v?BxB+o1B>{0yi>(-OJA`_BVdSIB+XN7!y&3pD z3H%~V%e)z+SkJd-)IdaK0b;yhVeYE!4F1c>+|8>vzolEbk6|}zn%&vXX`u4N9 zbX(tk4&y(r<3Bznf4aUM|B2)C`*B~3V0iQ`n!nQb*2lzt$@$1~#X6=i=lU(2tK|lQyw!+n&G&EFD8!0%2NC89&X1aJlq(G-=5Ghz@q##{1 z*Vv)MV|CK710+%|37xtnh)$0dk6LI6+VikQ-FsgwZV}2`?{Su;$vl$)sQTqQ$~-%n z;M)%$K76oWzqT(f(2qa%eZL*g3|OkJ0-YMg`~WqDb=mdUQDQMAE{ zXY9=84cB*0KFQbM7o<0$qQ6HOAj7P*Lf%V56$ZY(5PzS<-d=fl@snnyj zY_z?+0(((jOfkKnZxPr{t^AhQ%{kbO@?w+KY8vhjvT*;l+Kkbb)OMK6Me|JNRS>Si zU~0FE;n`|-PfyF2;RW-%r>oYK%YVj>GobSPK6qhOt=MC5K5ft*e~N!{yeO1Jyx7hDk(``lfKBqhk{<%@zStcM9O8Kk+HGK_k?Uo$ z%LPxQ{;EysTT%LKnuq$BX{WO}txuVTn&c4}G@&e% zILQq#oGczCW7Xu&1Q{ktJeJXZh+$&t7yB|p6`Cr=1s$4euxuSN+9Oyq9a={>R9|<# zE_b@c%5D_cUrS{#SAc*b%{5rIA{m9d*G!QVwM3vva}AcQNk(DdHPfW^bxEd0bn9}z zTdep-VchMmsyk29(~-8+V$y7bD(VDgVXXEhN?(L#Y6YSGPIUl-Osz0f;MGb)1>OoP z@Q{rzXeb4}jFNk(YE@Z~x1)FiR_Kh0kAEQ|OrNX#X~?t&BH5Y3;;M8(OXAVI$FQ!V zOClJNB(#0{iIyq|%2$-c4?HX>8_juX%cG%GdLp&4lxn*)b$*h3N^Izj;69OK5^29} zo%q@rakXZtJF&)Tp;v_`DGb!Qz!IZ(x{AGgF0pEB`z#*BI-!{&Di%1_?-3EV$p&8C z+lg0o@nWjlS6G6K$S2Jrv6#h64>TvJc&`D5&J8B7J1A9<42bdxJ#fpVTs2a`JbJW3QU{8xU*m5QD_$c0`++^W9o%!bmpRv{FHM3=4TAWOs_hm;- zlYyUc(Zdu8S&ZM<-B~6KOY-bDtFp_0I#)^Y<4nid@D;#Wy(G1clwYgvc9f?N)u)w3 z%hr+dY*&5yu%nh%af4U~%ClW}+L8P^`!18ZJy=7D!4ce{UxVwhn?(Po3-9vD9OR(S?0>RyaySr-w1b4UK?rs52ktZ|r z%)E2XcfKE|*84)FBXRzU;hLB z(y13^WPYCy8O6xX_0i|LdQpdop5lGBWFBlVqMwo?|>b z&P!|kz~j}o#!%e`*In-Jy@bbXXI=WjiNYU`Bg<*wu#V%8k8Iq8F0eyiSqlS3>nL$rnTN`t*Rx?+wtTXiTh?gcisk8Pl+|jOyD_U6-*e*aZPPnhvpIl6ME>V@frSstkw`6hj{yZUOMegjG@t9$+3PG|+kg98$e5r&md%>{%Z3o;qw zmmSi}t6zQIX-}*PtgsJ9^_^U0Xz?4ntneWY+_8cE30;rZCgau@#K&==F0IFvR6@00 za_<&jj`O0EO=1*M%fx@?mvP%`z=nx_tvm-RX zf;1lG4vAC*OMwpAV$mlMxuV1=N@)7_G3^)2gk(~i{pSbo?!s2v$dH}I5nO~<^?AhP zZ5M~#L#p%A6TGN#{Up`x@JAJ* z2uG%( z8e|y8&Bc;slf>F;XCc7@A5A8PYE5P{2Xo&oyjil#;+zM;2KAJ=b+Ici|8}9S|Az~u zPeK0NBT^acu4w_dpNsQ#8zM!sftR0p845m=&Z_pb-f80=+?eyVZS-dsu$QNdMSR`wY;Njnxi=U+PUXAlDnGjpC;ZTsv4moFT92LqY|* zXPV9E0Md6G-lF`{NH|!RVGdQ)hRmu6n%U^2NA)68lbkYolGHj+s6YO_zXj^2k7Cdy z@3{vZZ5|9aQDR1!R(c59UaSRA>~z4xJf#Wa|Uf@R|;%pS^eq z14Z5(dNBpDj)uO?fX2_56*6=gG7bpGc@sylcGIh#xNZM-@PH!B74!g0;_(kbs4>rs zlQvHZUhs}#5}vYh`F>)9gBivu#(u9PfNg4fLO5YdeTZ2YuLNVoNgK=`xDf}K$bsk= zNXW9u2H>~Vc`YVeTmH4!-db7z#|La6}M zJ}V+Y9UApp`JC`v{KsmN8+2bJ2tmu5>#0onGs;Y@8r$?hx=TZZ1&Ph{XS#KCd)Z?i zW(v{#`(_G_H~Gy9c&yWJV~X@R=!Em#9oeR>ts7CC!J~?ii}u}F3fIc9mvYsvkc!u? zLq>a!R)a$61|$Na`YyX*YJA;~TcDH%%j{~FKi)fMD_9DE7FsMFHx0~79}zDkzFAWr z*`XRJ;OkrKE(Uq4<;%E}ZS0Q)#P*s7Dwr9e7i1pa)u=6TjBs7haYF zQbdWRLEepw86Nd@Io1k#sGpJjGDSQIsP^W1nJ)*)hdE82GwmeRoR==yhBb$$Le`&} z7gRHEQga73B;i&CU*qN)08yC&j4)LxMBl99a3FXIE73DI#&r;PKVqwpXKp07=b4dj z3u);+L94fOg~d}5Sg$I~e=tgyWOif7fi8Ua(;}7HeMpj^P|a-722pnOz7bogdY)79 z`)rLU1qKPd3Wx2NHO_@M@1EBb;Q0+ZOBOL9>@XkopTSV<9Y7np%lWwWIgK1sp`qHD z1|Kb?0HUID;F!wU>3`s)3Bue z;XM{vWR?%Y8nu$A(lj?})tbJ$VRBmjrapB0-2}Zf>w5OEQH<25hTySEULD*72QQj* z)gB*p%Qk-8Qad7~7m?VEH51pATgrn;!BA_R{W9L0ek04NHJxtCLclT+)^lm7obAU8 zl|6Jun%H`2Wi^Y`+OZ{>p1WWwnH9j2>&AN|d+Z#=XnXxd>Xqr6C;Xnb<=}D@d8L|_ zGjyi)t7R%yA(9~ppPmQo>s4ADD;mf6g0-hz(_Sguf2R%1j%ro37ZGq)Q1pqVId4Bq zvt(Uj^L9%zA_zTu-NHAxW+I*b_=iC$n$Otde%&q-@n%|9@jz-TuGCJ_?Ow@CtG~{W zfM(QXx&VdG$`Y$2-Z_(h(Yc_lx1#1G=H5i9zJRI4_^OPc#9rR37e?U9Qr$b;Ih)^O zF4=2eM*NORu@Skng>On`xSRw1WAH1xCX-+?MZHxA&;BZhuA5~36w>lk`ot%VQc0!P zX^uj{`?nb%8(lVKbXR@%Vc4bA2R@^+VD)37lz`o=cV_)|5M3XxICCQUdf~OLhNrIIn=5K(5)UkbzTt@F%yaRTF-ccFW?-&sdRmejEuY~>%g8Fnx(oT&^EX98bKyM`>;s7Jvz`#* z?l`7+d%@as3FG4e%jVonMazgRNHX|(n$kZcypGciMMg7YT-Y!giAE1%5?CKeBl@rw zg@FF?m&>}z&A6V_FKbdGm@iNwGx~bn&@20M1hQvh7w0~L)@gMkhir9DWr@#>EsGBi z1o&g6xg>TiXH8P@7oR$&rRcHY$rfV&O{k&Jo%qYxFE8m1p;s`Ht#pMZAmT zF_3;yhAHCJ25SgY;A7e?+%3$Zhoe1RF5~-33;EXfB~$2L5^+s|(o?b?@(J^XLdn^z z6!RRrtd&elTXTc8uCls2JzxLwDEH9U(gU( z^}Oz~So~vjX8Fo}9&;53MUkI*q%?G&{V7*%%^2+AyxBT_L7{iok&2W)I`}3~qG_V~ zK`Apq6CoF3|5Cy3rZucJKnl@}o20Y5>^(toHeeuwXWwQ6Ix}YifCW)QFN|(osx4T1af482q1vxHx@IzBplzJ3G_ebZ)dhgSfmH@xb7|gwxj^ojwPNkOZ^aK~GgUFRx4%{2uC_Ps&2q>lt z?{^WIF}!oy2CfEvb8SuRinyB+65J$;Gbgpk?0R*m9m7g>ThVKir)C0fwVPi1HkV?C zsH{cpe6T<>sE|lsuG~aBA@_;6B4J1r7>yU%}oagGO8#I7$TgY zIXF)p&1r){K}MVE3*FiU74Ktt^9uXp#7~K`hD3|-o*IoBp0Y!Eei>MLX-%CCgdD&h zZ{|~Z!)1zsA!tEr9bg8coa{t_XYz^gtALvYB+jPJ*+Ag0Ij(%V9|&8OrhN~4gFYML zz?AiLnOEEg_;zcz+^pVeH|7e7cg?{^6^#Ax19!K}#cSudUnlSyLp?J=u`~hcfiXQx zkX~&SX`zxue@NS!3QDHwp6O7)#`Y&HC-^PCY9701M3G3yZMZz%XL`>xc9XRVa)f8U z6{8e85ZtYGys!SC?${)pFtnh_>OqfhTc(zScCamiq8ThgL$LGJnG1YI0xV8Ym- zYg4K~yKw4*li&U!!npJ!@xnug>xmPicMj={F!W?5u2XfcrBNTI(HO5~un3a?w64o7 zYLJHO#2B-aE~XZaeE>DPSQb*wSSJJ<6D&XT^h75_4i6#A$2QbBx)giMwZVLx&m4)U zl%L#XIq_Bli(Y0SOJb@a83yPt*Dt4mO&B4kWwUe(#3l#G{F?F*$%Y!NlOP_sGT<3v zRM#%!J2cpj2!5cn+k~W%n_;%Jcsb7G@vp|_a-BBCfK6cI4XRPrEB$Wsb9oh1ny=+4 zTe>Y-iMQ3`+m2!bwflfNcBAeJe(6=yt2KVZk$Y?w&}W7xv6xs4rIb6(z96^VE_sQ} zYo58)4h{tw(GF7r$((JAE5A}aLl?Q3Ell)wbnK?~p87kC1@V0sLfro9P!ZR2D@cHP z&u_eZRBP8SAgzk#6}83p^%R5J=ql_K)Ss&$Op)2$P|R=CoG2V;>)5y2fWY5TCsKD2 z&Ttj9;FiWdHBwRRLP)K{fn1tZcTtB~-pdO{o&I^k5vFOuQtA16l)pHzj=y zT8pv8&yeYK@Lfws>}ng0LloADC-&u@1|u$)z!la3)Wio{i zvVxv!MJ@!i@y-vb;bB!vuMBuPKd|Y75+LXkLu^0^boGK@LtsfB?#zky{xY48>i)hv zYA~x!c!r&hbrbU4j*jN3bqFaBk5>999$Bx$GOo};nP9>Yz8cctzUhGC3=(L0wGU6Y zLceO!IWD_|DUMUL0G5=F4ar=_3zD0Kg-9xow7wLj+Z5G>3rfk8N^XWjO?iT;x+>N@a

cE>&4I{%r5jQKi6trc(VM7x8Go@VI!jTCC-wRKf=OeuQpu(_} znkr~L(HGuZsEvjaX+U<&u~(MQ|3;=)3I{7xfl5R8dS5J^SKojsq}NNkZ9SD8;#8jk zdbMMhpc)4BWDf{%`uL~Yz&XS((q@5(fW(L6R-u98{-9+ePz@tTY~n`sQCj#sgb~i- zL;J8xPJci42Y(t>DY)Pd5~!$o`@Tn;z(p1Na#6)Pnsb@*-SKBQ8san+#|CBJYbjM| zPe2sHDA~jA1C>OX-pQGQx{7}!ck-aWB=_kMKypW6w&<5Xhkf%Rxp$+!NbVY?faGpa zavv}7BDsTB|EJ{6;PDU1{q!%%9rhEU>DG(n4xO*e{rxY=J< z^h2s>Rao||b&)2LgXUn_*iStQPWYmVD)b7tE+UJ!P4Zc~65~5{Q{`8rQIJ=(4K!1O z)M_mW`_s5P)4t#hNGcXNsQWHBEolWsYZfYCRSci(z-yQ_JTUT@zO&W&D`6M#`dZ{^ z;KFZMdR~yp}C0mr*sQ>M!UH`U!iC!9y8sheLK% z$F^=Ytm0}EK1o^vuc(4s=c zPiT>qz9AedX$E^}w-Gjzz}VN!^)`(#!RuE2>hWS1w$Q!cXfyhJl+YQe85&g62%M(P zytk^(=FC`{-kZ`|lk(+(P}^?21aik8qsm>bjqF(g0xQ#|PxiJ1ULpRn#)w3_B4b{( zQKo}}kOCeDh<9wrh>sF%iZYSfsjoN#L2vC6)Gsgb4yda*c!~OS32cYy-m7ud)LC71 zrwu_kd*6P6pnmzZ=V^grwmkLjtxDBP}7)#pux(@Jt@T=O?< zZog>8<3VXj7E}05A5kk36)VzNUzU+=&Ho8y81+{n$CwPjeGw}nu7WLN!LjK_;`PfUBxUPk07_;u1mTzvVsEdvUy=m z7!K+u^icCOruLb`E{@IMvi!&45tKkQKZ^It{2Q4S1(Gvo*-GeV&i5Od1H^|7i{LA@ z5^51SgVf~T*W}PCwUKWYF@!AvIXn@N!*~B(4#%oNz-sv|he!Oslf&U&iTz6sU!DNu zaI+h;HSCvYM#GoCMKc~o{zNk}_-9oWM*RL2&A5C2pV17@Uw=h2#11>Ei2LpbL+5t2 z30jP4(k=9!M$OA=W|F<~6Ux&}+6RxXpS)pU*sW(JXP49H29^YQcWk62E8bUbV1#U# z`uM=eR~r46!#`25M09ryN2pu-+x}NMyifQqIb7gH4$osJMeveP5gQWe{oF59^vsBd zyhmj+ui*~yB8OWT19EtV|8F_`#~(SI?tjYR^+TAZf0x5?|B}ONp}St>aL(KRDTfpO zC5H$7O%8t>^hXYVtDfgx>y2Q{@N84SCGhSvE=U7|pF{HyEm4MIJf6cyjMjgahG;9d zI}`FN2EErvGUk%114xJ}tQHHGCOFRsl$qj(nCV5Zvpgb(PXjDRYR?bpn6jkZXy< zJsT;PgXP`0*_2RF-aEz4UjgKjerve#xV{T^50|h+1Vl~>3IwhnKL8I2=3}GTBLZIJ z_su3|gB`&Nzpn5QJXqg%P|L3a9`Aq)m)@b-vy_4Y>!=G1#OIGcGas0n8HtZH^h=rN zo;TJ9HgHQd$l9?rJZk!cTF0vj?jT(xtlFZ^PXe*jqmxk2v5H39P1ZY7in_sh1j5mh)$w+J2+9iz8rVH4 zpeKdbPw5#Gf4Ktmo9n)6k}*55Ik6(;BI-Zk$KoLjG2;PNcOp@WIb%tZ;ghneo6xjX z3j_4`qfm)`?Aj$>?BF7o1JmZt+lpNj+-I(gGoSBQo%XnB`d{JNL?Gj>rSnftgZl+M z1yT^liC2n3K353|q%9Pz@3Y)p;P z9P#qR*U7@F8Miu1$YQu7-Y!Vo9-uglV6(x<*^uL03Kol#{NZR|9n(milLG}oD-yn5pGd74d(S}iY`!{|zDa@J? zuZ392VA0*sZRR^F>DvEp5KJZmI4e~t&?@((Ci!3*8Ct+IT!X7|U35+0_PQ{dUfcIJ zQcK7-bt?Fpz`bkX{=901XG>W!%36%iX@UP*crG@A?6e%nXO9MyIz_laup7s|azm8B zDO8uSoymwZx70v!SGIzP>O}U#*c1Zl zj7_0mUC>}HNnsLeJ!WB&|Cge9Y$cYwJrvsf`b;C*yFeqFqgXVzuAXV(+(NS`%#nf& zoz_(DV1V`k_NCh6=QcjvZUV&f@GJc+ka?SIxsx$t%_nZeduR>a=~wPiq)fP1;l<;{ z9L~7c^7KDhIb67qYC3oA)%+5w#=A}{q>*O3P`@qA?r+jCJWF9!CM@DuIcI7}ItrJK z#IaB9PldiIL)JRbBA9#yPV17QU8VXkZ(}P0!dx$PB|-M&^3ba!`NZAJ8x8Hn>9Ksx zyo#}SS5?&M61|@v4qkU^dj$P$>MfY(yVt_|c)n^^Jz$LT9IT&ZKZ6o!$-x-GbZ7}c ziRa}rypJ%aD&976q znTcn$_9#i;&h2}s@#krGgQtV@Lw%&>11WClCa==W?L!FHeGpb(_kxh)V1P}MaR%iV zqe085(=Ch1aR5Q+7QcKpa2xDmzIjcD5^Yn6Mkoh_RZc$8(zb3HSI(=<#b0x3P7l6J z!ywv!0L9DUTK7n+oEJNA8q&duu%chuBFY#7V%- zaGfX8sW9ydGgq`rP>EqsAm5|#vhLp{{X1BLIGtRJA*Wddj6B{VRzFeEFo-pIX`Y-eM)ZMP3FbQh0^a-5% zEK9Cr-)V69E5JI&4PS#jk2K8N{d$-bT4`PMczv+g@SM0CsUQwI)>t9vT+VjAEfxU% z9ENJGbh}MT{S3pa3GE9H!U7c^f`sE;)ff2LdsIvr$3LN|+Zy|-UjSH{M%zEIGT#zR zZLr!DG)Rj)USFA730z-w;CTF|U#=bU_E`Qb^vAu7O1DDbLlCzf`@7YQG01Qh|GQNe z3dk@K)^IZ3Dmf9;e$80?)zhpD3`xpfOdMd!g@gLpKD8$PfI;;1B6$*ogKzr{lsEnh zD4%};%1N(DXZ*G9W%)(20HAzAtqWG%I|6;$sf}WmV z8-SB1>5VnO9_DL!%N}my9|A6GzZFM#70kOZFKyc&7F-s5a@*N%lozzHZw;C29+btc zE0g5wlTJE~FcXVYjLxrK;VRStEc(ce0rwYKte~6+ho3qnNA+>{wwRYWulDzEA*DFW@H#wX3DND)3UF69~AZ01bmO-nL$)&fs z!*%KGipb#vZp?S_ug=zv~~QGB!xv^to6g z9*c+X3zxw^I})0j+s-4;IolBa=@b)oHGg3Y`G-|JV8-|_t2pnsRU9()LxmQ99P?i4 zoTiE-5%K+|rRp<=q;`V(L0mJ8wOp)a_V>@YNoIm>nsw9sfI&<#HUx)7hxgkczQyzV zjh3lHC2pANw7M6&ASV!$olDoy(&rydKn%kvr0#ZyoeavnGzR1_b~&k=v3MK1nh$

bIAr&+6>RsEM)+q*S8?<(!Z zN`0`DxC~%g5z`I%?Hd8<_q8!Qmtt<$m23}F@n6pM(=jVGU0n;~Z4y71MR&d#a-eg> z`a7tV1WU>IX#Xiy%+_h-kvImjZP*oC`E?r_;IYOL#A8vIx?#UspWMh=bmmjI$qYq_ z@M!zFb+*PLh)?J|a%$Ps@4PmtYT(@A=xR>(UB}7i*=8BpFPV+Nc^?D=al~}$i45P=sw!q zrgyS=yLy|s=r+T1=;Yozs$~9(i5Q*&dz4q-vuysebst)Z*+Xg?5o~CpX192iwWNND zvUy~$jHf7Inj4F;Q{Dbi6gCNJ#vL^j-W1og+;WM}?GGg7xdtas^NeT)>x@(nWr2`nPzMY@;P|h*X^Xm@73)q%NANH0XbmoFpRVFd zSP$9WdiI>*wN9Hix7yQkebRMlxVhou&V1J**nE7fqr81R&SUcZb$%N(awwRkByW)fLev#A^U~^I+-M#B9I%w<0nFUWH?UjP#LdV}KQhmK#A?Qcb5rlq&M1@cJPMEp0 zgHwe%RWD}ZGzCh5Ptf4nS&SbZq)&we^4SA*{FsLLVbA!Z zBH&iQ_ZD;p1S;y$d~cf&E5oUIEs+<4NrLI{>?{qt65Yc_u0%HwA@OdomY{wq?7OY}MxNH!kq)Hk$1=ia4^&%GB&i>k(e?i}CY$iOxo@ zo<={1TrDTw3w&-#+B`qUToDTC@Ul$581gjsIP$V$aySb~b|9f78{^c2qZAXENiNL< zO(8w)uN0%kfJ7mkvcb|poWLu7nZO;Whj9td##Q&O&#Mr9B_^yLdWNbEJ|Ph-ReQw4 z@7=B{Ts}np?%Y08K;J=}Vi=fwN**I$2p>vSjvc4~(d&9YkXuKpc-xO#~C$<>rR z^|_J5+yF+%$SfLMrI&=D5{7KjjPW_x>teuSetl19BlVt!`Z?fSi29lCKJdIFUz;}W zS^ek4M&k3)C!lr0lgYt8W_TdDb#2{>ScN-w`#FCFr}U|KwZj!ApfN+F-SPS0N%9(L z9dS6Xu2@AYgE!~OCy1sjHgKkF#6oxQi%?nh03xq%{l4Conbm<7_QR{D8DglW%<_HL zrsVzRgdS@%pn6dKft5>Df417imgHtXpG8bb`-*Q_uFh+yxD>1D-I(@~?6O=N#}`L? zR}RB;;+x1Rhx&s>QzZ>p(@zH)P%x&ScrB^vbe;-^kj!BVnQZL543Er*Bb~vO3}B&x z+*xi*-Rcb9z#}>nX42?$d=^-c48!RBJjBICTPc;bCm{~GYQ2s0{k8)9Hi3uu7l-ie zekjDuS1Xe+4n>eP_{>+}u8foh!*V7Vm6W-f;B)kK^LCX@s9Xq{!$2^o_T4421;*HR ze)jN7Fi3I6&28?@PXiqv_(np9M>!nd1F%_hQu%LJx9*T0eV_xdnYSTk2d#;-VnB9f zrzu|U)T@-mzHyPsq8)UqhycF&mH>2X9SC+hE5n6$TEc|P>{J}hf+zri3(xEXn8;b6 zH*C@JzFC!Rh}^YP&<+*^$P86m;FxLHkkrMzpf5A95SV3vZHt-4w^-V?bKeBw%$rWL zv0@iy0$T&Y!U%H) zQRM)%&{VgZR$%@CF;zvWJb^@aQI;{POwkksWQNV zXcs|bhmTgwfBvb=-+{$zyuP6y3_Dq{e?9P0 zI!5>}9}0m^j|@(N9L_ewJ9|h)@1Nvl*6(rp(^S67_n4GUfAS1;1b4DV#mywK1<$hX z(UD*#^tH+#G?k^NWS`u$a5RlB6>v!VcF;;VlMK#?XJm#im8!g}yd#4f<(Hh77pov& z{jkonQhc$Xs(dPooBPa9UnXkWpVmFQ#-k4uugk2cYLX&wEq@r@S6UbpD)>=5w)Rml zwWUY}d=oRci>9)a=W0U7-re4FVs6?HX`Yy;L&X8OOHcp7>w~Krqjy?>|5RQ7f8##| z{`Q{|U|b;G7CRk_exxlqK0LGP~r zLy&zu_tg}eobhMu4K5c~&PvmNF`DxoKu_4~c2ef@h*M3;f!yssh-T9X&>Hp+M03kE z|1V3*{Qi$6m4E*qmQ?o*)j-YXwE2Q^wYcYVlbv2*nk$<+)cW7lgmpfVGoCeoALVfJ z?s%h0`~QrYs_ET%viD{%7Vo#K|J2ee;K_Q3PneE0I?}h7{N^;;ah{LFJaMzpL&ffc zZa3w`?epK085-XIEt!!;2qZHYtXn?0&jZPf*>+we+vtO^G)0*#T{)wd;eV7jO<|{T zR`%)GiFeY6=&MuYf6o5=yXpA=j8>fx0kjhw%s;(SOG??Od|#{=zf8|KH)6^E3AU!ZQyR!+U;GrCz*S z!hVTp_~iU8qM^Y1C!%o&L^RAf{}$1}fHnj=NRF58NN|vwi2Zu)sbc&8cvAez1|m^c zi8xgd7S4wec^|Xg?u0VxMzqYYleL;%I3NDcfF_9y;7R!_csaR#{@|JbDT|@c#ivmw z9fkN`rhmO4Blh2bW+lwRvLq1Kf49pPLu!P~D=7CQ{pdRGASFaHSU-{aE$n@=jlm0` z8S)pP$@B-%bpAI$Q&?(F(QHS~(gmqmE+tQ(`v}`AHvF@J=28lmTzAy(+5gx7nEl)F zrJq6r4A5+=#u%GB8fvV215_$-_Yfkr6;Q;>>#u6|?F~yGGmb97C_~F8n5|vR5j9eQ z>aJt~i*VfNFKty$Z=b!Q%ap*MrTJHXx1dPvs!){2bA6>Gc7dMM^j{h30#T3IXj+vj zfLM)5*ff}+ldGov^;2GHQ-lr0=%Y6z{@4O4ASC>M5%wH-#Ioc5XAMNHd$TnoG8fBD9Yqax9Td9;)wQ!&xO8{Ud@%@o<1(_b6`+~ zaX!3IAEP>S___3CC&uD7Z<)_;1X%-dUHc8H!7v?MmB`}yF2kEU%3~M9{VfSH%dyhD@4hh9rjTJR^hbY&e}~6YuKM&i4~4f zlLVpO+#ld}!*NBXP@` zBgHet2%Jwb!8k4RLNT$iawXE~>gR7nu3oAmx-Vj4rLWapj2tR6O=qLfZdlJ)G3eLLdKBytiszk<6+5Sdm)K~{CA|3WDkClQi$Ukwv z&TWu*a4wlx6osJVO$mq;(Tm9*|Ev-&q4gC^tJrD+!~nD!K%6w^yDvg}KzpQDo9F1Z zI)8PZNZ}b9FT!a|=C!J|g?1cC*)JRa3NcfvR5x5}mZK|B`0nX#H^t9{6jMMHv~CG0 z#}Vn)5-D`hAW&Mn-Yt*liL`h*18)9~i{KPkeT5m0S zs0t$`$sb3(mVCrb9NriQ2um#3%nbO4v2%PfQ+^9hWGO&oiP4KiEuF#G`pi2HV?dz} zCpx6(OwVJMLEo8zYZR2!rhsR3EgLyM+vpCv$i-!&k5hLdGDVnu4FHKOds<>gvovNf z-MN4{nB-L7kB5^E&O(D4JEK#|jMJ4QIqm>440Vf`G9;h!Lo5NX1QoSj8Yw1)-6sG` z@U++Rwd*U*3uw=HVDH9rll$or;Vd!#i_+WD^dy}@g(U!`;xf$QdwJ)=P31aOC`!1W!VaKd{AKQigthxC76=cH_l&{r-o zE4MX7JKz)E@<;vKI#;~=zgg$-H1P>aI~~q>B>j(kkJy`}+>s~=oJc+*%762v1(;r6 zJWZxH%2<>+C1A6iQ^KGtOtg5T#G^~sem-DH&I6CwMEqJ^#8zt@ZhEpGYw&F4k)W&eg?`a_L|L-FooNJvI=5- zulK&G0Fb-34c95}n+jb2OOTDpAvWMkOSyi6e5XDZV~9fX&0AYrhnJ@kGZq@#igf#3n?C3pZL_630; zK{F(v%yw?jD6pn=s^(CF_|ZU$pmd;uBYo270Fj^TKpWbZf7%P|TnFEc@!HK3lmWPw zt2Mg>om9S~@=zHszs1AX->zlLP%7dzhatFK&DTZ)bhq zyAIEWSGu{_;GI5NsJ5b@h30V#vBfwO`NMVtbh71ZS+ zMPUeZ^&2RuOJ$b~u&`n%;0qa)!|@#?VV#!>E3pGqYv5>YTv)s0xMGgl|I>Xb`s2R9 z{L_8;`Ta|x^dDxCx+d;lDSeCIDSeW5N z_e;t!eFiE4w{21!?%b`rIGjq8Ke$eXO+c+w|2MCQ;i`M%LN5|@rd70{3Pkj!4v9*O z8P8v}UJ5(AHtLy8~USwJ`t!APA-sjc8Mw%zD|cHTORezeJu7UVPPJ?XZp}DTS19y z%&e^S7wc%38)7Gv>{dRDDyJg?1zzY1wT6Y{js3GldwZ&N713HPyrDa%8zJy{U`0Le zrr*5mQxNeB<+wdJaaQ~f%8_=bN?=7ay|3??ziRfK>L80!P=4oh-z0d z-MnfU8po*bB!SHT)m~SGmeyTcCIkRL@*PQer+Fn#Q3X^hd(8v+zayZGdY?~lw@8uB zaPIHOW5~Gni>s8PYfXNN%C_{OA5}8FwR7z@#&|90gv|}OZFpK|ZwFQ;V%<2;O@I{o zWkPNQx8P0WQd?%*S4BE{4?H>};(nA<`i z0oWO9Idtv4b37su*tGlfG*5;P&SowbNC5lv`hD(^qXY#`T=>GPR&ql=4^!KQk8fQX zrjP0ne5~|;{v-eQpA`Unht3M=&VYy2fA;N6q~W5}(ZSPm&LfC7^32y+5>tjg-MRf% zs|IB@ifktTnu|{eRRS^ru$Sy5^hYBOXZP;@_x952?6$}f1)8iA4wlidtsd5tEPOz?4z#79_&rTddCWDNhaa)f1Y&=i@4lsI)X@tpXZW4B*H z)oAC`E&C^%k{Df6xDYGx?MMD{o3ae^mWj{%;=bU-`dKNCZXvwSN$go0WeN zk3atl@p#p}V(RMQ(-8oXQg@JZa6E!#YDbjj$eEin%RP4%(tT#t*bzWk0{BjVw{#=3 z&Jtq%;w_nQe9=0qn_!HY@ynD$j=oa2@WBEIL~Ukk-q#Z8iS&-+$KIA%sq>Ma1j%Sf zw=*}oHZ!JtaF9M!9N_%+mO^l=o3R%{ZM`CI{d8*Yo&@j1ydxXQW3@-A1~)sn zVX>8w5stDSJos8%_^GelquNg!w=R{2EG-4ipRj*4Kh7(hxUycLK*vEvfAs@x-JTc_ z_nn|&)ug^NO^&hibc?!W%})OnVmUUJVFdPdt* zm!-MeBW-6lbI~gTIU*uG$BtrsFl%ox2JI29?J~HqRGJY=pN>TMr4l5LjaG4uAJg zUrF6qui!k4QGCp3&Hi09K&nk8?~}SGA6=xTk@m%?@iW%S7HNv@du#H-hoHCiT-wdS zck*HL9#}%&6WyYAp>@u8N@7M6u7$w}aVt|xk;4n5e2Ngd3n+1;wh={QtO#hPn2|%Y zJXLIzEZ7^-+gLnRWgXu~rln4U_iAaQhm1Y7vi5B~4kBU9%@q`{3j0Sxx_G$x-6EP{ z%tw_gF7TbT3e4jbb|UR6KBJtqE-G0Ll715!q(Hu!?EEac7CxC)iWVVp5uLhjPUJJ7 zZ`q(~`cBZaNhw~+t)h8ge%nplN6}p4&EBl04jP&U1y14NV&AixryR|hkh*hu-BW?J zlaDx9%{rmVPOg+aV*$t8xfn`g^}I+yWbgrr6IG12w;R>n0bhzPcz8nAf2=aXmAwv} z-dQ1YS3>vlh9Ct>47eOJ+rheS?)SiDe^Dwns^e+csGUs6X~fqL1G(t0EbE0rC(%+J ziVY03PR8hbrIjHh6zLD6crCGj>(^7^@!72d1ee2KCY zoixE-E?eB8P7w>k&S)x(o61Wy&A*1?N#qYD>%LMa45lOf3`F<*cyA}+}+%&8gL(XAJ>;B9L?3Rok$y?h z4GtuqBJcFgWO8m5dT@jNNr4LKTtxxnna+vb1oH3v4(=^PPmiiLBs9{#A^>We>>3D* z_fBCIz%cp7WpYFhiH=4CXz7Sio3?|rNX`KkQk zi~b8;2=Lc3kb&Y1c!h62Ku~OAb0|$ke(-JsUyS3~ptnvJlMtdzBDun1t?l)63Azpj zZn3zKbG}CFh5~nD{Sl?h1NWz~#Kb9~8*L} zVpx903fXS3PO-_lz*qGu*1{4C3K%GtoIP7F#1>2#DLUmq?03J0;M2aTuKvsox7tqm zg!SoDXAuO?098iE)BQn38IiYzu8!|XhtNf9jW7=xV<~u{!dHilBui(%2hysg*@b3< zIcL8&_G-0ZQkQIw2WBIJoA@l*s&{lDB?&`%1#|D?YE>kLdPHJ0*wIbCBs5`_%)F{G zkvN(ubm4WJQ7b%S0Q#b#myvZ>16-dTo6b*DtUv=W4BVcazCvIr;;J@ z_=}@hzl)749J*W|A|A2lVzN_Q!I#CV(VIiY?W{j~nU5ec@=$@K*si)Vv_E%Cb;n1U z(`~2q_>m%n==#3$^{*|wYK`{p)QXcYFoMo{C&-^URrnAv*;|94eDJn3s`wpU^Q;@R zt5$@+BCe@Qx<2Ycnkofv)17e0{4dJhF}jjR{nm|bbZpz`*tTukwr$(CI<{?goQ`dD z(r5K=?{Uw*XY6yw|9(xSR#Gb~qvo9NGvC@iN{A3u-$7S8Kg7?%X&7@!^%%7+JL1hN znlYQBXNygnFpDiSfz4|v&Q!}zp3#zxY~U!YR>hk&!A8`O(&?x%o+aDlINW72ibBeLWl90P>T@(L@A4z*tOX$pu#celEQMk>rsx8rp<$q+X+Bam7!+L2rvdVj{ycJaKWf$>i zl3Q`6N3INZCY8YKpo3-Og}^O4)RPv4`DR+=u9^h7>cgMk8jzPD@5)xJ7`nDIsyf54 zDYwbP+f5%m-WlhWAc&hKX?ZXwuMVY>-D&w`O0FGD>og(Qn$S_sND`+yB{A5|_Ku z-vh2m_BC_V;^5#_xmkgafgx(Wr4*YCYlQ9bJ-KdZ=ByOi)#;GOqdAEUd|7jlyq`QcLYsLI_API^>fo!448d%9 zI&-f*eD@sK3L`)2Qs75s1H|Q=iS@oec_qJPMKS22b>24#2Gx}+%b>?a#<&jX`&7>F z(goq^&W?CLb0jbJPINRV+S%s^md;vzLX4GjL!}K@)Q9geS@o!zHY~c;EG#+o`j!p$ zJg9B=JczCL+5-^vKKzJ=3}Zk06rl{ao2Y zkDT3qSl3&&I_f{jEz!9*BnaF{LBdu(l-xDE9&qB}Aw~}{r_2@>y1(? z*#uMNQb7MqNu6ipo_PA1nJa2!g5y;#H9gO*c^$UH^f-I0Zm#Sl{$&?DD;=2!&b4;m zt^t#$?jYOW|KMo}I)|zWItS0gajQf8xr~nik-uhUeETNF!vH6e4uc&*-vuX9kIatL zBlccd-ApBt-_@myk7kaSpHgw|tsm#YZDVpg6(-%vgt+(-Cr*mcQU z)o1+XfWeWli~4wC-(+V~BUHGF$0s_H%$Gzw??xzY!en+(n;xNkIZ73;Jbyq7+j^2J*fQcvMr-{H>y8{ONni*DBe^A58zVv3q0-1;(Q&-$mlf@~cxNrlfO6uk za>ROWgS>02vO*(uA5IwgshZ=YaM^MwVis!3vbE{B;7fW->*gQDCpk?Z%T;L(h|VWUwk?wAwCPVaF< z0f@A!00mw1fnhgc$R1?U5@I0L&k9S=!YRjC{E|;PnwR&^txjszn-W#WDJyPEhtSXb zOm9!|%TCTTrl<;LNL)FQKX6}}ey*ju5@X3?saT~|b6iZ{9cR>`LV@3rFa)_J%z9*% zC=bQ9N=SH>U0Up5mxtLatei$hiOjuRYJz2emW?231nj0G&J`ytm+;i81jw^m^+~&+ zR+kw%AgEUtzZaCc)6lpj!`xJQ#5i?$D~onu;D{h}4Gbi@D`-NoU{6Dvs;EupW&s<1 zKmZ;~@-s~vo!vSrt#=|vg6|F(Bsn4c0X6rqKOlz8q-vx7c&bPhn||e-$w&Y zdYYKUwe-z%j+6YmZ5w@Kco=o^e9iEE5#CfZIfJAArgCjBYwuU&IFUQ}P){scd{T3Vx&T!$8xWxqLQEpT#Gs3`f#exDX^D78YK)9qL zMFnHDe4}q!$nWO{p>#$9%_&2CLxVJk(FGgL0LNtn!(#*t0qu#W4vd^!+mijnenU6^ zYO6d61$*e4MeER;$`cw5MlgpOA(OY8_ED8?Uvw#(@7;Xs1l2-pjNOPG=7S2)FiC{S z>4AtW1-o)S9YS_v8l$ujKW0R>@(*WuTb`c%mC}hCc%RiaxaXZt~gYPL^7a#r}}A zB*1qoh);Gg-|PJ092)5iF-wJYIDt67dxsQ`e9O57{Jp~q;`&C=YH)ZmqQSn|c4gMm zXg+QS-JD420}hoBVpU!6qPic%8=76wgdIUwoe-v>OW&X%I(Hl-zxa&5ap11-71cX^ z-y_C8aHZ)0L_mlV<05BZL+%m%QT?>TneCeaoFKc+9{{IsQTzQ^4(p?{DJZWrxXlcyo(X} z(+^qJVUVu9{le64ZmSe8(+CLqx*) zLepk=DpAbf;F^B>ZfHj7eIzgjYG#mHbho-Xfdwm7LE?>xRs3)=bb+=~A?baD-V`M5 zYrH_8dN)9i@o@+O^-gDGzd(1;ycYHQaWEPuk+?&_7er9WE()a!IRBb5Le0XdK;Rc>#|fP3Abz~+o)lujQl?DJk*@;xX zw~cW(B{B24C>Y$(`33YI^Cib0Sqng~PnjV4oL~BrqKXOopB6FgN;eJ&zK1&;@6G`a zGkWhSpU{GAEf5HM2_tbEO8_NKCgBCJK3)8IvuOQO>>=JwnH2v8cs9Xsw$Wnw#|J_6 zPYMxRz8TMK&HcddZm1IG&S!dQ_QX9i#BvfRw#QI%3hQn~%LbpqoULNh=cdOtuW3>l z#$4#z>$pprDH?y6av{2F(YsaoY$A3{oSA}m;JjE)>5>^w;dxjd(?sC_Yk2f>A$rjt z)>U5Bpd<{Cas(YFo<@Ye1@A?z-9ycm*_~4T-eGgY_fden82?-g-HXa%^Yy%xxi|lH zn}AM;swYp>YpE1rzp4`PKpx7euS`Urg?3;h8|txDf@Hs@67kodZLy&rbdiU?+R28# z_DT}<>bnv#zwqP=nG8x#Q(zDknk0D2K+ESc$aU#YC^5(i(OpWAWIC!RDKK}}AjGyq zyMP1$zk)JRL!Lhw@HJqmf{7uK{CRUojQKPRG4`;;l?1;=|iSkT)&;}Wm^yLfnkxfJHk|m z+|OwO$0Gm4#8W}}J|_-YEoaVrPJ!yp^Q5fgAz$qGg}qY0-*_Vix>g@3_4WK_xko&< zgvF-QhfR(3`=mS!HgC}2AOEH^>dBY&$`JchbW&;eG^RpVYAbhA^2dsm?|b8w-p%X! zSv;b5czChnpVO7S*OiGzTXz_ubIETE)X@dEeU#bV&U!CjNg{OKzU%N`PVkyEF9lPd zC&QroWE4I}!*kR5cJ7=liOa^UG#-L~T-A{B#r~*XzW&s$B*@-S{NYHSIe4Q-6eZbj z7%T5f%z4;a*#gX>mu>wrV`Eboi`uZjDLpY$uYb7rG@Q}SAhn;O^LvRphskkH=`dyb zV+o;Z(OEub68+qnTUBnN&3>F`_EbkT)UJG%>g{QU*;$nW0cT}PZbjMY9LMZB=2}(t z((ODiNmPHrZMD@#w&y58dM&%PFq?9OhBF%F zq>3}PDBUVjz9W$qc{W-;fwFAZGfBpFuyQo$>S#Tg#D!kl_*6?+9b=w6VL8$&$)NhM zY;Q;twaRWR`!hQzN9=dDOsCxpThFo?b_RG9I%rglT6Wmj1PVKJax$5gWP1)RM{Vhb zj8-hiqO{*2Va0=)MyR+5UK*8T6_czW>j3-N7wA<*rRO)cMt5exSWKRd8s=~E=OZSr0-W3PbUjHSZ( zt0a9iq5$WST)F_`H#x={bWpP71=!F;=taExY#P;pUN z5iX9IliJ_LA!R@@JE(Y?xCdBC8?0n;#3{H0E=dZN&;HaPLFGs0b(|dt@2qwQvVf$oEfxs& z?R=`x)$5G&Vr@)ZUGTc-rYEKgu>6{W)_$P(ax_VnF_BqBd`>6gd7_u)P!UsHO z5G})DUHwg6-#D_%sC1u{4H1JcxO3tJ`}4wD#T}OzNe}zz!wyH?LG@!d$4ZB(AUIsU z0vVXvEQN(Ax<-_Of^8C8h>+$c5=>+b5gJ?}fToUoOh$$|j({4XoR+{5GYnK7fv;2` zGm{9Vbr+__R5E-3IK4_nclR!K19hC2v9000SUq}_=Yq{C(Q`waZk=96HL0no(jQ<{ zWZx;slNn1?G&d8J054w(M`bEO75O;o(UIZ&`}};(L3!k&!&$j`hklRvnlP+@8X7xD z$_*>1N^BiTg>6w@H&+&#`mGDTPeJMt_Z6JCKHU^5j4cP*3JOA)F=7e{t{(|5%pgJM z;>C*zPFNQlC#=we*F8+|;L}I~qOyPp%Osr$%PItZLV@ISd1WKTV#( z;(}8{KQc5HIh>B(=#MgFj%mAp9Rx_%YJ`%=#Z9A6ql*71g6uV47WRn}DqX57=O*+*Ji8gU3NEO!wU9hQS&iwy%92wQz8Lf0P20r5|J4D8Zwrd3}H1>a=3LDOE-OZRPzR4z-|#DW;<7uS11||Hq*kN#k7=l4-H(muOR35%jpg&@l`A zhl>lc<Fukt^-atQ7etGn(QwNGAwfcp@ang2Cby`&sa;O}X3>@1g?dD{^6B zpnmn|2{8)-C8ym8;e0}OlLRnBGWA=&C&)VjjjoFt#Ax;d3(o@s+SZTc=W+Dp>gL>M z=EM1n+IHDqdJ>F%7cz+#t(R0LJf{v}3Oe^kqtoiGAuk5zFjv*N_1pzdNbZx{5|_cZ zz%t)l2R5t|GBD=(!ufCwF`Q$9+)RW~3ZLCMljUufbmS@_$s~3{$AC~EFBRF%=a2$P zFups#jHPUx3L%Yl)NVm~1K-ynbC>5KVu7sg>WTOC0$QbX6Bsj7e@-kv0CtuuReBA3 z>;K!#F4%n9Kpt5$neTn!m$1v!l)!*;Y^h!Os6w|CyN_r_Eb|S8#0Ng9&c9PP^y@Jf zs{i9=WrhZ>dmXk?ABkao;>c%)A;!WFd#VnJ=48WSWKSPAz zGEEtK1Vf0K z7bbx$NE+||7|=V3c#hKPBl5c|`%@p~-x97rv#N@bqXVs};|o~#^|S2fl<;m^_nvD) zt>h{5toj7P66dIq^vOH++Nj`z$adIDi87XKHn*}|z`%RTi8OvLmmDJYD0*k5 zfV>LkmHVvj*#TaUHv6JK81e{Xf#WVD#vtc;ys)KvS5v$7bw^+<$9&ktn-vj zJovTG?!%>Tv#J_mYGgdy$J{gqTXhf>Q|>tZ>tvY6Bb!~iUYn!IGoxhCwdkFlFWdV! z3AqcK?){<)zkpllx)R!;l?Cm}!g{uV@yw!dbQ7Jls83*T{KSzkPr2e9Bltur+(P9? zT81K!T%betV#7k^l8S3PMMDx1gEME`nnpH8Lm!0B-oO^@`2I(VtWs74IOvl|CEqZg z4ckzhJSg8NwR~gI`j3=bm%cm27aCKTFqeCDCHN+E**DNfb(S$1-?rrtBl7%C@UQiS zDD@;xU(w8Z!FUumIa6X#f+iXc@@*@dUCgdJjUULLWZ0>Q-hem3+ZD*GUar`%u&+c;bgf^^>((vrM)EW-Uc@1l z>f#~A#$;5K8umd0E@cm!n_$9hd3r5z+s{R=hI}3&!O@7OlqcPMutt3w)L~Qgxm}#B zBF!iPM24I`%B6mFppSFBM+wnM$WJvGH`Z#{t0(M^?!i}o4T(bU)|fd`T0Q93sn7Mb z*@#EweapBC?*lO?|3Y@^m=_f+&KU9`C-H;;q20>HLUurV>|ul>E|uP><1lBO$>If+ zR@dtyPh3fmAtvDotbQ8C7lxy&O&RED)3xB;58HO_dE>J~wF*+CI%3$8wcbGo%J>sKTl}v|`Oy0FisKzsob4tHs97h=IFtzuh z8c8Zs-wvH2M#7I@ah-kS$U95NPS9LS6ovUb=bbOcI+b%H=4t`0kXd9wb{7e$O_^Tp{@XM6Ol=;td#tt&!JyQalH^rVlekG!)n;wg9M z;23qba}%_2mv)3mE|@LvSz53p=oPB(4Bi?4&czpasC?9~Sa08l?Q&(5lnUl*=Rk+? zCzFeAADJ8DzQcv;kpJcVb3pM2?w<~S)>HiK*SUB{)e(O=?bQ$;cA|XQ@e>H5{G~YK zBA#r@Imwh0o0Oj+{j<$W6)^ogFpX2Y$lyU##ye`TJ{aoQ7OhnUA~rE82>4XYY7z$7 zN%ujPFF=ppJ)raaTwFjsQYz^}n;fA#4;Z(jqA}|C!QVU5E%HOQmNG)V8JVW{*{}$i zs@t&Y?9PrBB`;nMz2cx}xufpiN9DS%)&{9{OqH#SNDG~-i22g5F8>^w(aK1CDZm}e z88g9ng+QU$;cYu_%?#EuUA$XbewkaopQ?EnYx(-g&)W*wB#$1C^RBJs?*;+t0d+DH zCZnBo8<7@lb2)MpCbHj(aboR5)9ORb=0hc&N1{5~7Wq)pwp3E%Z@rNp1@Aui^;i`# z!1cj$<4HlYi?%CRni-)GiA22zEiGplYeXf9mU5qI_~;aEe10jw%y96v!jct;=voh75{+w z-RcrCii7*Hb3-+k2B`A`?_Sa62N0jTZ`D0KI<#<5sO;CM{$;~q%Mjgun+)o+fg|Ew z=BKNwlfYc;nB2)df{A)9w6Mg~8*PvljInl74}HxiRcZQ*554~ko|A6n+LTV`qq7V9 zZArd!3e{7wADVKffO+I1!{XhQbl<7SL-aSL-!?D0t?oa4Mw=EiuR3dQnrhuvR^1j> z9cI+ltz^}0WHCCm)~tJgnH6HZpL$oM-&Cw$l^BkNYoBw|&NO@KU25Z7iuPfdu$~!% zajyU~fe2_LVU&DbE|YPwSFT7=4m8?T-`>eDerNpAWn>QDbDtLcubyl->BO4SZ(7W(d5``*_>GG+?Xeo-b)!;$U`>mMYgu0xx+vCgyI2l100H| z!TARe>2gHPwOb&JQedn-fo$CY=vLY{Pu12{tld6OS#>+vYYs$Vuv8H~f;tkvhE$V? zP?*;fh{E8QOvYnMh=k3Y8BHXfOvjPVVBxWpCddCCdSf0(nk9thnTE%?FKr+qNcvWv zA%-nvk{dGK=8gS?-Ht6eNXJW}UQFS_R0SWxV2$Fs_wgR^(i(W3ov1Bo;JtCHPcZ*Q zrK|gRinjn9XZkya%WK_cnQ;Y|EY7(;Y5c{ns-$dP@ElrmErBYKYuT4-xTl7Iwe((1l6(ciX$>Ysp8%+j{kBK0|6@~8 zh^F5cT&l7fk}N?{tYIx$r!Tj<(7*-q+2_0l8;X{xudnaV&Bw^kW66i6#2tRoLd&y~|^f;8=@GV4KywUFIa zn66q>yDq-zw$XW23^i}Qx`rC5<^5*W62(>f-n&#^C5EbIJy!Et%GoC{KilQw<7kPw zF0NzQI6>0p_&l`43XB{afh1H=NfQ)^q@#FvQl^9>OQMOaRZ=)|VC@jeMN=41DLBf% ztbk@aFd2XqP*Bpasm8pDvbO!p3M>Oyf#qi@E`4T5Iu1Sy&Hc%iQ^6@!A;KlVHewNU zp6L))o?k|-Rea>7&p>!9mbI|>(Z$Z4!gN*Bq^;EXj4Exg92W@6)7g ziWcO3n&M&%_Lw|r_Gt>TVL#Re{7tYj5)!_$KO zG}*#7+k*YQC~W4?c~2d!=iM@{qeu7dk{K*`YHAU@h^!I3fT+c3p-=R)@`nRmgV*s#>4OViFAaidoT3k2w+?|fs!#Mg zKevfmKe@I`_aD_BrPS{69*rwuF9QCK4O#p@oz1K7Loo$$!f4_5jtnRDCX&p#6OEZL zoA1+MK@;y5blJ?qwHzy2PC;UT!1)1(W{2Y zxOHpNxhivhRa1M(iw?>Y8MzPgt*poij#PC!VP+LnfjSB(R>BgnQczZ|P40!-c{voW ze8pF}4=UtX6~ozLI&rmFQnH>kRQ(E3u|YR~*O079loQh}L;nem>!~HBnPOj(Dmx)C zC1@<;V8moxi#O_w2`Gd>5)R3wmY*-Xe3P-^aGA#$kfb(JUy093!|23aI~CmAZE~uw z@2ag9dXUpXzprUahEQMsli88(K3o?afmIwq?cBdF@;JAbf2OX!30}d4T3^Lntn#Eu zK~7Qk22;o=KyW~85x(Wq%Pu*EWZV5zS>}ij7T=7LT*6~!)5>1??lDQxBnGSAz@*qL zByU!P*4eQ@elB6(=F)O4xfr$?l@qhdgvAW3AZ@P!>~zS6+cQ$%a)v;oN{BkM$&gfw zrkcR=PPRsP-Qt2^akf1fwOumkOD`kbP|B-*x6h+K&~Vpp@iZY8C^!hWmK_HMA-&xcJ^h_QtJ zw%}1lmp5>@G7^|fR3C{W!3ld}eb9IBHgXWIy)jIAV`gBy+g`BKS_RH;S`-a0Ro*^E z#Y+Q3xGqHJ1n_YXV-(_UH!eJUP0^Bd&8pe!jR%G&U%lW+j{QSJ(dJ%E5|>m)bomRr zi8I>xWEsj~6D7&Cw1ox9Eox&ES(Pi+FA%vtvBe_^;@_>d8K)IL5WMX6@wV-Lc=KRJ z_;+pFz3uw_*t&w(dxQucm~i~lzI)MoitmQ#z5QRt_t-oCJMX~$);|^XcYL4se;eN` zvQ!ZKlF<~eX+1*Z!XGX<6bu@p+{;XAWhUzy=bRGw3*v)^I2p(jWfBHRPPrGxeu3>E z3uFdk?y>rgmvaIcUlKEp((VNiodz(u>UyDS!eS&>H|IXG9LQ(T{gCBlAkElyEgOH* zdPZZ$ci|MKr1x|(KCadpEUaeI@uZDQpilbFuQCsU zOYre7vCAAUJ`*Sne{Ve34?xkOl~kXV;ZJ_~)zY)&g$lhc_qXm`DqFp&P<{qqlOM)- zpXwz=p7#}efMNbiZs5hvEIRot7H7Fh3SxgpSgHZ@fXHAd$Y4%&;QSyKCTeo(w(7)EiEk%JrPe9r zmf&K9T}$$m!vQPJ{dy+U&j}|FWfLzi1Ln6>CAhyt<;#H&C>}q_xr_pJZ*S+NcJSDw zdbozs@k?nrM5}64Np5ZgOV0AO9UC7iXB;9T37kR?CmqJ;%aNKj^E-=V1Vzz$$+4S- zjtuf_mMDX@{~5u0pm3>0l4ggTR~=2bLtpulj&w{g=4Hcy%3OT5fFMX7pe262TJQ$Mmw*dA^6?M_HLOeTe4FV&5E%dI z%V@~@7(Up4S!($*pyvwBA4)krg+-`Dds#&Gxnw=R{YT=NUyY6+{oKj*Bi+ia7^q;q zFU3c_GyIIUC*M!~9jF8mCY3*((&RcQyD(*N&NX=HrSUxwvGO?39ZUz(mhsOXGj^w{ zXB_}f-+p##-mo-9&8*$JM3W)GTWI9RY2z1Hex-8b$>fSv{j zgW$or%`r<2GT(|7C<~7iSRh^Y167G;_9uk|(ZtcP zHE0!TLE<1)!U;~74uW6++;r2eL*eyylDKT6(gC+z^^AlVJm1Y^o#{@0&HJe$dg|Fl z;IPhdQg~OxifF6}hJ?=fD1^?tpP>do%h7*;AAK;ug=LaFAft)CR)eg+*l?yXXMKsd_5|kuJ}m~ z$6(@77*!3-P+|a6mcZuz!I%WPVRFlZf7V|*&jk7jxX$+}deBe6bw2I_t`p|- z*A06@bBYBjQMuKZ9U%o{g0nJVIdHN(EV=5A#2kb)0k08mm%N3tU; zQ&$fKNErx}0;i%#f>KWrZy{3qir~tSTcZ6Mg_Kn;L-yzPj0~)SXsW|+>Wxo@RS7?w zUc#K#A~IvNPU{+|foQf<&Sf8|0l=OP-?4QF5gM)4O5uYQkUVuD1S8+zf(;F* zk+Q6c_KLDs8h9y>xB>XH5jLh+kO&r?i@<}T1Oi7wkV=ulOVCE!5J|+`?!zBFqcx%t zlzA-d4clO?mytsC@Pq;NdI{e2p-QLr7o~YC2*OYNqS}%^L?CErCvSbZNIe z<7j&}Us@yi3LZHke-;WpLM%)*Ge-0(-NTAJQVTb-MdS&_Jy+p%Ho%>tA;Ss(9{n`UnI;Vm|AVkA#@MsAfe53mL@qlC z_TC*wMq8>SF+&XraQe2#!y^kE8AXSU%Bz#-pr?IsE^s3D%K|e^1wmO%(*rYG2O-StBy-tAGyh_}#dnCD4O@E@ zKS?bEa_;h0xN{!tI$FOYD!v zA9Tl=GN-ZZrbd9%u_-)Hs0jJN6Z-+Itny9sL1h@4p`ZD1HY4^bYgiyUiP@@QE$lc^ zyNV*n*{#t^66Cih5>eF{Y;+dP3jisr`#nT%g&PxFyt&kB%aDmxFma@zs!1ZX`Bu<5 zD9NkV!uRy=BvRGZ4I9vU8HVl{Kkig6SGn1t6r`l2+P-#x+6Fh5wIt8}gB<&=tV~RI z74Y%`6fdZ4Au<2285ig!Ax={pXz*Et&@q@TgUfe0Mmg_;MP!$)zhVLxaJ%g3gB*uP z^6)qn_fI-C8_8z+XIKdqsjl0B9o$9+Q!3V<=_xVU%eO^1$sI#c<_$0>@V@78wo;tK zV2woN#$NHfoX8SCK3(mkH}(Bn@DT)*2C1jD8R^hDTX$M20;O1uMzU__9*e1MRk+&N zs;VhfeODvjqG{TMDB8)%RaSpa0U=-@K2U$kH%u>!6J_JtIFr-)zO*UPQcJ8n{1J0Gm`H z(XG8!KR5Lc0aaw@!^L102=wX z{mpH=_pRHX z(KztJI;7q)-3N0o%dek^vs`!(_M52F<@=OyNX=CH5w~r!JwePZ6$GYYnkUq_g=${~ zhk`O(_@o{&AbDr1G)vX6tfOGj3;(|ba_|ILdxlQ#RY^Xw|Da8W?YSq>!0$R1(Ym#! z@`SsBAuJ)siDVoW{S_2H++3=bdbi%|fK_1|Wp|QB#6c_)O(`JrdO@QV!4KSbhY;Oq zCYVisL~+W0@vSZr8B|3d>;digS4-Fhc&aJty1ReVd?b{crrD1@< z!_$)FE*`5?59}B!PBkT)cy4Z(^P4x~@y(fr-ja`jrls5nTESQ?5_cLkev$cS8#$Qc zM3&8^;yn={It3C2zMiAnkLg6&<;}D=a#Ur3tX_45F42hDnU;USh<@Y|^rgcE3CVC;R|%<dL*J? z>CPw9W9Jn7AJ}>0{VpjdJBq!Y=W#+p_9|zlDpGf>c)xNtg9}o|T~f_ARIPF(q$YZg z>hAl_#vhZU?oM{3{no5M89IVXaEN6nDpCBebC0z2L3@wljJzaDJp?nkuIjnvYbAlr zGh_g(v|yP6t=HS^ml7NpJ3+nsmK3iYJ%H+(SF}5(Zf;yif1>!8lcZJ~EnU4EX-bhu z{)VX&ZKHrzYiRD#dj%=xw1|OsF2t2Wq4|!n;{jUBQ&Z#?g2}-X4Mm_;wbfl{I>y;mv=J})ZrRjFt-t+t|AFl3LZou2dH@}6XXlH z8G;RjFYE8zxaWbjf^dW%6ljKekK}@2hd;lNSynrfvl+0rM^$*+yGfogKx|~k2k3&T z^3r~7uxgk(R7rW^I9C1s1;L(*QraM?3KE#Da#sv*w`udU#7ZYPyA7XX#bQvbzYN3( zCZ{kMG+U1(EC-B#rtYKGYU2iaem=Ye1pZMI=kZNx^#Sg3gLu8U71cG^M`dvas(COpH;(oWAony{>D%XVf_XQW9d>NHI!3pBR_ zwa!|PP$v(k$7t%?$KrI;!-x$sGEEF&YGAK*K(Boraxf|w{RlOUNBS~dirF-}|9G?1m-z>k8(bfsyB zV4}`eWTylcT*VVR%su}3`4Rl%yN=mE*8t;WKj3BXj!yUvgX9a2r0>jNak3X;olPC!bz}E zDAwzaONk&|;0}ti!%{hFIIj(8wp3GmP_v&hnbV+^zlW{EXEt@f3u;oCK~2)SAGR2C zN)0KRi8A4q>WDd{Q1@95DPqShI;2dPa!#^VOQ|RIW5Jh0mHk#>4Cy6c3r8(3AlJjs zj!z{$g*xIRxLd-ebbKcW zsp^10lU5;;W-qAM14^WrK$Dt&!b>m{p1lQ?baZXCnz>I%7lHhbBX;=nq+yp$~HXe7gZG5`O2+l^!+9Y{=F=KqnNf0#GBt5db z<{Nu^r5fO0X9*CdYn`F~)_8F3#hX~u3gi{V{|(+q_!YKO@S3nYV($WIignK)I568| z!_3OkA?_L(N$!p(o^&M`GiS1lrA33%vwgphQ|woOnV$lvUxvTx7Xrpm1)zQl?5(3l zDijRangAPX1rP>GVS3Ag!&G5i=+4EtDllQG%gE9)($aXBs&)qEIkP}ix4@$dp`&U7 zRZ~T|bTIfVc*{Xlvq`dn|k`9o8pZs7ZCr%4qbN*%ZRDesD9UI zky5V@ieD8};R#d%C|4dS#&Wpi3Gs`4px<)hsx&Iq3jR-M-O(}vnaPYPMiO2Zh18${XAgxG*dM@GyafrNcpQv)A7v6$BI@HZF-HKIv2OX z@3(m6-CiWTeo*xJuBS>*NkCWslly)<{n|0PQ|46RrHv%0me`w#GD z_kRQ4OpFl$SF&3PleEr*##BD1j}gv#bqF)n+^#1 zfd6L<+c2}<|7O^-Xa6#68ZpvPln&3>q;0q@bA^cF>m#_u1)nbtA^(QhAcHz`7E@&s zx?$siNfFkIZR~CZz~Xd{rnd={&5B`tyKC!8 z--|xn|M3Jgvhv&Hel50=Y;BdaAR=JvkU&A}HuPBtjrrcWeU{ z2=kol|KydhO*iljG0e8Dc1!CAxjNN#{6^e5$GC57;EUeU3q#wGPKYXMb~x_}o(9zQ z(NFz?4`A2`q5jWC7KbpPkwt0^$Z*l~zhHRX>#9Kh+DkXP@U764J(m7q!H{6VoazAF zflSOvRQQdRaV64d=ZHl%sbm#Cl7#Iu3N)ksb4>$=^E2=92hXLWkB>c;S2U%#`=WAZ zj62kKKh%|metHiN3$mLy>{3101DJTFv~1#)h0MGA7sADhx!U%YH&z?Y;bFu#xEyNc z>3>sfYoh>)ZAK$6EKY&iUmQ)Mc^aZcYp<*QSfE-Je8Z@@1mYChyda^UoosMln$t&K z%%R>J=p8_gof6>T1?Z7^$u<7T7Q%Rk^*AIRrNKJL*^yOI7`(QUc}@)Oc-*Rm7Sw5q zLSgvhH?GzSiyKu@^shkJoc#HB)`aWG~;R$c8UQe3}?pUlV z#5+22L#;OWzDqiA?g3e0*8x|G1L;UsmUp|ZwA~34Fm!L=aHQ@mYuc^}u_U6Qs2G^V z)nN70b_$#9LAB;<36#nvSoERN9k$n*2({t}?S4)4&8XS&&v8G7R`XAAls!}Un@4KG3STSG=;1+viL8HDVl5#q_uM8fQYYZjcHhm_xS@A&jh zKLJmsw|l-Qe!9KcPFdvNGNmlA!-PNXyYeU12bJpABL_6f%>R&NGGf0AW5mAc!5||n z@`sS@`ytqeZ-C}=nIrecePJ%*1g&EUg9NEyToDct9!S0yzY28UK8ag%ysL8_#9jC6 zsmy=IOXW|M*|k4h|A4Wtk@}&%rwnP)zO4lpJEAk;Rs4ve&b5C2rn>TdQEF-V`>Yk` z^sM}`Uz7tqUa&oJD>(m%6s>M)g>%M=V@j>I%(8n-Dm;d3Tmu_IgH~L9=1{eg=Y_7W zbe^v(s9EcWCA*h2*|F{)la(?zCsoYxZf^fWny$vYhtj;I(zIsNo;9o5A65^srhlOV z+mr!2764Sx1!%EB?U*Ptf$qS0uz36e~BNxb@ zQ=(@X9e(0K{Ghp!)&w8P?r#g`i?AcKN_Kl|(3&ddidG5!iguY9qE=#rs7t00O zzHL;mitxz^6TF5Aq^2yZj0oTz6bQ*FEGV~LMrp|y6r@dAy1FhDxdsGNDCnAES|Ot3 zpQdB3Ue2+YIfS3T&Us)_Pb!6M3OVJqa;rT{%EeI8TB#sfYoxqvWn?JXwiC$KTQJ44 zvC$?~PSQdb*h<0L6QVq*KXUkZQHhO+qP}Hr!j3$yJyE@BaFW*zGr*HXpT z*5fK5p03|5Acfo|3J~TkMB0tv-yfhjyKFdtDqtE;5rFV1tHhguPNf{52!Qb4{@H_Q zjVc=YFLosq2TVB>;SB1h>Bi@GH;S@}9BR`XR0G7O-^ZKPeo%|6!>{KTbQSMPMzEsb zR2Lqsxld12A#}J0RdqL2`R%9W%#ghwt%g{LUn#ph{Ez5^;uIUJf`-Mz7f|t!>sWQZ zJFU@m^B3Dc*L4V$$xS33NsIe;0lJbAwScfot{#Q)YgpsoJyaC!0ZJ@^zFTONGs**0 zmKhLHDl8FTED>UIG? z2lInl1|C>hcg}C~Qq(`wu>o=}YV-{D+@D~L(@(HwuLO?Loa;Zqn%7pYIEf*S#k9r* zc=mL?ncvyjPwv2&&H$!QHR;(~)y9ciRrdaf{Hp(&LdF%xcI;2XRf{09 z_w1Ag7_kC`^&$*`A`F>6Lk@L%W@3r4;QD~z`U-CSnejr^W}V;~C!FAZj$>N!KgVI7 z+}dIw{M8bt-0Xw-3t!q^4Ag}UL4w+6YoE6m&SFx5S>H%888T3qF6yG}rRqQByy3o( zhx%mdz*y(MhN3le(x~Esqdi?Yx-(qvdp}GUISN;dZ5QX>QjO|-XuU<vV_QL_td<`I|to7zD&P^_Oqzq&-#{Df6d=8 zNuQ*9t1T%P(HfcV`3h5AGmwG|&}NXsVcQ2E1^EAqC80}QWmNwpTieiHSn4Vyd!wOo zNrAqtS|gc0v6VwJ-nEM4whE7s&>1=|^2_@!SE-o{yDm8A|7Qwk->MLDH~`KS*6Lt2 zc8KJ$jGZ%AtDi%Brh1&t-_8*Fq$4XXwV9wSpW+5$IoYjE)uEThf?Gq;qU>;Z;(W}J z(Bo)+{=njWzJtGfGc_QmM)tC>aGZ>dwxQE%lXiC0vx>sa9x@OVEM*xuW-+7$F*=L&FlvD+9TTUmSexD0u&cMaSDiQ?Mb+Y`St zNm^AM)DbA|ALrdMprj}h_{-ftA0$4Im z`G@{oP;L%x^B#C}Yt{@v^a(Xx01zI;yyFspO6A(AP3U&SL*cbi1+qDxQJr6qTD90m z5G6{q;-4N2=AXkwNirR(j^7gY1FHR>Xsk@v+pe$ljQ4YcAGE>)ihfp`{XiN5VyflL zyr%!t5TGlr!WO1`C+nEN3j5-<*9I{cz!MO_%y3br@VuL%0#tSlS|Ps2 z4bQxe1(4a19cG;OKJ3>Pb8%PV2otNEIM6U2s!_nXR}`z=h$qNkZNqOqiNR>k-Y7!> z_aebsWZdX#$|zmP=$8%PdIzGG@@27G%x8sVvamCrZYbEZ_?yW7XAEKflXhA-DkM?IsUWdd(CXSVQ)xoq= z)1Cbv;5363iztV5F%$o&m&L&z*NjKnhqzO0Ua|F+~e@sEQ z*c5(xgU>Z}AsJHk!CgLowAnO1_Q{9t$Z*;BQ${6Y!UAd2tB+%g{x+EFMC(`8iAX`k zpX>;KZuiGGS^4e#2bdw>NS(+6{?NML$2eBqJa?ulkFeC^pFUm zP2jD#50K~P%lZtsy*(XW1&p+2PXU^i6$$A~YH}2f7FKHebu62a**MhAwl#p0GHTFQ z;od9ERau6asS;vOJB;eS=slEYzY&~~0Z>O@6~$7H!T$BSmTeh!fU++^V%=;vK>K%~ z8yi>l72wOX>HC37v+3K=D~(2kW?m*@8)Nl}I6t8EL3ya)ciH5ifF&x|#qX02DpQ&t zkJABjbB)P0^l0wyfBagLz4A3layN5N*6&HyQKsu46Yv@(TATO5kkK;T=;XVR4(;LF zE{Wbfb!j(OzG(6Mf=>Z^!e-Ht#6bIBaN=c?nD<@lCMQdcMkdwU%VxN666eIFzH()} zD>b$7nPhX6pJ)ha`S2Rr-Y!7 zzR4ItOsz3Jah~NB=0kj4zgI@0X!^1Rg>yF1wQxYVTBXl}E!Y*USZ)(GWwA^+wSlzd zZ}~r$Ge@-VwQq+&Nwc`b8sj2yYOdN*buOXj{a+QDY5lS^J3OkPXEPeozOX$oLpIo7 zZ}U%*HWyWHlJoXHv4(^2l`O z6y$H)Q67%JpQC%DsKt4r*z{qNamL8-Z^tZl+!3$1+1hf z(p}leut>>K`03#iKfD`~R9zb|9)ipvu~4{T?eNB$!JS_5l~H(um+ALim@V)DK`P<8 zF5bHv?YfO8Ad1=JSDdbE?%vRr0U$uI!ApB;HBV76Wfd{bfM*erT*gz$~cah5!PAXPWiqa^{fdUT+x+XQbnE;ZDn zKjub5%f`(u4g|-50OMA-Jvu>Di3d^|IUi?$64D?+Zo%}8KsH&R!Y$X^1Z8SgqrRwz z=2II2!k`)gkkOaZ+co{GXPx`!#z?bzMUz_it5ZxtDAuciP$iTSdP$K2QaYxajII_8 zPwJNJI-t&_VUOJ{ocq+XW4C?-lD}NxzJ3EVa!ZF<>bf^slbpT%!59e$$#dd0PafKG zk)9$rA$3{Q<12sJm-#(6c4ia)D~J`Wi!61OX7Px4zIplp6J!EWlHunLP}Sx#awOki zS5A7FB-B;Dwcv14H}qRWm>pg8W6P7 z+xY#i;?raQ^<;kL?*;d)qKbJGHz@N02zp*2-0%flT)yl!;e+J|zEI|q(tq>y9n1)1 z`3-Y?*+pJH945~6J(0cNB+ldszRFRi&bYaJy+Ooxer8qSBkI-EOj=AeBM@E|L zVYM6oBE1E;*BY32hY6Q|xl5{D8hl*%&#qksBa7Zv-D*=yc}YQ}mFKbZ=l?nT+$o<1 zN6?~r4)#aoL4_AA-~r?xw7mi5oL>eff2su?uP@0mDD*%5I;u`P*y_sHHz$VwFcRl1 z!SyYmY^N|DnY>-$8*&!phIrLC{~9Pzw^tral(zM4MkJdq0HF?bj+X#a zJ(v^mHs%@h5_{DME23s1^F?I>F&|@8{Zhcpq>vzfOBw>ai6SFKY6%AnuOH}A)pH!F zNcO~r+lb~>wyiyc_E=(5N(V3vM)_a-wt~urDf=1`)qUIOHt~iy?lQ#{_7X#tPvUcm zpU@3umNm9Jg!oE=2*FRp^VNa7NH}$}S|!>-9JE1yUQ9qgDSR-G7N%_(t#4u)+?PYOXZmXN`ptle zuBejRca|*|w^lgynp|u5Baz4Z2+WOsBE>y9Tc!4*)B1}DSsis5QeLl~)87H&v}(t2 z1O(%10i{mQDOAvp&L{0FV-#B?`VZPZl6^`j1pDbQwifL=RSh93 zbjrEUBI-pi$gBzji^7@Id+H{(`f~jn8jqdnEG8*6OzT@+uSI244QU#Cz*hrILJ4KU zvEita+GH64pi?E4UK}y3lAP9bxwpt}_lR!so2&(y9O*MEBa}HDQUsJz^Reh(fl9dT z+2)KTR0-6Mc!;T-H#_V&QuAW*Zl@6O`DS7zbC$24p6-agv5l(Y9p#3w{pOF7tM7Z-nwVr^7# z{GxcswTUY%Z+4{S1B|jMhWmoEa?nM(^0LH@mD_O%BS8N1%r>+9<#~ki48@A5d+w+o zT0SWf+g&}3lR6gSgTZC6>>qR!TCIh;%;9tvxE0Qp=pBg*h1}nBa-a)%87)&D=dw9y zlz^>jv2Z`M-zEYeGc*IQY*s+Y1BiDz%Fg4N=+4L53&+XFg1XG`NIiTbgbg3AK_NHE z7em}YPn>FW7_5gRSY#c9>w-f)>mX;DbV7x;8lAwK6S#cIAI#vOrDvUZN#xl~nP7ej z8^&|i-HQ0u6Ei!G%jYS6nO1jin^iix8rMtz{*t}4X-u1Ql!RmB7iJC9Z`-WamTqQP z&znf3*Jv}G!r(Th-O1rnsCP(aEt3(OuDWTI<4NK*L2QArw&W`)H}b+~xW(Qm8u^go zwEhYp$E$xt#jRtNWmQhN^@faF_h5ZD2)ax%4)9cfmTck6IYe{h$h%B)QIl^M^%Sil zrjcF)LBC-e#2xB9#`OGmrO2G`{Xj~6cpj%nZP<1-rxeDe<-1l$($>~%qm;t(?kg@U zy!b}!Ofx<*B!U+aaYSp;l5!tSYPO#eEtq3%oZCL!B4qn{Tlo;-^l3q-u-m!N5MOC^ zNtgU-vk$H|T9B-)u&_i-PtV5pvA2xIG6E7UIX1b_+2RkwEIKVd%9N$Iu6h{8K#}T+ zD$=y!=$K4DJGk=Ptn&mu@fS2Q7Q|%rSH!?A_x5 zyJ;v+p6pvj0CY|Y5$^6Ki6G?nd4m1wX)w4$Q1W&JlI~5{Hh-tPHqEd+4~+lNp7A`C z*8`sY4)K%X%nvqb(Y-x%X^aq?q~#cHXA>>Nbzg7r`SOB+PoA9QaxdnLVXW2Zpc$oW z@>3rztIVRM(9ctqdAgXlgyK0#1Iz7Od}7h;X&aduE~mI3KTp+KB`-XNX1&z$nGy{_zOrn#cuM zrWQ$LUDgVTld^gUsMg+hS&Va~?%abvVwZtDrh!sVZ7s9!HeO;-Yc?{!{=^y3esnmO zpAYFW**dTtqVFgOb#A4WzbkCdBMe}IQwXQl1*eSPA`)HSgCnxK=(e>>)T9k2FyR}K zkKr0S92s|_wjK$~;v!Pdju6izf^32#2&Q5<=>hnUILs0FkNo6m==gBth-^!>Bon2w zFTlQ3yR#4G&dey$1V(EH?Ex0XO#xL%k+P~Cs+&WRwMa0GExGaC4V zh_RGV*zVAr>uimLGE_dG*oF`%A{9${ zCi!FI)wmXKpgaa!1EhAOF4;ss*!mT!&^>e}u)U8jzrmg}X2PJ=@YR7P`R5DN_g7|( z&7QDmDo^SWOX&yD8kn>684My$h6BZfxeR3lT=D`yl}0b1$9QcVn?e8Ys1-mojH-&} zFm8}!D819{EClvUr0fAaKV7Tju*sWF$L+NJ@wlfJ*|qS-l;v%>^jVz;I7488&fa}E z7HwhRkP_^7hpd@uO8dj~<|5rMGTKUgy@BtW;ST_4&(b5@>ia`Jg5?V!A5QKQAlY`E z$q%SA+2Xv7yk)oZ=v)?FE{WrwV*jl+`<6NgPg^zDC7wKcZoT2%~ z;9FxIj+Hdjft$occ>l<;38AynwSIJ5TDfKvy;w?scUIjFG7uZ$)cj6K2Y76nMe)(R z2(TGg2q{Ew4fr=5RMP~A3|)K3Pxzob{w$YlBN+L1+=Xt);e@lBq;4p(nUroY9Q+sb z<>>aj66^C3xcZwH@GsEIWM5EV{OR>sFptfn)|Yh9m=W>LD@zWr9ec9jMAkaFBCHlq z`09u=e5*pHSv^uiBv3MQ%kD7;#(jpt4D=1R-OA7f%wbNYjSo&T>(SBbhud_**c30# zxWG%Y+zW287GL5tI(9wy@ux8#p^KO^iQ93zX986m2GvRIm2p4*gkf-77G03_kLZxO z<&%!Mxy(@ZU>d}y^kBtd{1jO4; zAa>bOz&L*?0vD&9T%iGB1>qjHU;TDtKZEQ1q#l2UZ}X+j?E?f$d*yWR4xm9w*MTxY z4SI?*k}gKGm_mgCQ=r}szw9t2vh)(=ONjpx0b$sQ>8#p8e?N=%cK~^hDH03=#@u&q ziYd*WWTjVUJrz^sQ>M79Or^^AIQBa+c7br|;YHle3qB|=9w^2K8W!eL_L=l#0s9Rg zx!zJtb8)UMhf+xUiPx?u3QLicLtVhKS-uHxy?1LI^W@;?`G~Qsm>S5R1cp@GJgs*L zm!nNE8dPS^z%4!Wd^}pYe)QpI1^C9%p1=OC^XwKCYbE39?*r+~b==5oh>puw z{Ml&Jq!8Jc^9=T2%>@HGSTqNV%mC7laZFq(>+~OEZ9iD)|7)xzNPDbQf5wdRCuOXN z@-kB#3K68B-wEet5D9tHw~W!NGgl-&5RPOG+Yh1Sv=E@lb9etASZwNVu;g5yTVDA1jver4n!L@RFxX?Z+Q=7Wc>ys_s97EHgqOq^8^DUGPBMKk##QBcI9?1`= zF4S%?BZ{%!4*}PSDEm&{_K!&2NQ3AHQ$%( zW_vs3&zpP!aEUxu`xl)Q!iLm2e9!nC$($@cKYL4`X&YbkpPdtN(m|j>%}Z<#Yk{!1 zK+q~_blUfKi~EH zK31$VW)}4v33^k1+!`Ncs;Gxj${#T^&2{XFOUo(=K6NKt>>eTz289im3YJK6G8iwM z@X;8Y)SHN8VvWa;QF=1yI~_Na@B8~kdN09r8e$S6CR+P(UWO;H*rsKqa#0dpotjbZ zK?hshV+t~u1*vBLSf)tl0ZFZyxK2#tN*@SE{w$HCC!+Z7&)65l9K&uonXWp+9No3J z9FjMX*Y|&KPnln@E%I(&zt&1Y8QQHZc|9BP;l>r@E!6;e~68?(CtpPM`Dw_B&ZbpGv|g(qO_dDg7+F1spIGB&14TT z3FRAv+~syY(n|&Ge^pm(k#*yjne7#%rk>uFk;2wSc!cm`pBnA@fM+`&P>JCA8D4i? zokla&tl3S#wOK}X(iM;Me9`X1*_)9!SD$xSrzIGOL5zbUJ%cI->S>!ET=(wK-L8H) z#*E>DSE$soOpnqRin3qBjQ7PB)QT#_@pVy|@-t@QVef+&MTU zXk10usAhVKo=u$QV0XwaHKW^ot9fS;AzU&$PXL{0r1fFwRQ4w5WWK1MCR)!qN6)SX#4$|^hSD6{eL;uu1qn*xHUsY2(@EY8?yq<7 z*x})XGGf-CEaz0Vrcl+$`Ny)cK%Rc*FD?Wro#Yg~+P};KLwaLnjfN0G{2~z2)vxAf z5x!x-(F)fT%UFH`U#zcn zn{T7bDeRwn|JoU=^3H;uO~Z}e1x{{+v;rSqPNn!7V%fB})NN|HnQO{!+k$lcqln7a zJcDuv5RJl4w21f~u=Mx@Og}$dh2Fmbu0r2G0l)pq1_7AI&(ETOrwh}YPjCEB09q*E zs>mk@`8xc_0J25!Y4^F`ZHKOQOKH&Wy14W5fTM$j##RuQW1^uDppF0|o+r;vDm8w- z*|t>%q-&%xh3uZ5YI5zbJr2}lTWs1};eqaNM%L7b*Xopz4IL7iY(OmDWNqn->mSL{ zd%P-z3bns2CiscC{)&0ycx1XNk_A04T>E$#vcA5i%|-I*d~^00r_KcPhUb8!d9RE@ zfVWwWVr&r+Y7&w)AA?}bvSjNRhCYjD90L=gXT~IBB|ZPaVc$!Cg&!#UND zIm#(epabW4#sQ9fdL)mYLJDQjEdD;2F%NeBf$=%a4BJHk-7hMar3OVR3?jl9WmU|& z1(_7z5@`L2atZ=FuByE%@w|~3t?^V5SK0@B$$hYo0XT|cDu%Hr4wG^O>W<%b3LZr- zSza1=(sk@6;qpt&;xktRFD^qYAb4sZxc;LS4-2Iaz0yWkyJ8|#iXDl4o5npSQ*Qto z+CZ)8_ATW$HpESbRH*)7rDvv!nG_yiYWbk^-N%(9<$f#mYebi0*KsZCxw7r*1_wcf z>YHs9sv3?)nMQsPf_*?V4Zf9{1eW##vg?-=oitcEI@`dUxGr)VQF0AX7~Q{nXn{Fc zlHn+4ni1WKe2`QeQDk8fm%=uzB+K3d0dq32s+SJ7awZz1yVZD}h4-oVcJ$nf3msrBL(ja{CwH)f$!hEC zHVxKFetWWv?2p0C9D?G*ik(;TUL;zaLH4jyX>|`e*uNO7CVzu^3T9Gau@nsGFHBzrmw&mgy-kH^q==<8^F%bPyH1j zP{mOYId|h4+wnxq*0a**Oz!=nwyZ4N(cE6Aov870ip76pSmy?Q2_YSuySz zGveAPB2uUG^Y-UquVX%(IJpGrLWST7B>{8jFe`^ZG@iZTBLYB)5zHliOJ?{R!qEP# zP&H z>N0nf8aCRBvMrVtb?ZYk#UaEVg{rK|5-%$(Nwv3^0oy|pK``_R^EC31E>2|%oN=3q zoN+jzBGwgNkURijON2h$10108T7IUWA7syZg3ETnS>&A>hZ@>Um$=uoAfxoP~RX8vXy zBnnN?=UP)xmn43ibmA2ii+XW4Y4f-3o=R@7)4ad5gKC1Yml zKz2BY1%yuy#W!`JitHkg)^0KmqY##Cw^(ll zoVhv7BG$M`hdgg;LD)o26Dxk*`Jy^8nFPfXMP(g}FO~i!YDnFYzdNIx2Qm z*+U8uy2yq}?KX`%gBj`hF~}g%KE^JUjo^-VPi^0u^{YoBnYbSy7y2h$=Q#80WA&1k_u=Dh?w;&dpf9wG>+pqpH=L(zw+6m(wKewaYBl(&KwJ-*`!rFHreyO3d3N&1idLVSWuo^LJ9b* z1Zz2z13Z(w(ouE&J8d^-C|g6#FK+R9;cjlK!hR6^>o;q7aabgmx*h}$ua!qZB|Fs( zSJ(QQlp6$pHg=| zDkU_u53{)<9&VsM81CzNbA$3KdAn}OXCr9j6Cz@dGVv>cAwAOJiF!r&cGxVdv&-^M zC_0vfC*!dRzi>KnfX}O%La0+&6_7*LUYTPR~n-()&s1lTrJ+>25@)n|sVv3iJdOwh+r5uAX z6f&mLh4TNXS31Q_k7}~&T*d#kA{v$9TplmR`Nb`t8gtsr(j3RMIHa6C8J8Fxb)NOy zw|zIS`uJ`tewf*IAF2IOoN2_)W9N~}y^qWLdu9N%&e!CCZNf)z6;)}(p>u=-3La<&BW&c{Q+^Z zEX-dO(nRIzt&ZOy65~)4xL7n%Am5IvBv}L)NwOBwLb=y8gQ;gd<&wUcMPqIqD#?GN zfK4T9$W`t87BS?eo_maKB}(A#)QAP@F9X6o;vGGGT=|6gGyj=(>rmsd*#qD#` z_qv-MZ9a~9gkN-^7!Icnb1FZuhO(P2?vFwz-`fn#(2EDz+ zocj52zi9qs~w~_eyXZI4g6qi^wX9w zGC0wH#;u|Ju!|VOs6j}<-L9rE-@f*|U%ye;;TH=M7je5X7p}u9Up25Y2?xBJlku>64?f$o(t}8DB53#jSOUdtAp)uvFAJJd- zr*AtK4})K_q8|Pmj?g^kY30cKa{rDwg?{Ua&&6P3!>y3zNXD1W8W!s!>vWrkGeh76 z;?Kc9;@rw0knU}`E8nVtCYQZhO6-a#0D4t1bKQ9{!9ZG5aiA;OY z979T2gIZ&JV}4w1K;x=UuE+t(0m^=$rg#5qH-@^-Vp>O*oNkv|RV-41W7tiDhDld7!SNf7sf?8IHcAqhLcQi34{qekw37q)Wuoh%^%MXjTo7;__gYteT-4^o;#U{4lMlD*H;X_pHEZqj8EMU$bLrp+$XA0 z>X1*JUchQiXtDG(x4dculHaO|;`RvYGi*=~IY;X@zR4A{;C39M+Zts?bf@InahqOG z0B9zJ6laJs5eT}MUC(0w+?^7plw@2-pPP~iX;sZXbl*<12}EDYV)RzRGFya!_?qu| zULs3SD{Pe`_o5$L@sAO_o*YxDN_lO|(yP2%`4VocIO58T=eWQviQ6*-o~9M#*}WhJDt;w>y%&M?o@v2X$zULxE6m^gjC zG5Ri@!(|-g=mAb2rr=`e^&Hk*#kh?6wx?tY!UzTV71>R(&vOcxXh@&ihZH#2v{jZ%JJfyC{3svw5>0GwY(`wj9%YXxN&FrY@h^FU0J_InC8m;_CPJ0Lf5#c!Jmi*ezN#6rgr#dp5`a&UIU zC+r#+OB!c(-x+`VU({sABi_F*{6sJcm(MY}z1!m}(@N%7_~JF}8&MJ&9DaM5vU;|A zvM_t9j;$_d6Pcowfqjl6_f!+i<%T)f$yPJ-uZ6LZ2j^%-IpYYE{1VyZ8ZT_dx4Xsa3c2 zyXf^LC2ht39)^=^R@HjaeU5}!%$wk3M8bSVf7J47@qqy0-$Dk|b0=x^3h?o5=fK~o zt&UI=y+RTsD4yLMQuvj$hdbsKYApmR1nlbtiRTnb8Lyn-_!A`frhF?n#Oh)ZxpTH> zhObE&BiV$h3!rRK%*Ss>@sH-Rr#UIPMyfkOx|$sycNS{(9K(4|W7`#+)HH%GjBLsk z{(SgPRTZ*Q(%{#dl>TC&|r#T zCKucYi#J!FmUf3COSV83O$F^p9G--K_zOI=Kh9tR6z{P)_$}IZGDID}ZmYK7$kCc| zkCF%P=dl=d$D4o|^Gy3wjk9&Kgs?dyN{%=h+n;1XdbR*)dmPc*N<6bMmB{Vs`ycbD*%S`^@KT_-SWulXe#Pb_ z5djpU=~P*|u{0d_a{6A__$G4Q+3#Cf`PB+m6&WGD-kgA^c-9%a)x`^F40n-nG4I#u zvjXZDH%-mhTqz0Ts6kOW7RbVm^t3(GOp%(H^Kw^gG}6p5A4&I~EyywH60LQDG~}{) zArm?Lyf|*S_I5LN-txa6-X8cZh&mB6&LAnD^W=mi{6lAJ{4u%!kdJXedaehGcdQ@Z z`B#>{zB95ffXgcZUxOWhD`q}QJQ~PN7(0o|4yT2saunZ$N$7N70Wc$Nu^nld1tiGSvh=p}m z-h)i{nT$pxzWu*e1#Xk@Tnn>n8JZAlaT94zUe2@)$MEFI4BQV?a?1}o4|Dlw76jdY zzqOgR8AxPzq&8(dlhQ~Mw(<~j;Sq)x5~-kg(}R>Pvew9;Bn!pjQI4&@@t6M$g3>v~ zjs#EAeF^k}sr1&}@h!-cby&G@;HZjiE(;j=IuXFp?a`F}Xj)^k*J-2pPzQdkKmOTn z@p;-6OE)nodzt0mcMe0Y6YMC+Y z&Tu#2R6hVouGNv&{Bow7e;he?o%^wktBBAYR*s$i7(iA&xoY zh?kQg4UWFwH(mQdQEReRD2EI$pa+kT`#X#R>>JW=0hS%UM<4n5_?I`!p8y7diF*Kz z=7>q=Q62`)JH@87EVHSnyxkmAR0E8}4NUD<`Y3%ESTs}ygnOVv{x3s&CN}>3j1Ada ze=~VQ6n*z)2TY|-x=XmoZv#@^9F2}a=tIU)ngZrL1P}@LX# zq0o%D#oQn%p`%$DE?&+f1XwbB?V#TqB-BGelkV|h^xAy|TKZZ7w6y_&0riF#0MEO* z#|+3^sYQMf~%~kG!5yt8(Vw^&11my2kpYH`viS7<>L!XMTYWRxu>p^ z!Kp?LDaw#N_jw$bpreO!f2ob|bPd%yOV7L+m6H@jY>NmAvbYYk-5Xm66oJK?UW-yb zTq_a17u+`6xOUlPhqK%7#Th2J4tCl;7j>suncJu&+=vF`@q`X@-wF@V-{t-&6&3W- zNibEzjNGb+GH=5&d*bN^KJuz;%hOlf{?yzDY%!hj z>(=V2)xUN`dJnK{DPCk3Fzg%(Dd2PisjuTsDLw4oZE5EK0dN1JRfkOV3r>+CANE!( zR>F<~?2loHF+^+VICSB3oLcR_k2Aj9#IXIFm{8Y51OFNBh;&UpO_cPvrtkL*GG$BR^X`fq#jQQx9YJ2ht{<_HeW)TXc;Ptn?w>($5Rx3pK zzR{sL+({%5jZeOsu%yy{#E-nUY5zPOK8zvi^dh}QfXv1pwnnt+qDOwOdHGVxY#%b= zO~2T1IsQ@?P&3QVhq#3kM3N>&WhOkEZ@8=~4a3elqHGJkZ%R(P4+&E^lW1go8@YJm6W73jl&$yq2RlpdqaB29emuUvW07bgwdZ_l0Z6drQxqwD#*Ca8sIS&c< zSn3N4PCB806$YNwot+IwBDSOm*6Ipw;s1y&ru_AsY$OuElgciA#cUfeR)y}+y0sVP z`UgCvf8~ZI45`x<`(tv4r@*}u%-R87e%lKNP9it{OrzpFQhtFnMPG*X8qG&%2+ z-RXjY`u#M&%aB|I_m&I*LOj!v7%oFeS|nHQ@5__yFTmE-SM&qnj7Q$mcTdM_wKlp@ z#Tv4)73uybpS+T5#`(r%QQ<#^RWbQ5z%j)-#LI5>9vx_2Bhv%rF(aAswYjT~6;MJ7 z^7h$lIm8r$(E^2;VRf5`D5u^2uSgHlgEDxiZA#8MTk+m`_}r2?`9dlQo6Y*X~@r^aAj-7^vL;SRN)(=4C!PN(TVLR6CaeUL6bZ*pXe z!BQxPVpAoTWb^LybPg%sRf*IOww&dfOPY(mKgj7+SjL6xLhUOgeIL#{msQ`@uT8%` z70veDf|ka9=CW5;&~H7SN)v#N*UgAe!@)oG8l!H^@pmnO{V43Q29eWYJ%w#q|G0YD zM5dbP3>#rfg@`FpctQu|ZNw^Tg5Lg*=U*2dcnljO?2m!Bc0Fi-4%hlF)e*q5~Q|~^odD}|15F!fw#`)lyr_RRB^r( z8n4N~kCKN7QP%VDSCJ198U!(VESwSY-6=}k2eH8ZKQ2y_1lkFunl259S6Uub#7rwckQuYzKaVC2zcyJHmDQGI`!8<%KXx}dsEUsi8WP<2jKtC3&(nPXhrg3W z+aEhh-!)5DhUe@CCA_4yRH|X&gL>jL=oZ8YT%=U#hXA6gWpe`&=S4~nWZV0a)}TjD zvfN;I$z+0Th}wP&+^#(k;~fz0rALl`H+Zx3X;|FZG9dw3hbR-w(FmKy)hj~TQshtx zl-i=U0?SCZTngCg2h&kv&af219%)1+sHWpQ&%)*hISEpeBwpeu@#q*zT@W#@I4IJQ zb||Je+M<<}D6C**B^0?dsWWPGgEdRYxz=19k$=RvP{1NhN80QeR;(#3aeq^g1ucpz zvu(3@8{+>p0MY82FPHa3_@svlnJ()Hox3}!D)f4F{YVes7MeaeBe`+4{u@Ku@hN?M z?-?>9VUj~<^wsciF>qK{m?==~z@|VA8&$`?ohfuhN1=QZV%MxRSsD_nCjoXhi1!iN z)-R0`>_nm#e`(H0%5b%wcA6rgY%nTXcNaCv7W@g4ZQE|W1eR`vCR-Xka?NYs#*g!r zUQy5mEz934$9M-eAg`7^OD5y?fw4NnLM)8yGBP%Ez1NVX7;kEHYC+|&b}aNw$hMz( z7!pN~hE!PZAkf)MH|r;09xTl#w1j0p%7%5;9se;IKhditq*auBP(+`To{D(%z~f^DGQ?LB${fdZAeT)ALynI?5$!l}18q{El&4?l z<{X{_Qa}G5#?<=%0FFR$zbf=X<3$r2VBR1d{16%Djav)ERBYT{11@5t{cxtytvk;M z+}w&P0gIO!dul|+C?=R?@=99eLN9PYnvZd-L4~^QNthBGoa3QW#XPqgNQXD6fEE2V z2eHxc0B?mPnb6?)pp;G6L84)%Od~FbTk9RS!a#Jpt5)NRZO&ARDN}-nDUe5pUl4IY z$)$m+po^Cvv3mJz24xGa*}tv4@aBD+YPa&4VgKVKX|2Qua9#d~gM;m|{l9;(SKI%0 z@mWJBW(8yt;(c!4X;yb+sa`9vWKwmhs0Aj0)75OCCPaA6Whri$ZMrbauM~~7ZnbiL z1MI@z7o5p*GTvvca6c*MT?0Lf$7y!eDe^Wgg*#H=encF1+e8}$^H7@_ai4@*lI4n_ z?(CHfRVnTt8X;)m@a$lxN-`@kedkH0AhrT6yGWRafEt;;D(k)D!(H66v?|giR>25B z%&?Hb(rA3(igeLKwZKyxY2)Vu&j=SH4xAUw7q7$JIxianZBEm$1R!a3_>y&m4eW@8o?nkhY|7 zY(xImAsxUkzpdxtgzI1O%*g*~&XqO6zQ`xC&pcMQW3fb zCvLNSr)%jyV@Ev94}vBHpcz?*49x{`a39Y}fRE7oRT;gqq$# z$7UCOF^3l;y+FI@7k6$Zi|TFmQyQtatl337Tlv!vD{O>lvx}OoR;C}cT1~H*+52r8 z{@^_y(Xiqv4LEzrBQA8CV%WzU+goUNXJ-rT?(c7*olU=Kh~4g@rhhrW|1t`!^vkJn z0_8HrrD@-pGeW;L5c;Z%H%UlU3EJk65%o4~b`eXX$UG4^8J%$+DVj99Xd50^))+h3d9i!2zq|8nZ|^z0jVL?UX#eFm_?civBAFrHp}mPCiY1^TpCfqjMt?fPk%kywQ8ia(K@k{3vip2~&j^LqWr1Uc#Rq z_j9ke%Vrn7@P1$UO(4`9>)#VVjA$4Vb_#czOd!oHhU{lVhJ=MhDDcIwTn#V-H=|U% ziSbp{Q_3oy$`QY;cnsGPR6I96tY#jWUmR&f8NBr=6&_yF&@7_tDT8yAl&SX|h#MZ{ z3nd{UAWY#QnZQ^Qr9;YW_=m3fyuH)j_p0hv^%SSFfLfF8XQDZw^>RUgiwq*J@`fbL z*vG;O%U^<^t{@H&=@Y$3MUsSs?1C)0QX?v%)=x)&sAFBr(4 z8VRmcM8cjV0cWARH#~)w)>fb2wpI@8#35j2uTL};0S;E6#n_H+*_t)mv zU;fFzVd@3+lOZOZ20_$*hiZCg&mUVLMg`5SO<>bEc7oT6+9XrsWlN{tFK03O>s zYF2TCMG$At8LtduH$WKrc!DzME-4H7CER=DL77TCBpHzBL@*M?JS2Av=}o8h6p%dU zoc^%%oEEY5D5ms2fV}B!o@rR;oXnRim`oxb*Ia6d9}#hWl0?ioo3FgtwBK@Rs78Xw z5s$)>lvFe(Zv-fCPAATupOB%oDoPbjM|7YjYba2*n=g+)p-+lNHu5wV*D6!9zxetM z79^ZB9wZm4q`aBi{G96f)2uVO=98ChIRpe}S1W8(2KDbY!syY!L{(vPZnthon zI#Jx{q7#q3VP4BQawz3U&*io3}I;_&d;rTmqkpW#WIxPxnbMGOTdoNaC z?mer`z1rNH&)my(jhMt-jC=OLdQTG7>yH(YD>d=7zB8%j*shJH6Yw~ux6${ibehGj z$NN-g-UDuHjRvhwquH6F&cKqre?_5IXljKfdw>55&CbCJG@9qNMpJ7v^Jz4+niDXMo&Pu{kk-p+$Uf3p&ZCO^y^Ab-exTe;JLqxX*(6Yqz$^dpdF74~4t8gLx147Dcb6@}Bj^S0X zn1_axjII~L%IjNVy!TGocL&kZ6Sf#_zj+`Y0h@Xd*?WL%cfc*V1Cpe62Gq`g+8N+} zR|T^@zkx5{MePfyeF3ZS1%Su){*9Ph0^>3J0q%KbZX1q!^8q{r#q(|FW36=FlG6D@ zt#sB(XRUN*zpFxOZlZJ^)JkWqbl&I$#X$Bh1=U)_4RndSjMe>FLwP4>BL=c32o4`S zMotKx*SYslbgWlY)(TdwU_E^Wt6`hfB&ey&Z=nhI$m3i9?H-uNY8=_qx8fYI(^Poq z9e}lI$0-}ma~3(5?}b@VlnUw>_W z-OBSdZdT)O_d5JlFk*#=Ce_0Kb#OI{Z>PMlGhs&Ky+|cFGpoN+ReVU(lkjjA6XVUx zVNu8Ux`h%RKEBCn)&Tp?z9n<*G{&FXw2tsPV7B9k328uyto z`CsnjQ|Etq?D=16hh?4erFK|8F^A=Sr+isSj+dvE)8*kNaak^F%ah8|@*vZ)ER~1l zDdk;xsEJmV$*fXmQu(H~Bf|XaNuF{Wie#N5<;m(Q zb&ix;RjE}K`@2qyQtK-Bt*hK&T9kXsgR+tXPmeFr$^B(ES?4mY+sKO2i#*I5OREnFuEsV(8!60UQCpxO|AScdS*GCtf> z5|GDa1m88TSKjfR8Nd%gAFPj+*2hX~UC_L!b-_ASN39D!c3tq!Q*|tro8xij;#em6 z#sgQ3>SP*>Z{2$Mr zJ=-tk|2Wt?cwXoKxQkDn|KqXe|M>DH;bEP(ID1g10rB*$+8O$IoS_>x$OAIfNjNrS!Z6oNvrB}L&e&w`-Uz=Z7l1v0`SVp&k zbjZ^%U71(phA3VpU&xc5Rg-n(+UZ;%@_gEKR;Bv5tJ(Yp$TPeCWad0B5Wqft9w#!D z+|@K@8AzV=On#C?Tsx6HKj_cPVwu%6;&w$s+TVFm3X- zEaW3!3FrKFFkMQKb1kYDNS3phkh)VQc00X4vKgP3jn%n%zAd*+otvk2+SE=PRHx{v zeKrr!XEWDX<<**cE9mlZs7CkfuermiU1gt-V{Vu5)jR~Hw?1lLAGNO)Uh|?>cs)S=y`QMrCQ1RCX~E7pHiJq>8|oA z!Iz#OGL6*66hY-Kn9L>GQ1CSA2|669jjY&&{F;zjn<(HZQ#Z)}lV|zyMC<~oemd>% z{1iePzx=;no4a0#D$}$@WyLqB2QCkZ9I)NHL0-GD{GFGVsQ|JGx^MBQw5axHC|Eu1Fa4+xaGK43a1MJ3E5EPPtEx#i%3(0l* zue+E~lmBb^lS#8?vd;W&$jbA3I5J;^%P*Mr}d z+^WsK2VnNqxxQ+%@2<_hZ#>`Ettg{)vactr%hbufYE`CIW$I*KwJ!4jb(uQh*F7sm zg%hMJbeq??zV3Z$iw8(o=t!^Ad_Az4F6H@Jp~L;@W%ybe$tLgA91ii7vC3}<$))UG zQ?R^1POrrUQk~c9&L?{|->TDjeOr#1I-OVTmZ{w`s7~iqdu1M|SEf$s^%OlabvCaD z<&Aj=s&1X%q0aA6tGVVyt>)HAylOS~DXFCDeC3&5`*~ zw7WWWSFPH86RKUEx~opzRj2N%Q+L&=yB_+Qs#AB>sk`ob;nk_T9&_q0Q(#)ou5T?} z*UCRL@_+q1B@%ST_87;}5`E0h|FwOve{fLD|F!e{+5SPD|LZP3*(7%xEp&ke#r=qM z5srvZF=lv3MBAk73L6N4IP0R<@W5o&vV|H5StoU72Wo!UMSadAf>{F@G14v8^7|Z` zd|^|&v-!m4?;(}k7c)kP!&ixj4AshVVD_MF2C84eu=WT1{kR{ z4Z(v9T4)6xO#nEx(_<{ugA}k=@A}lX1LH?amLBebS6<6u8Yeb=c@+ghTHOp^ut(Mkf^}P77 zO8UD~Dkg^QVSeXFY9CpMPhGuhJ5XHUA$b{LDN7x}i@Z2#A#L zc}R*K5thc~@ni~C3!M-gg3J3O3mht}!=vjJ`=wCsx5_u~F&j_3`N*QN|H2OP2%tq8 z+`@v&Jt*z`>uExQwhRPL$XlzIH6|u(+TuEeddRDopD z>UCDP60n_k3AhB&Bt$uzADk1fN)E(!xC$sNsZ52$DfVSh9F1l0>~H`0!%_elhnj5b zzKK+_9VF=@ID%RvL@$p&`EWho-(GIICg3u;2W|q|dsowk?mb-zcoB{1qDXsr{0Y6& z^@AK?Eotkkv8QW6Q0#YO})6Al||0eK51DOToe~5P4I|uFU);9Vp+G+3Z z>#ysTRk`eIw2km`mdbj<&+_J5qD+fd8XCs0lf_jLNQ`OJMZaSR8nr5HJuA~o+tx5>SYKT4#pc~kj%XFo+HS|dmgc%}NR4UEgtd4cUBhu0T zppB3ugaL2l4C#j28Pg68Tpiuqrfv_VlDxJ}>DpbO$rc~9B#%jJBQULR3LfeH6Pb9| zMc6>=$I8RWcFb3MU`B2o*5kO#68a2Ak6$xgx}8jL#$4_lH1 zsi11yvd`9Vl3)>oFLqT;N)iM-PFC6CSOzMB1_8OYhqf!&#!)iD?PBvdiuff7j|Csl zh`>VVqVN(TUf9z(*NK~*`+w{c6WC7p6wqzfAN>??d`5E)1f$SL1EwoBO)}QI92>T48nlk z5Cnv*?4q592Kpro3}G}PIE=IvX$KFwnHmukk|g5em@w6FW$Z}Ze0uHt#0CAGGlipL z9v-EN%K%42Al}#9657n$0~4~b*1)zLWkZeB5--+xi77kr0Eb8T%6qhJo*50%;Id^c z)Dt_^2Ep0PTZAL0%>+rBJu$)fM;AokA@Q_yLn=iIx^zDcheUy|;xk}1=7Q*|;u?OausoL?k+PPjZfPz#pu<#X?*158DXuRbyCKb$&-eD0n9M_@&U44r)f%uGbx{W=v@;?& zQX}(}3+t6^G$MLImZ*;qWmHidy&@4FpPF?2ur?6bL_Ag5!#(|QfN7Kpa^_<2fgT#U z&H)NK)8r|G5ed_Xh%H3g!!~M4#l?BEJ^#vq?OLi?ZOiHy-r|9PuOuR^tRLBFtgSWH z@{Wz9a7VrJg3U&Q#&`(m5+vbL>*wRl;OxXy2rfwIEYHjW)x0}9o&yZa?Uh1%NZ}#6 z9C1mU0}D}83>1HqTdW$pcPBZHVc+$o>pK2{R`X}URSUhK4=wRf7X*739%LC7gBFSWUUWob^N>_~p#?1XdW zMexJpPm&0;(a~sBRIaS*Rebh**FE^rctFjQf&a6S&A{no4`04E8f(rY3$qX2j}dX! zG%~Xp#z#2UbKNwXbo*}OF^5qaNxosrgbQ+ zFWima0RK8p)c6$@Q(#X7zo44ZDI2yJ2e+A!*9^h0)X6JplpiHYG&b6&ajOH;10Khm z*#{P@5FldAF3o9=q@+WJFgg~L!(wpzAG!fONr(=RB&xB7l5&x5qY?5;ro~2-czu;L}e(~A|?{LyR>X~SIb z81Ym|8k+J}ETWBf9EU&P2(y3)S9=pgEf=;vqRKRIN!7?Q{H(q;Y2F^^mC-3tP$&at zoDgu1>W5l~(BSEp1W{EK>($~Sl|;0LDGf=-wrNDR{ZU}tg^iyKx`tC;UVwrEy6v&A z&1eGiA3xi9v4hS4i>bySL_T3no>!KfUOH~+M5Q&XW*by!VkYf&m-~@q>%xu>X#Xe- zjY5Evu+L-%{*m+YExza5x^e+WAi*!Sq=$t$hKI8{L_&)D1YL|z2k+FF$d zHsFGA*~zQ43RSAawbfg#jb+%Tx2ur5R1w;6C;2s&QcFi!Srg@E)GsQnO+YlZH_J2T zLc4944IFy2u7ADOB5CB6<0}w8bY@g!7cwLWv`E>|y6B9k0Q%PJ(aMQjOq`Q(5OI9o z0J-3nwqD$8sJOnprzMhTtf4F^fo4#~+8J@Fjv|UBkxsjF%q2HZ)};~CUSUs?(WHx- zJMG=}Ui(?oiY{Fekn5B(t(G<8G4!WrTkF~8OFcebB(ws$);)3ih>efsbnj< zd~p!A%3H#=ESMY4qhFoN7(W06T2GHV!(8>`WoX7N-a&Z03;`uU4&ifs8aSboQX z4Cs}vs3$&)Cjmlhkt-Z05ot#}z{b;QRYqMO=rj@Ff7V~g1rcLt1vx_k8+Ha9kV;L| zgET^yBf^lqz{T?M3|FX+HFtMXrR_X*TFLU4v>otwcJcOca86j5uVveSW!y4q2@+C4 z0#z=cdG?61t3d_Eu zl@Zm#ENrQ)3(%7CpH{eN?S3$zjH+=32x!9wqMD#?$742^0L?TGR=&nbXARr#0bI-! zZMi2JVE5g_S;8Ye9HY}j8&&8fXHp40UU@@ToR(g&&7y46EeTC)dEuJ2$TROv&tAtl zD#MT(%Me^*D6^vFN-$Z1S$C1UN(_v^vEDMOGL+ju5h&gD5JjRqAewQ`G`8g%!hV<;Va2RTlduWZD?Kb?^)#imOmnKpP zg5y>yB3Eu__c@f@>4JUYlsw1Tib!i+l78m5&!#uGV_+ee@K0?-VCD1&uxu!sShrB- zl133?-1z-?$`ok7{>})ffW5BbH6nb zCu-aZsc4yY?XHw{tmNjgS^D0Y29!#@iYJ0%5d>Gt#VpaRU1h;A#OQUGJMWW=pjxRd z$Ch@T16XTxd3dK&lggo+lKxJ#h=VZLNhbk91Ilnje}{k{(?oGl$wi>)MH!)GH-3%8 zdgm*A+Q3cwEu%GX0X}1`OCSF;^WjxnyKA{q4cshR>f)H_`hK9n@jqWqtMAId9CaI+ z)#=YGm9SDRg13zEmfOI_$EZd&$^mo@hk8U=%V-9=q*d#6K!O1Ax&fC{D0^YY?gjjSEDng4C+FF59$+jNr)d-LrLf=j)=(JD>98WZf&1Ndje!fkx${2A;YLm8`cPrt`l&JXeCK=hLy&?n3 z2)N_1v1Y ziEKJr(cN!p3jtvi>E%pWw$E@1=XPHIJJpmQL^zdB3DeSeZ4F5vIvNIuoXZ>(3<8~K z`8eQ$pbaR8jnv2pTlWUA;z;r>&Ceqo5TrHs)8nI;uYsplxQ|-}ocmi9E-Whd1M4RA zFFSFlnc*;%ErO+L?UaI{XxRDhYd&3>_8~tUgl~3j@3!-927h-3?9D9;(Ow&2rEmb- z6Ci%vmVlXZHD-FxGk;haP z!V#}$`B^`mAz|j+m{+WY6WY1hY47}{vqsp(1r|m@YF+HK0vp+N0(o{pN6vLft#uR->v_I)vC|2ca9?t6{ae`Yl@z3ugf ze}XF_SHfKU!9U1Y>x#K=)kZtp`^X1TRHbCxx7^LQe4Nf&H)-)xz5K%$f>~)kDz~s)o3FBju z>P0q(WcKmR$fX;tiTzvvJ`IXlDGnIttaRIm8k=r~%3XZcr(?xwcI*^KS%^>x zeF0~5U_yl+1qL&kTMCIQ<8r%uowW=k!Y)d@&Q4aI-=#cXB^FXWW#}!K4F2A%+fLIo zR!NfW#;m>8q66GD*0ML7zQY(#{6&fN~@mew0_J9&;uJLHK0rZp2!r<_wY-!EI zC{~lBTU0}vH8y)Pig+8Q*75qDhop<1?QZ8gsqUCyzB$?|>04(S%+_I9b5*$K|jlTFCW)3?SnKKfnjdH*0|^iMF*Cnlw0G0B#; zr%8)4ZB3aTeXM&Jcix~$ddzX|fMmHejEl;csm?G=1Z4qD zfZ9Vcv8nDL(9i^egmM*~%w|jmQ!^P;p@`)#X2)X+e}NKqJEOH-L4z?<9rJp77I_tY z_HO$p(7&elw==W9omu^jxO6pADLN*tey7k?k^4Eurx&i@Q0Uy;XpyBnWi8KI%u`mg zgOacdvzp=9Qf7s`qXQC*gNUG?I6s$d^xF6ewbpL71p?H^a+G-#xiAJGX6UE0vt#GV zZA7jiXeYBW5-!!L!g(YJxgtRhwe#P(&E;b(tsft5I+5UOxWo|8EtT;sU;L^b-x3?Ong5E#wxYm$$^O3AN zbU{sS`!jnL=4YEVT66bG^&)_5nV^1@W8^lMTNJKg2vD)#GrAFJl&u6qEXAc@o(;_b zPcHzHi{aZaFObWV^)Fvg+oX8`1L5n}UOevUZhFwlx>sYsqfm3Hc-$LMAyv;j@Ji$n zst4QBdk$b1)s~i-%y`_RVQ;6TfIx0wuOnT*R79&U)aH&Xy(;q7yhFIWWwLmYdOm5W z)>}K<+v^qOw785W+b#ektKVw~Y?zLXX|5=OK>HFnp4y0OYH!ASLsLUFwTSca&g#Ub zppAQwpUs7Y^v& zf=ElKhlgdI+k6}Wy_fb7R#l-GUum`mcZJC=dbYParQnPO?*kavZH#X23<6Pv}o6J5dpE$z$*)cFi|%zLW~t_~D7F zir~15q#!;0&9WSD!34@&gmx`owWTfM=V7lwQs8a&23%3i3k|PJX&KK}T;m}TVXDfD zSQ4cmo_}J*dNB=+jDdaO|Nig)o4mxk7ye7r=X*>U`Y49&TyN`zlZugy+nilmyUwyhn;o*x309-enWGH+JQ3|@ryKJMoxeI5 z09)T1?cBa~rj)0o+cz2D*XZh5xzajV&oYJN)AiP$N~eLXuNN1I4SRr3NTt?=Uum>M zrv;rjSC#!a&ayIQQ=H_kuqh~V3iNfeX@cFj04Ml|k7uvD=**mh${PcF4(0{?c*cX8 z$4;P0+)wzf-LeRH+?kkpucw5F4F6jIMivXe?AE{anCCN|+pUg5oad6qq?HxRIk2`o zihS(PCr3Z`{(k!LgA*lii)&R~J(pb%;!e3}TeqIO7c8Bu!e?0UWfLmd7y zPxmS2x74p8e6b3|%udvZ_Xxg|2fB3>k;;b@8Gj`HcnvB10{Xx*4^D&2QXF=73OPk)K51KT11j#6|{^UD`R2WUgDG6`ok#irU+pn`uer0C)0_;qvBx}>oRivWzk{zrj}hcJPUEu zbJ~EY`Fu>8;LmimG>VW(*NFx=(DhRhfl$=~PE(HHm(mJ6b3)pk%ro)8pu@?A-7w)? z^)y=b?@IZ~gFHv1IUUo%sT$cc)txmnFn7X*vOGV{8o1zWBGRtyupHxV^d^mpo}@$8 z({_XZN(Ma9YbBLLbOH1&L2%gRV(2QTYSb&PjQqHF#D|o1I6$ob;R2F*uVtcRzM9~m z?Acv1r|K1};)J%n>Lys%^rLN13i%@P(UP#o#Sk-_0V{iH&JSs=zW5+J)v|`(y*fHx z)hf$)*N<8BVibYt2l*5f991_VnI^Od;yFW{v=xsdL^Zd&z>&3MVZQa-%++7Yps+SM z+1?w&6XnUa*D-iSnr*4tGQ71UG%tA99MV<;17 znTG1(F5FnN9OFwc^fldDOSiUIE4qdY$8ZR6_^&t_^tl*j{#nl-&ZFqy$lYX&yNz9O zxp{Q^C7ioW<17IcLl|_3&$@6+Pu%CJQSNK#vuW*fmY~Y0Z-@^}12FTSiy>FAZI%T* zxcCQq4$Lj3tOxh=kCt6(7ew^AB)y0ahk9U#`I!%d&@aCj)+>Me&D#t_ho~8kVV3OX zh}#&3{3D0xVDG>_NG0h>8M$Vy1)-^_ry_|Cy*WWh%BUyLY0|qOf)2)|w|&aOUdVLk zDGd+*OK$P>P;(c#6$Xd#n2VwQVzM0+h~xDASp{*F*8A36i(STQHS1hA)X@KQSGOQ_ zbF&xZ^(QQv`dhPnx}xcR*^oeUS(ZER$fJk^su-i?B6i$)DtD(93(bm^HjxC}DH?(S z2#;b!nTxO{y0)Sh#QFfFQR-ofUm~nfM6k@S8Z+}RqMILvQq>`zp%*U?m^Ytk!A6YE(d|7pk-G81tibdZ_nG(nokftsCs zhAa$epJ=@S;^XCHOPMS*g!YNif2ZUPjflx{l5vGnnns)Nr;vKlLHdplQA93Cl;r?~ zu+u0)c1H4iLYxuA2VL~)^$(wZ+(K_Zy!p6=em*++u!UZqoP0dlLN8C=p1pl}^sem= z>p&=^E(m=^dkoqO+u8iV1(I_g!yvPO40cXiH?rnB0+2%VB0oHhl~qyWZF4)&3L)=PG342@*b{P z!Us-K#UZ@mU_nrf#kpm+{?&Wcx9P1}U5pPF>_x&-%{u`5COd2BEti|D3ODZZGBCVY zGcqTw6CN6nZE);}N1+8DSSs;(586l>r04x?m+%{C>jRDnhzm*7X2W-wc2u1BZo%Ak z%4r~S&WZBT9ZGm}D z)QE>!jwp^oqO>TZ^3C40(2((F;kpqqonm1dSx~K;YCy?y< zeGV?xe6@~wibnWC^936!j(Do@kf2cT1olZaVI^4CvrLz9baHyudGq$emP-O?6QGRv zC5pJ~(g21oF;&u?Nfg9g0`tG}9*Q&Ipex}0K#~A$NE-ZUa|S*RiYUwcw*}?du;xe) z$GNoZk>8>iZur|yH>U8j0t<5530+?!hZ&FW!yp&8Vw!6Q}; zUb7Ga87QI?c;~JD#{%HblC^JRn#L(uEm2Q)G1#;4z?|5WF(t$mP zfHXxH6pnuTwP36TV=WjTn_%?AWo|4KZ6nJHMAOWI!q8{u4?`L{rg*I+Ja$1${upM6 zL5rV7M4%Vlyu1+18hd0?FoSy@IhaMeFD40#ecYQYTuPqISbA=F3oO3IT7KnM>59TU z=L4Kka6eq>EST|w$&JMah({r^7YQ0DH3QRxsvB?t#6MA~xENay?Coktkd{pnI9aCo zZ_E|3w~pCVIR0&iE`@@*=j_B5FaZFs^Pn4ZIaft}^uN4oOD4$8zt^#_JH1wJ@vQ~W zZCcw|oBztSRyum?)~)f^sI}THlMWX`{eb9SFcare)zF>bn@eA7qwc9wdY zTGnOR8Ybyht940P$S+%Z*&gofzSzEczW-vPo%bJKz3#f}eq7F3^!CROA5UKQG$f}z zAkb4UI&AvtC}z!Qpwa_-?_TrTRtkIlGj{){xA7B~3fZem>}3??i%h0hcK2PGGg#Wx znz#VV_@li`7-myYPBfE&conc@2_v{L+{7%a?CPL(t-q|#Q5Pal`eR^&c@jhx`WLDq z|Av_zODcs6W_EH>dd)3G)v&Da(xS<0i>#&#v7cIFadDTFIZwLH7p+wsmoW;G7fXXj zxo)mZ6M_UP$|K6H?SP}GkAw3pnRi>HKqSO7fR=7+ndylcraU|A-HN)1?cu8{@6o)eNZ@&hds@(4K_ z!}7i&RbceNe+xP#$Y9F)JY^wL0uKf>&@W&^#t$YeTpoyAw9#98p^#5W*Y(Czb<^%% z+um|=Mg&=CA%_E4p-B>r+vsQTDVcyh zn{OWI5U!rgg61t-Hmw!<18)`hHk@Z8Pp<3{uu#rYwRhJ-zs9T``hoVkcB1?tm`qhuylf6>c!oF2_d?{#b^^&Kbiv7OGDmD6iO zBqX7x2o3?tapT<2{#$sF00}w|qW=ATFxk$w{5 z4Oua`Ri}1#?NOLhI)AkqJ+13kJ{K*WY@h-HOu-ye5krf_da-=^tX7Iz4yVLdX24Kb ze#sl1ImjnQ97&BtW%f3g5x9i`k{4r!Vhp7<6bW@`QL)H`F9IRTiukFpW{MG1Mh(-< zxcSi|@G@b-N0s3@qEEpMl0lQqTzh5$Pa!-Mf=fOgg**vThSH~UWFmGc-zzuBGN_Ur z7bZ!~44WZ}tj1JyaCO};#g;bF5dOg;c6~bOf^W$ZGd}`H+8t2*DxXzV07%@QWN{;u zzbp|Eg<|1&;9F&n(pVlwg)aCcB2&m{*lCCra2!5W!rKw}pqu;Ub4mcq;nOKms{V)4 zRE8B>%XV)Qn>$Tm0z=hCZWn8>3f;11NuYKfGFFhHRhDn5@XXYLC$W-@XsYU-*DIBl zJ!&!_k*P>X9&qU61@;pd$n-^eS<4-+U4D?xeW;Thr3NHYevwwq3iGO2Iz`Y`7nJ{$ z7b3y1{-~Qc>iu_5|E-cJiG>72q*Eg;Z0_Ce?oQ1#XBn|M6Mh3@Uqsx143BZZc@Hu% zv3ny-PZu)kfQ<4DhP~kq@Cl;)+)N2cF;=OH1)!psa=QCcb$MX+s3$9-NJM#=B*8gi z`t5fQ&o}%ikycm5Chca)YWpHAUgq1WIab<#`|jfCyvIJWhJ6eMgTdbJuK0H_807zb z_H6k4yW#Hc?(@Cvz3t)PyTSI(VE5T~VDN>bOD^jZ0F%Et6!2V(I6hr}n6iM4rWW*^jA}`2v8Av(riR0|*>Z%2bf>3#? z#YwnNWfG-<@#eJ*{iI$^vfnKu)@Ms*YNm*JP)gXkp#8*y(PfC(rSN!EbMoTVg=CQ&bXxKu$N%4w!r;^LMJl-;?%F^T{M=C^;alwXt82r);6;PksSkNKta7BCUk7fjF%nBy>EhN9YR5geatrh5T#z-drVvAQ7jx)HV%asi1^ za=z7AYfr3GW#O%W)SD)=`ox;m^P{t$j}IIZyAlM;9%@fM=@}0UZ0gO;Cm%I~4g~a>x?12Tp|&x4Ews3&V4X7ST`y|h{s(X$Ua0}T&GLV@ z9)W)fs}c&%lSo9DY<>FF|GuT)Dt20FRD3`6^A;LD8L_|l34H1D(LR*U3j0tv76#X&9s4i1lpb^UTO5;Bs3-CuTn$M~MXNn2gip_4Ouv%}I z1~8+P-M`Xyt~J^3GV82z7wPq9Of26nT4w-hYO$5>8mVxsm|#(=T{;{S+wyNsHSW@U zq3!>#4ceFO)%R0=zvuGpmIGfj5Ol6nyI4 zu4Ntiyl{tiHsW3YO~!wLC|eT;VhPltqTOY@r%Ie~ST=$etD(}(u`m4JW#Fxh9Ligv z6mk$rBYY&oIEm!m#?>*iYE2mr(=5rD%6R`3D$?tMBGZf5wbS5tu;x=1Y7>a}_(Pi|PXLKEUD=N! zO5*r%@ zZyoe%sYPb_5NvwM*^U3<7hvVGkxm908BHf-b4x1kY_a|dJ0awns%W+`dO}^HC8ZKf zL{ernmDTmYv6Q%B26S#*+6vK4Ce7m&Wwi>fDGeu?F{-NH(F?E@5OSSFpuHHjL0g8Y zY&#BAQ!3f1J8(Pp*`m8M4+VfjAAL+ewSCB78?;HHnbiE&ZUIcsMNzA- ze>;vY*Ph|G#`4A41XL=e##FyZ9qJl@uInD>s+vO~G1#gcPBwl%7#1gp?_sh3*G7Y! z#4?ibLo<@lSss@Wl-X66z~QN?@<+$r*Vg%%&r15gi1~r z=>P8HDLJg%vOc9`iuD!BdLh&4l$C9p7gCo4w$s4EC5V;cgoF_h2=d%2q4Hc6?O?Rn zkJl3cBlQhXm|bLrEki~LYQeUVSJToLa0fzuJqj7vcF0|<)(v-W{@wKcUanPRN(w&U zyo|#!+`x#lO0kD6A;PFIAkp89(4r|GWx$GFmiRJXNr|Fq59=8*!D% zMLX+s_>iplGj!w}t=W0ia79})awD>dr(Zkra-owq@9(KXwV;vG7J z`jR5!RERB=ZHrR1d@rAms@QfG#{zu*JgO-yMN|Kq&fxa5y}H5@>UKIv!?5(57={5^ z=v#o9sU&Ke%A#lJ7Ti|P(#ZKO3C{>&b|~F~LA17Z1|QmDt+bzlcK3|`-y&dEH@&o# z-1wd4gmE=T!Bf!AXKic$lH$c8Q}*byctL8|nSZ%)M|2RbJ@dM4AYR69j2Q(F$k zQ#t}4K)dVljXA&eaSXaqlMK7yZ+`=-G0^>J=RnhtWywHjkIvelWSG^11#bhpJ`Udj&%Zx}$0sI-7j9QsdYpNmH3rNEG}S9N3=zTV^I8a`qL~EBj~>i14W7Suac!^uevY5RNF^gjjeuH z@WRyV=EFhfFPEy{&2qwatEFdzU99x5bHVrpj<_{|Uivt*8aXYTVQKs~bHSI9arg3C zkB(=38v`m`Jl3Lh#;_~`qr%GVpPU`-AO7p|^z7Zi(fRq&VF559TPy=exAx}f;-`1T zx+lly7wufpll_-RC$8aFjY@`@u`yGFh%6dr5nKVo<J^-@QLO zSZyjauU5NBoY@151AG7W_`lvCUA{U#IeNSQ=IFf1N-b-;cH4*^PWOvLw<)NWQb#eF zBQ`?`?NSE4>z?|MnysB+s_b}bHj169u>jZkKaNi?FHX)ce?B@pe)X>=ILNnHGWMeI zQPtl#6JFZW7h`xq-96vp2D(N#TTsPZmT#kSX1iPzxfByay>8mV1NZWW=3dN|a-Y0Q z*6+8(xp2m1RlX$~QA4gJS7C!ZOLfbe;8=_v?uTEg>Tq-1O4`fXyh;Y|1E=!G;8d)X z(z>*rXL`dNi=EjzW00(T*BkHi&%lqRyK^;e%peo0-x42&oX%JIGDHE`ayc~Er8{Dv z%@pIA)=o;~L{}-Bpx#N@^y8QiVPKt#`Es#!Tus@mW5nk`wo|7$GUGDT#MR@qib44- z2FMY!&+gUBslA=D0z*i=8gFc;AAjJD|ERpNzc~d>HRn~)@wdbsJB4*Aq(%m^5-zrx z^e=6&vGC8Xu~FGyWz(Ax{bgO;m*lUa$7NaR>KRnIr#Vq5VFHZ8Y9Oyf9c8JdgSlm3 zZ9s@vPyj!0=D!Qhya@%6D{emw36sI#OBV9w{@b_jF7_{u-@RRFO= z5r&~r(M|(mD28=;esuQp(b?t2@tdP}@0Xn`C!W`xH!D};H#BcHTwi(PsCMsN8sZ`QYWoNVe)%w9~ z=Ff7uelVM{>(k ze|CKFugm?zH^*-;-=80yb*c*<9-r^OJUO~NetUdzynmu99-JMCYka3Tva_TA67BK@ zVUN$WGzXGXj1l_>y3LuJ8wTfFrrETJ2Gi5gafC+TI66ZCfe!K~R{fdl7^|lEytVgdC+-H_9Lwpu)Ldp2inOY%$;}h3FSMwHtVC+z zhQ%$maF+eD@}?SOuXCQPDsj4}*=r<6*apIPkm1uTloWjj%jOmVon zTu0S^km>xPWjfg^xlehH?)4j&s9G!HVG4ajIxTCfrMOh)1FM>b<| zC`-#_T>Q$neUP>M zd1WnXVXP)+N!k5O#$rdvF_J?W&=s6}HWn?hCa_?m%p2}#iC;KA;${F}K0cx|;8|{t zRP;05o|)O*ke`&4=IcRcEiV9PE*r=GNIjm;zt#q68Z0XdNsS#5vW&0p<1g@Gl*{mM z0Kvc3gCN>qWF9yGNWWDb$F}oa_TCX1>p-Ao4x@_bSc3T#5wcQmI>#7v!7Nr*p|ly^ zOS05Dn3pJOve0eooUabGRdL9EtHH`XRYj~cB$8+8Z*9K-n8 zxQJ|S7>$9fA9J%|9L*zkdx+yr4kXvJ$T5_^tC@*0h<{3A_BPi`dY|MUUAaYi48~K$ z_-C_T6ca`~5{$sb!Krwgvmp4oY9@Ctzen}OpHJab-rw(W{x+q4O82ommz-!jetUkg ze{ynpdh-7D@!Ruibb0!wRf(~pp_hRICk9N@uLfF1VIha2_mo>5W7 zu_1F?F%R~}mhZZtm#oDVR-v8txRmdC{p#}U=m0JErtuV1`Hd=Li#zcyVh9549yKvEPl%B3X8IUV?P#lq-8o$k-gA79m>>?4uwdK z!f~}Z;-^Wsq(zrGNX-C?ryFMi3&sm>eYluhrzSk#Y(q_T_1bt`TuhBeTr>(P7=%c! zW8}WJ%K9Z2XIZn5QMj^B-r}jyN<;Zc;o>DXM9Oe<<%OQZ(lv}$_!AY(S&9d7UHacG z^l#^Pb{iqkjuu`!w;4=xkhB7m()xn{;#9KX@gHx^`QLH^qVOx~HkQD3408}Z32Tp( zGRV**2`FN6#E(Eo*bIl$mOayXJX9FG;P4HallV3-B|%CjpfNQ^K2GK%AWJuDk>cOb z37%tf5$jjB@V~Zp-k8H&)G4IklcS*90;En?TcxM1Z>lixJTl#AsB3AdfiI^AuGayt ze%N;RO^&uEQD2zAQJa4nplJ>2Th+hL#o^ zSy!p7w(G*J`|v2PZr!8UZkN|6zrs5OjWq@VWnF9B+N^~hb#tD@-GGIL8KIVHUjNn& zMyVD*U($_Di8sBHAcGkrXsih#Q7;o%lx>{y`_mGp3uP#8dS2RM6FCF5*;4Z5YxAY3 zKfil;{9MD+}0i$z8XMHqBL_$hGleWC|)X^0`}( zr7dy!$o5OCsFff1f&J5CFy#eXrp^LSLge{_pG#a`b<1xVQU2|L@~ji~hTa`%AeZR-^GN(NwhtzquoVCS$&(lTYJ zU41xIEGp|>^|klcxBjd||MmIm5Y)O(f*Sh2y|Xv8>Hp4PFx-Bi|M&4MrIavf`eLYQ zzn04XQMmPjs$h9qtFd71iV?W$3VfJo9BL7xbWW29)6%0S)tZ^KbtPeXqGVtso*w&k zYMrz)iKn<}yzOFb=>SMy#bo{p2Z)}tIPq8#8zNj;r1@yl9EneC`EA-DyNzmdZ>FsY zwPhxB#+0w-Llht;@ZYc%v!2l$3<7fV9>0%#P8b`vW>?FaOT>%gkfC@1gLCAO&~M(V zv$mlr-Wg;)Npn0yeO@|`jk|BK5==R7txxGDb7X#k&B|DO+^KhMkm zws#-ofA{j_&VPCBOD*1(JoK0GiCXqZ{nl6tIp(|#is=ubeu(k?N^l6;Pus4N7N26c zK#ahh=}dqEG=D1p`56Xa^g=-P(d|8vQ}OZBe2dTAF$L{8jM`~Com@#>2!{(8U_UEp zn$7W7;PYqjqM)JE)hXj7q^5Ri@1_eqE>qAIsukNUGaq;A@A94*%`s^sQ3vQ2B|c;U zVt&aR6BZ}WuKP#De)(k729GINXO>T2NgDuUGa@&W!=cC6;T_ZFvdn&>GA2wDAPW`p zAeKP499VWdmlbenHgGS9bVq7wHtl6KzX)m#TWDF)CpTe0pnqvv&CPh9eEC26OFyA; ze~iOE3KxLRl)VfCU6f2LLyVg7fJUxwRkf}9qGrm!r={lCK8`^*GTe5k@WsnFQ}BK* zBLRFK98fiy!BxYPRpaV$v2Z#Iu0Une5V2$55vnYS10zzIm&5_+((4KMA9;`OdL#@v zTg_D1)V)FaX%0V%*V#MkxK`6M-rMy^$oMi<8tT>qWlk0@d<-e0`x{?In&FuxHV`;o9{ z8odDRwje3_)#>wZTapWLZIyOe7^6;c|E4HH|K=?#&y>^yp=%Y>puG@9t=--&?CoxD zb9Zg#MWI;ifzO|_1SaB*c%j$|{&mJ!L`VHJ2Bt^meHh_y&OwYc_XBGL6;oQVmLbpyHRHy%c_G~*R|9QSMeD)y!xsRuuw^bL1 z2@a6tvxHO^tQ9nN)3PLolVHx*ZV55d<0ZvRkMm+CAZ(HI8d1r9m{6s(+QLmHP|7~M zB1tHCgVa~PF;82;&DMwAbwTk0usG2n9#sem8AQXdFcwEcDi(9~s}XhAZDq$I|B^N2 zvUxAICcw3AueQ{3?Y%}R1I=H6j&a$mjn

DEROk8sxcFR^gkp#S&rNLF{N5Cbov=v=vao+H+gbi^#+a2@S5@sgDU z#ykjM6w!Wq8XTsJ>vyACFx{s`Ip7R;puUxA6uyWJT7pOcu|M&5HD)@Qn*;TFz_Hp107*l#J zFx|rIE(ca%nNFFkNCCB?(}*70AHe)blXm4_aeO)gZ%K#>tn<<=rjley8`}fjddmo( z-*tqxk3?+zoxxxrer6Hz8wAXMIB&kPu^^( zC=bYsYnqM4Fp|3qT^8wnWRl&e6o-f*;euGvcPJ9vTP{>?HMj2HJeyp#Ql~I)ti?L^)j;?yLSoyj6wfEb% z_N+qx?F<9mT6I7*{T~jV@9q@zzq`W+`hOpfovy7kX~mb5*XQz8d>Pi<9#>OgzN?m7 zwC^n0T+nY<-{9@QDY3VEv*q`-BKv&h%?j}I$w!tI=iBwaLE(P$0lG!~cKzRr;Jq$6myGM5?)SDqc2~sCPRmXZQP4$}9FR_2CgQ5`8!2qT3tr8wrOBPVMpv%R^mybEI z{BWHq*h`bJ+rjD^HJRHnt3Wv;lXu0EQ&D^gAE|1_G&47svy%9 z%`68`mifPs%|v`U&Aym)zFCs^#$6So@4I5?n;zqZQKm^9tq9^-NRk82`|7(@kwvj+ zd^NJ@44RNDM5$~(>F%Sql%e^h@@Bq_(G*h_-?o(>b7c=PiU{RQ`)y?^FIkwuDfs;P ze@pUQS6IT{&JEZmEZ@qI3Q*c^1I7Y9B?0zsE4E9v^kh*`bU$!c^vHY;^Zlxqqa1uy zYv0)pU!eIs?xh|Q=HdtW+^5CmUUkBVTxt2FsJUHKko3A+awkev5Ef+0@^L2hkd;hg|*7+l*4C)Fkz)ENY6+ZtQrLuMr7ptbQ-yY`)nTNz?2xw$vxOSnwUvED>&Rj zjwKCunrX?&@ea2u4(c^hB>m9vxUWc?lw-?FV5~a)t8TzX4kfi3nrBj~f@iH9M@_L- zp7*F3kVeKMy*g{xA1I40T_^I~fc!5+?1seG-AEjk4jEvb{qN3jHy{6Nxcd;v5O@nD@hd# zQWDOVp^%M$lyli#0o4^?F(+0Yn8akR!+E3vxrq4Z_aPZ;K&C9=k)|PaqRp&QU8V&w zSU!;#w-HL?njNR}q0ps2$UZ zM@yS;40nr+U2YjTC%$|ExTd_UmsG%=kMglu`f~KGkQa1s)CJkZg^>r+k|QlgdM=Pb zSRToH1n@&Dxa}a}7_X)|&vu-2JOT^`*MNk)G!9Ud#kEHkz9Y z) zBq+O*3uOihb&b=KicLB@g>1&Jj*UCf6`YN8K1G9Em5V#8q0R+yAQ+lF>-5RPt^>&B zCn^oqmM^Ra=&B&TrIkcdD_(`V${Sz1MNfcI)Ri~`X!)O$2h33UwZk~*QB*QmN#-dCz!?$aej?dhy z)}k~&Wa{IA)A~~hh;$|fi^AjtExOw>0-7tXX-lD9S79ey0F0q`jcjun2PEb>R1gj+ zLYi-)oYL5jxFi(b0uUziu@2{Dl5n?hs$e8#K=!r)97YdKyL9Z}A~0R|0?V4}K1{*- zQ&5UE*DhO^WLzE&a z_Z3VS`Mdaz;+Vvj0hwAGvG1crpE92$4BWstlw~9AgW@A{D0tADA-Nmm|9cVk|1P~z z(E^2xf=_z0E09z^7{Lt;2iv={kI%9WznG^rG$K@ghE&-EL@(yI;@{D5Fqi`zLXY7EdNK6D9Dcl{ z_&4-oIGCpBn8OH2R3*_ghCTw1vk6Y$rE4I~#HzmhJ^k?H3Gf0!(>I@W51au&vC%{z zqNZho`(cnF6aeV)1)r^KfuZ#4^fw$?_zVATeY>3F@G?N*l+D2KI`_>Z^XO9G3~c8< ziNz>?GxU-5qku0#D_;&=KQQQ!mrK+ zm;`1O==&Ol>GR*+Q^6h_h+#fXBFbU}=c*BTBGt7a>UIxghoBuL;{bbYv(}PZpkKBJ zvSs}#7c6~CcvhtsmT5r7Fz9)Zu^_Wp@_`P1Y{t1PE87-8wL2Xn&6W)hc+!SZbe@dg z$3c74Za>Mr=wJUp`>H{ax2dkn&!5GXTnzgu7bfb}a%p{a3#EO5&<1zEQMnOwU(8Ya ziNtbIM857Tv+6E}$kPx$Z_kinQ2AGP)0*4o7ru%%fxUJ;2* zZ~qVUzW<-z*8dv)O7A>M?|zM@cl@lpo8rk`6i)B3NAA2NzMKAbM;BB6-*0#GX!njx zr*~w!xVu5)dFOG*B3A&Qj}eabDX*UYuxX1m@)+I3m?7|g?RDDwUZQNP_k9QS9}mGe zM)2Ag*5{V_?}L8fOFygJ|0Cl2n8u077C27)DXLBcQ*-|x?iBLB?Cm^%i2rskPtjfD zFrC_$Vrui!f0xydqgt5-&}p&g??O-TX>*BQq#VB<^SwC^_ZJWcnU39jV@ZmeRR2_( z`oLKmetaLoT=&+ydH@PdR9GT!G|9_@cDmuSBK_9^KN{(P>-4{a;j=vbA3Vr^@8`)8 z0{ik1(Wp!ZEGNFd7>*Hx!;H3{tA(+q26$t1Ivzuhk7K`RAZaRM9j=J3nvTBmx>+5u z#?J!P)Q7U+Ts)4Ee;AYKpb#$S5ja$WLyDM;D{x4>YZO0~nzY0HgUfe6ADx{YA0CNY z>%mX&{`of5D(;`0v~p2Jy5NbYW&BU{8y!C+eW7Rq^3yjS#p&NNtxfvVCpX~@jQ#!7 zG7d~rX<{_?yC2>@ z5l6}i4wH}N-;*c`&>V#f2Crk1MARz!CZS9emqQSSoat%w2)rWk82bSFJ`SgV%@7cw zXQsTNUW)HI4o6^^-e~6V;|T0Md$#ke1;B)gUhlWlvhYX}tq;fLpZQto{9kyR%>S^x zyOTfvcLsY8@qg~+F=U-zKF5zp09xiAeCj$Zeiv6mhf#|;cGCiEDUaMb(=xM@2@YX^ ze?zhEBYV-f3(dnwE#e#EBjtJU_!5j>q)ZBE^AiiG{DuHB5FkhyAR$s!!JPOAOh^ne z4nQ*?Afd6nIcwwp%<^dO7fS>ZX}_&62kT0kt9p z=O)-IwX*8|IryPf&i|MMwK@OHr|$gU*&gQ4|LyJP59j}VJdX2UTztJ4@$=F;_r-^V zq9b+YpOFA{TwbBY<5pBHkqGl1iEX+jU$ehS)B&O_SF8Zo>kBPF<($&BlGWi{ck@0@AmGq zy$AoVdwE>qa&`7^Lai2NXDF8&{3-ktmtkkZ%{Qj6*-2Mo$o){b#3F%D)e7+1YHy^ zn%x3x=>Oo^v%LIwcjwuI{`X#<9Q{Y(LdJcQNtg0kUn7~(U`9I&C9bR6( zIzBmCs!C1a!{hV)mnTP;$8V1>j`vSg<%6@M{fpywZ#x}ZMiTfia;vc97kgx#FGB79 z06eUNdRj5T)|s(yVZ}+b@UFWFRpEs!h~85aJL!BuKO%~sLP~E)?3)co>Rqw%v~+m~ z8L#eWthk=^=X5z$M~BJ<5pZk`wJ3cUbs3olS=chOij*3ONNT0yc#zh7YtL%ue@(#m z+*5P@@9b`G7vsM^*#Fmr(9s*Kof{eD-A`vfJtH3de8rsO)BlN~6n8mfN<> zg(JNg(ADymVtperVCzDCC#ac%jG z2212H_}VA({XT2(e^83=>)F6M`S0GK5dU#!@L>OQFOQpsDb0-51lF*DTIXYOgX{5Y z&6p|;<%W5y`&M0jRhqNL{#QlU-v2w#_VV%Hp6xv3|Gl5bz5m~+sWr{Aa)~yxh0^m# zj0o*{WZut{2G!8Y7sYDu(jDd8fKh?{EY|C}glx;-PA%%Oek|JZ5$6Nbc1sxTT>0v# zwc73b^N{(NXkJ9A3%jmNP0*4I6i!27I{w*7K+QQd2O1&6H=t2dKQa|%dl9r9!6U-RUXwy-4dn!qMp$YmmiqD1q^?tlVjb+vDbnotDMF&DNAGs6F zKNQ3P064NX@W-1W@0up_zSMnNv9Y$I7>5iz?p>&1_Fh7YL}s6MKlQK>^s_C+oOA}a zpY73P4vdJ1X&0H_NcUGt!tSzNy6Z0m0`J{yKDp9}9*4m}a^b5V5~?zEakZI0<`nmSzn3-y9UszTp4=3BiEP+GzmfBo1D*RV?uaqY`ncXR-|) z*VrRIdhvMcW{Mc-c4bnBZj5L|LW=mCh;u)n+trCMh6Da1#)yI;c>F1egU?|5$G;81 z9e_92;7R}g0`Swt#cBVOI^4HEKk2B>K7jUP-eDWOcmdkmgFzeoqM|7mX~UZPzfma2 zn*Bc<=Hox?42I7i?*I4lIK0^1G~kU1EDbyOOgQ7Hvn0UcKiU+4(_r(+Yk zLl>G~D+3oA1)VVxjqgSr$Sz2OK#E^R2uJZ-2ZWT`ARP=+R-{<`E3XgVvVim9seIO? z|BXTc)$)HkdH?@sJI@~I|GhjLr2qMN0i|B;cT~Y_Tmfq2VfR&o(+G_G zvMedy!2Yid1yaZV4W8%ifA*da9{fM<<@x&if2{~1Uxf^;5z=Gnp=~@8emT{(l1&;twQmbI*GFew4|{}m{y};l{2RAnFl0f=p)lcI@U{9vYYBwobm20 z-J|WWQvgzAXYNHw5J=Mpr3MwRPqVi=u8193y`-|`8Y+zaGjE<0!XmUG@iVhdE16w@ z!{lR^fA4DDY-zB6V{B%F!0AqIm8cEz??R<*IyRZ$5VKpg4b-oqqm%Y-s_c=G0L4ZU zBo{V}GWGuTMqxKL%mNiIz#a)1#G%MvkelqhEwH90{1J;qnha+HE- z#kX=x8D0j*v3{I6La@sRr%u=zT%$Cbr&WdJjcw|0@oU(ps_esS1W5{TK zj6?B9`zdI5Mc7?+0!tlcTN{c?$^IKoGz@!)E|~K-r^2&ARy1rSBu?{hb;6)j31TS? z@^w-rf^Ac?#4!*feh4#-xVLD5;y*m+^=&-O`M(}NJ@GUoF

Zy8GX=g8YB@YzZJ!3f^Z+M0!jdhLAb|+QAiyE& zwR(r=m*ZwiB z>R}(jzAPV;Ut90~`S$4S{C7dNn(zO!qy57-N4>efbRU)bf4IFj80PK2277xC@}GNo z9%(yA@DmE=;DFD0tMxrN4k?2{AkzihpfN~|wK++U3>EmG>i)8&%%k)%PvU96(`r3> z1YRQs&QL_aaX2BZ*45ROM?y*h)Z)hpi1{bz`#{wMydi3|G{tO|jC+{$dHJqflSbVn z@*zXK-B#=J93k)_{U)K}jp=)zPdqAqi2q&kLj8`g167O{@QIfQBMTwHzZ64IsL!qq4#NZn0D9sEBfx(IE(qA=&4aD|&<8uc;dXD>;rPQeg-e+tQ@PQ8d;zw3u|d|t zGu}e0^)Hfu8C=L=`#$J)CnWa7&s&njKulO75SGmjlt@+X-eh7RUbH0#;b)!g$kgbkew|?`26fLJI7HNovB(TwTUwTv>28;O?pk-<=Fc)^S1E7Q zY~6t??U8j=KmNDLe2ii~jiE52uPc8wtZlRmU*M`PbNp4-&nfYNpOyu49O8L02U&*q zJX|3TR)MS>E-Rt@RT*6Hqkt<3osg{wS4K$BSqvGP-j+1Q*Rd`Ns4834I=K1-+Uzz$ z?Gb375kckOi}S4wK3Bmj;0o{pg^1GAn2b?nX>g*70#O!F#F%i&u7Zo7rLPfdkH9C; z=Bzyj(zJDc3flZu&>n%E!C(MBgEkH^!!S5R0lYm&9tnNg9)Z2VQ_xm+&e_`?{&oU! zki_U>79%<%f!`hpkB2LuHV{7|8jSD`oav&CM!4z#KVJ#Ba?;Xr!0ZK)ld+b-)lTg> z+as|1eETVoZ-m)F`f6hS=1WEHpcoU;g?6PW1MLn7NgA1eQ&k0sI^qbT6i-7C5uciX zt+RuqK$ml0l;1VV_7Cspm=^_E(VEu5RgQCN3ZnrRgVVZNci`Ou#WD63FQZv#(Ewm@ z3T|dZ@Lcj7!_;JiqDW~{z}A)UN*W`6*ox!-s>27d8Z|cuL1(o?*T|(a2)2lzoG|EA zmEQndQ7ZJTopfGQ8IjkF-cAGV4$ zTpxZ}31fk9JM)U5VG^GHrEsNDXO{!c zf{T;r=5UF)pf}L&iph*?$UK3$XD+FL96o0!XG;4cV;4HX| zz@zSV?!dn9O9EyFDV<;x`22v#L%1fLCl%aUU%K=AgC4d z=veof>9ToL#yF-pOxXifI z#`Q43lyUlXd zc^z%O0|(M}4gy`L^60(;SHh8OdC&x0`OwN0^lRVuv1A8RfTX@>LSmqj&l>vql@TWK zMJU@3Fr=tPGIGb~j0%S~s?VK>_)jS4Bj)vaJJc?});!{H$`?p;{)>WRS*5P^_2yB- z)qvh>cr1Fq5C#bV#(A2*CSBEVeQWey!^Lm(HGE-;FV8~}L%!5tR>0L?L+ob4by3Aj zXK)GTnM|s|;&7b`)#}o~S#XK{R4qm-;EUH0=5Hl%eQ1j-a2I`q^C;kFiU))CCN?Q2 zZl{pV${@~w6-5OK1r>>66roTsvS@+gTQLS@(xRl@J@B5QNfJo4xc-F0Dfr;Ym8_Hj zQsuG#&vUSU@-pr4fP^e2L4e{+A$6T{h+T@O)X;Ll8`_1zxT{9i_0rn?X@Y&!S0DP> zrtUvd{daocOpEaI^T>!)1dtE`5`#I3ksK-tMVNu!rsyTB-2B#5T%4TSm@X;57F?-2 zUcEfz;Pid}4VshqHg&*D1I+XTJjd1nvnE|KjRI@v(p%!EsvQxuhWLu>6OzP2`L*CO z%=7Bz@ggnS1iCfnQQJ8-P@n8;1IbkETLafR_!9KwIK=EAWr)hpW6TV`Nh|miFQdgb z9L`O_RR@#o=e)quRe^Q~uG;NQ!S%B=EOa$@2h>NYs8#n=Va$kpfI|l-y5btk;5w;e zUL5=jUt2C({3MW}d#lf5Q+{;)0|jYSutnGm#T=2L7D}&lBQ)T0 zgkCFK_wOkx2HN6li!y!;`H(@Y{962RKCWm5Uv;7RsjLdvx{4Ow6+a@cjCiD19RZj4 zwERI&e^8$$Hh4NR!!E3ycc{Ibd<{OmrXxe9-{(K-2+4bS1u&nl?_*69_#S6mRi~!=&sn_9XK`Cl@Xk;TnqW(3_<^&F=psp=%L1htO!@t z-OzxG{hY-~v~GzxxYnUxCR`I7!T|r~K90)Z$|~i{Uwr`WjFaZvbF}cfQA`*T3~?K{ zQCM7G$@sMNdUywfNj-poCS%~^I2F+smfM<(zW^5wB|GXQuy^3tCS90{Jg%zjQ+{2M zRG-LGQ(6e|E0+$D2yob)ONN&bfag(67Ghiz63-!Tb9G`|H5^X=PB=e(We(?DzZFL+ z2Q~xG1{gCikXLg50(MBXtMFve0e96?HvxKC+Ansu*t;X=5U zy^2YJQ_I8UglH0zIe-}|L@_@DO<@;|aDOrE4WIWPp>VN)vFx_H7ml97eVD{6uNl3WK0uYZUy-ZH^)h3tM4J;}}I&?OPbg847(A*R88NaE?MM z48|L6d>H!j)+utO{SLKX49nC9207R2s@_6^^uOCad7m;Y@R2tvHqPb`vC1Tqh`( zaGlDciY^5Na>4`y5>BPsNZXL=@@uUx&aX96d)%dt9oecfDuXRezh*H>rZd2zptBn5 zB8B(?iuo#TAjB`ir7Ld`s7x8d@B~+i$;;tND?7`t#pR_=cc&^rN;$oE&7-QU(;mch z<8aj>><%2JMU{-9)Ie{eaMkc2cP6)yGObF}AbAOzA#jb$jf6{kGvcsDPJ5^6z!M6h z7|&sRTd5*l6D~7fP3wwyA0e=$>DmYlOUgwu+f zFc?FAa>|w&nFm+G;rl6IGfaVCH>K0v41sHOD+_i-!Jfsbiw5MBG@nX=wHj6TngWMf zd!Vcz!waQM=xf8Z))H%=QY*sMz&2=IxwMtNsPe{jO8h2>+8A6-w2&IEhDKByf$Ox+ zgkAMx>LD}Lac0l1HQXs&OFHXVa0PgRyjw3oKM_)T9RljksfvObFRQHr&YAX4j$(Aq zNCZ9{Ndu)yQ3r$cn)dakM%{?`-Bfhfhm1jQ)}6u74^Z4qq3ogs3R(Y=k0vA^We|!92 z?~g>%!ngZxj?SGzogO%z0E!rAzLjV|$j?+8H#i8CYz-s=6u}lrBTnfThOro@iKV4egFM*$M2gpl4zxl=8>Xdy3S%}dzV z%qm-WUMdHU8ThuPB8r0V)4q%-LiAVBSz6OWH zQy5|@19S2PeKW(Jj4@(NB%P5Y@H63W)9?tja7{>%4f771kSX?H0OV`FLL@jEX;YCQ zB6o>g;Q*+G%g2iH+c<5eh#{&u57h%3Gbix zSVl0Q0xq77sGv2p2mXoK4E%Hd;^3#^;pGzm)iQ#=LxW>^%Kz~Lto;Yoh z#i=bcPH`fQ9ItT%SRlge;E9%*@b37f0>B6ZPej^UR#RPuaB(GcaQEU`nToh3cnaqH z$QHON(oY?N$fTo+v|Xm9>*csrFbV~BDWF1Iuok!th;12s7OgixdzBJB$s;pFG)PZ{OP=H(q5Q_i<_dNSss@c9u7IWvM5i?q+?Q9mb-tsYe- z_PA1`x|o?x!Nolm5OOV6EDkfBlaS{U6$e9|FqGj?uWx}Uq0i^^XH9G4W++PsqXTa! zC^%2^G^GTdcDtm>SQSV@aRAIvCQ$U10p^%Vjz0KLte_g&mI}!Va z|NZ9Z;-`0qB{nx16Klbd>S@a~Ud^^~rtV!?WKaf|n{%t##u+>fHnA?e)S}D)be{#=u7*!!V$|#+UUBu7slaAE??vQNneNZZ!*;!X+SZ1IHWYLyBHMSqfLF?)NU2w5(3`d$&A56>!PRiCeIs;>F|(gL1YD_=b!7}32^SSvP9>L?Wd=?!$4ZW`Z|5r9 zUs^@}Hgdw!)Dui|QX}MMsLb@m5Ms6eXX7ASP1NrRP2+jQj)W3dvoR z{!T4$yI`C!K$(<2%CpGKf+@ygBu3We=V^C*Ma)o)dX7~P(dei?;*eSer^G*3Dod#h z23vw9v=-D;j=YeG?kR{+Jjav@Vg@O=L4mZLbT1PNuJ;t}N6~&fFAG?~`DYocT$cm# zFc!PT0c;v@yfF4o_1ewg5)uJE*=I>mrh2yI18MvyBff!HfoAy+zmy`s^2xzvEN~Zi zwQ!jmWPP|)toSk<9SVz-G@`t`Kq#@255~8q)w%LdGn^h*lkdrhBd(l-?i28znMj%JNV;(32;rL4&ylmM40obzccKel6)b3ax$-ix7p#u3)CjIh!WW#JuuHhF83yL3nC81?1H!#i7hms($^P2}O?_}aQM({GAmR~2F7LGD8NMkW}m zIOk=c8C6{6Ba@a4QB3BX0Z1qy5sD%Iu(q#34z6_U4uM5FU@520frzP2Q_R_Paqqv8DkYrnoGgO0_B_JSYO3}h~t2G z*Xb=-t8!AfHV_C~!&UC5q0|rAWjwd;GX}(1c<-|PDP?3$V0F(UUpp5vUrg@YR&SL$ zE3*bv>kU(mF&)Prhyd`*F0E3y_?=^n0vt|@*Sa@iHJ!;4a<=8wE{XyXYKIE<0t#fx zae%`qqB7LzQl6jkx{0JR?k7w+^N^3LW5Si?l_G%;&ZOy>G%_Y2Mx4R1%3lrFNPHj; zI_X7^s>pq&cat%Hc(rb<>Y1m~-|8}3$um#cCx z-B-U7FzdV~YQf|jwo9Ix7AXVM?LtukX47mmrC`>vbfsXrQ`5M?%t9nDlmT{lmS(-IgFsu`QreyD%DsOn7SVDM82`5zsZ_xFpIV{IWPry36-K`V+AY;<`07(D#eo( zV7f92Rbx|wyX5q*24+jCfiUIQb3iJu4?q>~evMGn7|c4R z-T`Ka*bRxVCCk$j0smAG|Er8{Lh#lsCer81nBBL!sxFiFB}I2oyda>YwVm8Yy!Pp4rPRcu8EHy8P&@p-u#>rs`V;GUBb z*nm}~D7fdW1UV93k)zO+t690KbDux$ysy(`(E!pXY32-GO~mdnROU`m6NpkaYvT zaQN#?Jg4-aUwWZm>GNL1++lf)|HSiL8*EcZs z56C=%43BYOQx%!kS*-M`!yVBM$7aH{1a}L6mQyGy+mF z&LkizeZReD?^>-^>(Qe}Kt)VRXI$7JaSpK665gBo+}!K!NW1wlS)kU{)zvMW2Q4E$ z)~E$kz&`#L06v*R9FBmJHsf&G)9Ir;G8aW+LfB;z2P4q_>sF+g;|?A5`+wb%)jL#6 z*V>|n=-`rXo=cjHFFBke;jr0K@bkXano}QuNS8zti%ph zeHc#@aqwUYrg(uu=@Tn@$25(Z*lVS8)h)u#ksH;^k%A`Q8ukvH+(jgiKblbyo1abLcRmwS4zSWxNB+J^r@B6 zskUJ=-5#}!FTGyR`1J^IN*5r8M03JbRG3EQYM4zk%mUONf%Z8ccdC7FKQ#*_*ylyy z*z(t(tJ2dDv15M(^lyC! z&E8}lTO&iKCq4?Nup)v4_0|2y@J3~{ODKvx5^@42py9v!nZl$$hm@h%1Tr^jy>njf zpC9e*T06%GESa)ADzq{QK^pZ{I8E^bPLRnMu_O+`|2cp6R!u~3E4GYbqKtJhKHD!L zMSHsdg&y&dFQNMiP)ZPTdp=S$vy;=Od3uOKU2Wj2QV^w@CGw zEB=l(c` zp*KUc|4IJU#s25MXuto7|08}YwEw91CA$f}I9h30kWgvL#-V%Cf1*s;aI7uXM0{_7 zlUwlS{rLr7lrf3|=t)W9U$S-^(o`8dgx)^W)=q*!OX~oo$_# zUL|rNVOohY-tk|zU-7Ope5s8oQN_FX>-H<&3I9iTlOj&+P#r_d4y)9X3k@0jqcVn* z^p|+bw{JeX8~-Nd#+eDBqI9i@_$p39R$Rm#K_UsY>36tAbfj_G(amUAPlaTLVhTti z-I7R%BsbfF>P`t2H!Z;^kHFCrj#qLu8x`1$P-H^H&+wZVGls&JJXd5;0>LJWv{9D* zbq^fKxF~w_<(g8ADe^@FQo<>t{j|(*SFwr+gnZ?`>i1f$eGns$%;zZd?wW zsj<5<18nH~z>EnGpbtf|XCZp}pzb3wQtGHLLZfOOQIEt^88QGNPL*xPj!r~IFk@k{$tEid$cw6sB4oiv%Uude)( zR=kd3G{X#4HE3XqU!-#_0r7IgS5$3FM%ut0r;HzOIfEuSSFm+1SzHo>mod4aSzw~~ z$DKl_vZ;}5CF>J&&WFTPfc>6|sHNQV7~$`aeK6$wir^0gw~Esy#pmo6m5$&?&VA%> z23ACg9|k`RinTc2#(Zb!RsEFroY1sG{k|}rMqR!?^7Z(;syZq_kX1*2H0q5B34T;^ zp}q>c2gZ(NTURp_%(a8VmAvFaiKQ$8O>QcAKO*iT%i7-r!z41nlngV+ zK(!PD1u_OoL^Qbka2?-^Ei&W-z_f!sd+HQWW}^ zF`DDhN3ro0Uf z%%d1h@JFzv*L0`VIt~Geed*8buBhsu-n92rO0h=$zM*f^js4hpDjnx7i6JB4mLys@ zCvFOcBCA5xXQ(gaeA@XRRL!0+S{;FQIK|;dc?63;wZ6P(SBnv+_NXfj5l5i^cvxE} zp{N_9Df&19Z4O+!m1-DM;_$z=df#{Q3d-vrXsVd>(V|tQW6cz(?$B@g+;}9(G5?^? zkhElRA#jc-xHYmX=@Z6~=im^xTH-`kuyVpo(`E2M2$s|IRVx!p%c-jWNC;AOs~xrb zh10Sj#ueVwj>LwxgVrQ~Q?S)AsbR;?#hsmq#}-BmVx60`51jw+iPWmtyCnSAFWK@1 zFZBAAHZYf%zj}Om$;bPtkC@k2UG{m+ORJrJTIX}tfP#MMMr4W4a!nlBA-kUP`)V3O zCrg0G`3qy@-EP+{$dj|-km7Lwx(b}G0H-T^Y{?_+T?GG~pq6CfH5;CedIbJcO#YAg z(PtmTXpTcKNPMKZ$7U;JR8XIrHB9Mf zL|=N{XcQf&!)tz;WE@~RlY>4(ldKuFP_GwVFGfB!tQ+`DYhy@}@UnXx zsu`BLZ^c?sNwLHX#%N9$vW&xVICainwWC~r9FVbAT%8^5AHF&2&HYDl5_WOemEQNp zq*|7nMIIns<+)K=hLNq-%iBy`=CURCsgwx_%PN>RN0Wwo!p2x!&uMBtWtmF5?2mOI zED7XKcsc|73y1@#EW!AhmV`oBmnO~>SM*zwfEy^{>w!24)1wxL0D>8B2b%5Z+YYZ5 zA}N%Sm=agb8G^nDYOcKx#;JKLxY8e$9ht=J74J!KWmg7UcEzvW{OYt%L5MywKpBce zr#hM`MJ$OV$+fa6I$M*Yxq-JWaY@yYzvocO&$Y0(06imsjD_4C27xGYd`ea6mO@1j z{EH;ov0mwR!PV8LPcjATd^CwE|;RRv%M zFC)aS0U(2+tA{rFkHLTJa*o#BNG90%?&0|}K2qT?4nJXzkeN011<9rk8#lLwQh6VP z!C8qfb900960vQFj>0B#omitGyE literal 0 HcmV?d00001 diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/Chart.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/Chart.yaml new file mode 100644 index 0000000000..ae66e88ab6 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/Chart.yaml @@ -0,0 +1,32 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Project Monitoring + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.25.0-0' + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/rancher-version: '>= 2.6.5-0 < 2.7.0-0' + catalog.cattle.io/release-name: rancher-project-monitoring +apiVersion: v2 +appVersion: 0.59.1 +dependencies: +- condition: grafana.enabled + name: grafana + repository: file://./charts/grafana +description: Collects several related Helm charts, Grafana dashboards, and Prometheus + rules combined with documentation and scripts to provide easy to operate end-to-end + Kubernetes cluster monitoring with Prometheus. Depends on the existence of a Cluster + Prometheus deployed via Prometheus Operator +home: https://github.com/prometheus-operator/kube-prometheus +icon: https://raw.githubusercontent.com/prometheus/prometheus.github.io/master/assets/prometheus_logo-cb55bb5c346.png +keywords: +- prometheus +- monitoring +maintainers: +- email: arvind.iyengar@suse.com + name: Arvind +- email: amangeet.samra@suse.com + name: Geet + url: https://github.com/geethub97 +name: rancher-project-monitoring +type: application +version: 0.3.0+up0.3.3 diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/README.md b/charts/rancher-project-monitoring/0.3.0+up0.3.3/README.md new file mode 100644 index 0000000000..eab333c34c --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/README.md @@ -0,0 +1,29 @@ +# rancher-project-monitoring + +This chart installs a project-scoped version of [`rancher-monitoring`](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting), a Helm chart based off of [`kube-prometheus stack`](https://github.com/prometheus-operator/kube-prometheus). It deploys a collection of Kubernetes manifests, [Grafana](http://grafana.com/) dashboards, and [Prometheus rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) combined with documentation and scripts to provide easy to operate end-to-end Kubernetes project monitoring with [Prometheus](https://prometheus.io/) using the [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). See the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) README for details about components, dashboards, and alerts. + +## Prerequisites + +- Kubernetes 1.16+ +- Helm 3+ + +## Install Chart + +This chart is not intended for standalone use; it's intended to be deployed via [Prometheus Federator](https://github.com/rancher/prometheus-federator). For a Prometheus Stack intended to be deployed standalone, please use [rancher-monitoring](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting/) or the upstream [`kube-prometheus-stack`](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) project. + +## Dependencies + +This chart is designed to be deployed alongside an existing Prometheus Operator deployment in a cluster that has already installed the Prometheus Operator CRDs. Specifically, the chart is configured and intended to be deployed alongside [`rancher-monitoring`](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting/), which deploys Prometheus Operator alongside a Cluster Prometheus that `rancher-project-monitoring` is configured to federate namespace-scoped metrics from by default. + +### Configuration + +Since this chart installs a project-scoped version of [`rancher-monitoring`](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting/), a Helm chart based off of [`kube-prometheus-stack`](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack), most of the options that apply to either of those charts will apply to this chart (e.g. support for configuring persistent volumes, ingresses, etc.) and can be passed in as part of the `spec.values` of the ProjectHelmChart that deploys this chart; however, certain advanced functionality (such as Thanos support) and options that pose security risks in Project environments (e.g. ability to `ignoreNamespaceSelectors` or modify the existing namepaceSelectors of the Cluster Prometheus, ability to mount additional scrape configs, etc.) have been removed from the `values.yaml` of the chart. For more information on how to configure values and what they mean, please see the comments and options provided on the `values.yaml` packaged with this chart. + +## Further Information + +For more in-depth documentation of configuration options meanings, please see + +- [`rancher-monitoring`](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting/) +- [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) +- [Prometheus](https://prometheus.io/docs/introduction/overview/) +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana#grafana-helm-chart) \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/app-README.md b/charts/rancher-project-monitoring/0.3.0+up0.3.3/app-README.md new file mode 100644 index 0000000000..6b83329151 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/app-README.md @@ -0,0 +1,10 @@ +# Rancher Project Monitoring and Alerting + +The chart installs a Project Monitoring Stack, which contains: +- [Prometheus](https://prometheus.io/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator)) +- [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator)) +- [Grafana](https://github.com/helm/charts/tree/master/stable/grafana) (deployed via an embedded Helm chart) +- Default PrometheusRules and Grafana dashboards based on the collection of community-curated resources from [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) +- Default ServiceMonitors that watch the deployed resources + +Note: This chart is not intended for standalone use; it's intended to be deployed via [Prometheus Federator](https://github.com/rancher/prometheus-federator). \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/.helmignore b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/.helmignore new file mode 100644 index 0000000000..8cade1318f --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.vscode +.project +.idea/ +*.tmproj +OWNERS diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/Chart.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/Chart.yaml new file mode 100644 index 0000000000..56b23171a7 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/Chart.yaml @@ -0,0 +1,30 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.25.0-0' + catalog.cattle.io/os: linux + catalog.cattle.io/rancher-version: '>= 2.6.5-0 < 2.7.0-0' + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-grafana +apiVersion: v2 +appVersion: 9.1.5 +description: The leading tool for querying and visualizing time series and metrics. +home: https://grafana.net +icon: https://raw.githubusercontent.com/grafana/grafana/master/public/img/logo_transparent_400x.png +kubeVersion: ^1.8.0-0 +maintainers: +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rluckie@cisco.com + name: rtluckie +- email: maor.friedman@redhat.com + name: maorfr +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: mail@torstenwalter.de + name: torstenwalter +name: grafana +sources: +- https://github.com/grafana/grafana +type: application +version: 6.38.6 diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/README.md b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/README.md new file mode 100644 index 0000000000..f3d317688a --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/README.md @@ -0,0 +1,571 @@ +# Grafana Helm Chart + +* Installs the web dashboarding system [Grafana](http://grafana.org/) + +## Get Repo Info + +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm install my-release grafana/grafana +``` + +## Uninstalling the Chart + +To uninstall/delete the my-release deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an +incompatible breaking change needing manual actions. + +### To 4.0.0 (And 3.12.1) + +This version requires Helm >= 2.12.0. + +### To 5.0.0 + +You have to add --force to your helm upgrade command as the labels of the chart have changed. + +### To 6.0.0 + +This version requires Helm >= 3.1.0. + +## Configuration + +| Parameter | Description | Default | +|-------------------------------------------|-----------------------------------------------|---------------------------------------------------------| +| `replicas` | Number of nodes | `1` | +| `podDisruptionBudget.minAvailable` | Pod disruption minimum available | `nil` | +| `podDisruptionBudget.maxUnavailable` | Pod disruption maximum unavailable | `nil` | +| `deploymentStrategy` | Deployment strategy | `{ "type": "RollingUpdate" }` | +| `livenessProbe` | Liveness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } "initialDelaySeconds": 60, "timeoutSeconds": 30, "failureThreshold": 10 }` | +| `readinessProbe` | Readiness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } }`| +| `securityContext` | Deployment securityContext | `{"runAsUser": 472, "runAsGroup": 472, "fsGroup": 472}` | +| `priorityClassName` | Name of Priority Class to assign pods | `nil` | +| `image.repository` | Image repository | `grafana/grafana` | +| `image.tag` | Overrides the Grafana image tag whose default is the chart appVersion (`Must be >= 5.0.0`) | `` | +| `image.sha` | Image sha (optional) | `` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Image pull secrets (can be templated) | `[]` | +| `service.enabled` | Enable grafana service | `true` | +| `service.type` | Kubernetes service type | `ClusterIP` | +| `service.port` | Kubernetes port where service is exposed | `80` | +| `service.portName` | Name of the port on the service | `service` | +| `service.appProtocol` | Adds the appProtocol field to the service | `` | +| `service.targetPort` | Internal service is port | `3000` | +| `service.nodePort` | Kubernetes service nodePort | `nil` | +| `service.annotations` | Service annotations (can be templated) | `{}` | +| `service.labels` | Custom labels | `{}` | +| `service.clusterIP` | internal cluster service IP | `nil` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `nil` | +| `service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to lb (if supported) | `[]` | +| `service.externalIPs` | service external IP addresses | `[]` | +| `headlessService` | Create a headless service | `false` | +| `extraExposePorts` | Additional service ports for sidecar containers| `[]` | +| `hostAliases` | adds rules to the pod's /etc/hosts | `[]` | +| `ingress.enabled` | Enables Ingress | `false` | +| `ingress.annotations` | Ingress annotations (values are templated) | `{}` | +| `ingress.labels` | Custom labels | `{}` | +| `ingress.path` | Ingress accepted path | `/` | +| `ingress.pathType` | Ingress type of path | `Prefix` | +| `ingress.hosts` | Ingress accepted hostnames | `["chart-example.local"]` | +| `ingress.extraPaths` | Ingress extra paths to prepend to every host configuration. Useful when configuring [custom actions with AWS ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-alb-ingress-controller/guide/ingress/annotation/#actions). Requires `ingress.hosts` to have one or more host entries. | `[]` | +| `ingress.tls` | Ingress TLS configuration | `[]` | +| `resources` | CPU/Memory resource requests/limits | `{}` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Affinity settings for pod assignment | `{}` | +| `extraInitContainers` | Init containers to add to the grafana pod | `{}` | +| `extraContainers` | Sidecar containers to add to the grafana pod | `""` | +| `extraContainerVolumes` | Volumes that can be mounted in sidecar containers | `[]` | +| `extraLabels` | Custom labels for all manifests | `{}` | +| `schedulerName` | Name of the k8s scheduler (other than default) | `nil` | +| `persistence.enabled` | Use persistent volume to store data | `false` | +| `persistence.type` | Type of persistence (`pvc` or `statefulset`) | `pvc` | +| `persistence.size` | Size of persistent volume claim | `10Gi` | +| `persistence.existingClaim` | Use an existing PVC to persist data (can be templated) | `nil` | +| `persistence.storageClassName` | Type of persistent volume claim | `nil` | +| `persistence.accessModes` | Persistence access modes | `[ReadWriteOnce]` | +| `persistence.annotations` | PersistentVolumeClaim annotations | `{}` | +| `persistence.finalizers` | PersistentVolumeClaim finalizers | `[ "kubernetes.io/pvc-protection" ]` | +| `persistence.subPath` | Mount a sub dir of the persistent volume (can be templated) | `nil` | +| `persistence.inMemory.enabled` | If persistence is not enabled, whether to mount the local storage in-memory to improve performance | `false` | +| `persistence.inMemory.sizeLimit` | SizeLimit for the in-memory local storage | `nil` | +| `initChownData.enabled` | If false, don't reset data ownership at startup | true | +| `initChownData.image.repository` | init-chown-data container image repository | `busybox` | +| `initChownData.image.tag` | init-chown-data container image tag | `1.31.1` | +| `initChownData.image.sha` | init-chown-data container image sha (optional)| `""` | +| `initChownData.image.pullPolicy` | init-chown-data container image pull policy | `IfNotPresent` | +| `initChownData.resources` | init-chown-data pod resource requests & limits | `{}` | +| `schedulerName` | Alternate scheduler name | `nil` | +| `env` | Extra environment variables passed to pods | `{}` | +| `envValueFrom` | Environment variables from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `envFromSecrets` | List of Kubernetes secrets (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envFromConfigMaps` | List of Kubernetes ConfigMaps (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envRenderSecret` | Sensible environment variables passed to pods and stored as secret | `{}` | +| `enableServiceLinks` | Inject Kubernetes services as environment variables. | `true` | +| `extraSecretMounts` | Additional grafana server secret mounts | `[]` | +| `extraVolumeMounts` | Additional grafana server volume mounts | `[]` | +| `createConfigmap` | Enable creating the grafana configmap | `true` | +| `extraConfigmapMounts` | Additional grafana server configMap volume mounts (values are templated) | `[]` | +| `extraEmptyDirMounts` | Additional grafana server emptyDir volume mounts | `[]` | +| `plugins` | Plugins to be loaded along with Grafana | `[]` | +| `datasources` | Configure grafana datasources (passed through tpl) | `{}` | +| `alerting` | Configure grafana alerting (passed through tpl) | `{}` | +| `notifiers` | Configure grafana notifiers | `{}` | +| `dashboardProviders` | Configure grafana dashboard providers | `{}` | +| `dashboards` | Dashboards to import | `{}` | +| `dashboardsConfigMaps` | ConfigMaps reference that contains dashboards | `{}` | +| `grafana.ini` | Grafana's primary configuration | `{}` | +| `ldap.enabled` | Enable LDAP authentication | `false` | +| `ldap.existingSecret` | The name of an existing secret containing the `ldap.toml` file, this must have the key `ldap-toml`. | `""` | +| `ldap.config` | Grafana's LDAP configuration | `""` | +| `annotations` | Deployment annotations | `{}` | +| `labels` | Deployment labels | `{}` | +| `podAnnotations` | Pod annotations | `{}` | +| `podLabels` | Pod labels | `{}` | +| `podPortName` | Name of the grafana port on the pod | `grafana` | +| `lifecycleHooks` | Lifecycle hooks for podStart and preStop [Example](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/#define-poststart-and-prestop-handlers) | `{}` | +| `sidecar.image.repository` | Sidecar image repository | `quay.io/kiwigrid/k8s-sidecar` | +| `sidecar.image.tag` | Sidecar image tag | `1.19.2` | +| `sidecar.image.sha` | Sidecar image sha (optional) | `""` | +| `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` | +| `sidecar.resources` | Sidecar resources | `{}` | +| `sidecar.securityContext` | Sidecar securityContext | `{}` | +| `sidecar.enableUniqueFilenames` | Sets the kiwigrid/k8s-sidecar UNIQUE_FILENAMES environment variable. If set to `true` the sidecar will create unique filenames where duplicate data keys exist between ConfigMaps and/or Secrets within the same or multiple Namespaces. | `false` | +| `sidecar.dashboards.enabled` | Enables the cluster wide search for dashboards and adds/updates/deletes them in grafana | `false` | +| `sidecar.dashboards.SCProvider` | Enables creation of sidecar provider | `true` | +| `sidecar.dashboards.provider.name` | Unique name of the grafana provider | `sidecarProvider` | +| `sidecar.dashboards.provider.orgid` | Id of the organisation, to which the dashboards should be added | `1` | +| `sidecar.dashboards.provider.folder` | Logical folder in which grafana groups dashboards | `""` | +| `sidecar.dashboards.provider.disableDelete` | Activate to avoid the deletion of imported dashboards | `false` | +| `sidecar.dashboards.provider.allowUiUpdates` | Allow updating provisioned dashboards from the UI | `false` | +| `sidecar.dashboards.provider.type` | Provider type | `file` | +| `sidecar.dashboards.provider.foldersFromFilesStructure` | Allow Grafana to replicate dashboard structure from filesystem. | `false` | +| `sidecar.dashboards.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.skipTlsVerify` | Set to true to skip tls verification for kube api calls | `nil` | +| `sidecar.dashboards.label` | Label that config maps with dashboards should have to be added | `grafana_dashboard` | +| `sidecar.dashboards.labelValue` | Label value that config maps with dashboards should have to be added | `""` | +| `sidecar.dashboards.folder` | Folder in the pod that should hold the collected dashboards (unless `sidecar.dashboards.defaultFolderName` is set). This path will be mounted. | `/tmp/dashboards` | +| `sidecar.dashboards.folderAnnotation` | The annotation the sidecar will look for in configmaps to override the destination folder for files | `nil` | +| `sidecar.dashboards.defaultFolderName` | The default folder name, it will create a subfolder under the `sidecar.dashboards.folder` and put dashboards in there instead | `nil` | +| `sidecar.dashboards.script` | Absolute path to shell script to execute after a configmap got reloaded. | `nil` | +| `sidecar.dashboards.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.dashboards.extraMounts` | Additional dashboard sidecar volume mounts. | `[]` | +| `sidecar.datasources.enabled` | Enables the cluster wide search for datasources and adds/updates/deletes them in grafana |`false` | +| `sidecar.datasources.label` | Label that config maps with datasources should have to be added | `grafana_datasource` | +| `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `""` | +| `sidecar.datasources.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.datasources.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/datasources/reload"` | +| `sidecar.datasources.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana | `false` | +| `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` | +| `sidecar.notifiers.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `smtp.existingSecret` | The name of an existing secret containing the SMTP credentials. | `""` | +| `smtp.userKey` | The key in the existing SMTP secret containing the username. | `"user"` | +| `smtp.passwordKey` | The key in the existing SMTP secret containing the password. | `"password"` | +| `admin.existingSecret` | The name of an existing secret containing the admin credentials (can be templated). | `""` | +| `admin.userKey` | The key in the existing admin secret containing the username. | `"admin-user"` | +| `admin.passwordKey` | The key in the existing admin secret containing the password. | `"admin-password"` | +| `serviceAccount.autoMount` | Automount the service account token in the pod| `true` | +| `serviceAccount.annotations` | ServiceAccount annotations | | +| `serviceAccount.create` | Create service account | `true` | +| `serviceAccount.name` | Service account name to use, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `` | +| `serviceAccount.nameTest` | Service account name to use for test, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `nil` | +| `rbac.create` | Create and use RBAC resources | `true` | +| `rbac.namespaced` | Creates Role and Rolebinding instead of the default ClusterRole and ClusteRoleBindings for the grafana instance | `false` | +| `rbac.useExistingRole` | Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to the rolename set here. | `nil` | +| `rbac.pspEnabled` | Create PodSecurityPolicy (with `rbac.create`, grant roles permissions as well) | `true` | +| `rbac.pspUseAppArmor` | Enforce AppArmor in created PodSecurityPolicy (requires `rbac.pspEnabled`) | `true` | +| `rbac.extraRoleRules` | Additional rules to add to the Role | [] | +| `rbac.extraClusterRoleRules` | Additional rules to add to the ClusterRole | [] | +| `command` | Define command to be executed by grafana container at startup | `nil` | +| `testFramework.enabled` | Whether to create test-related resources | `true` | +| `testFramework.image` | `test-framework` image repository. | `bats/bats` | +| `testFramework.tag` | `test-framework` image tag. | `v1.4.1` | +| `testFramework.imagePullPolicy` | `test-framework` image pull policy. | `IfNotPresent` | +| `testFramework.securityContext` | `test-framework` securityContext | `{}` | +| `downloadDashboards.env` | Environment variables to be passed to the `download-dashboards` container | `{}` | +| `downloadDashboards.envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `downloadDashboards.resources` | Resources of `download-dashboards` container | `{}` | +| `downloadDashboardsImage.repository` | Curl docker image repo | `curlimages/curl` | +| `downloadDashboardsImage.tag` | Curl docker image tag | `7.73.0` | +| `downloadDashboardsImage.sha` | Curl docker image sha (optional) | `""` | +| `downloadDashboardsImage.pullPolicy` | Curl docker image pull policy | `IfNotPresent` | +| `namespaceOverride` | Override the deployment namespace | `""` (`Release.Namespace`) | +| `serviceMonitor.enabled` | Use servicemonitor from prometheus operator | `false` | +| `serviceMonitor.namespace` | Namespace this servicemonitor is installed in | | +| `serviceMonitor.interval` | How frequently Prometheus should scrape | `1m` | +| `serviceMonitor.path` | Path to scrape | `/metrics` | +| `serviceMonitor.scheme` | Scheme to use for metrics scraping | `http` | +| `serviceMonitor.tlsConfig` | TLS configuration block for the endpoint | `{}` | +| `serviceMonitor.labels` | Labels for the servicemonitor passed to Prometheus Operator | `{}` | +| `serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `30s` | +| `serviceMonitor.relabelings` | MetricRelabelConfigs to apply to samples before ingestion. | `[]` | +| `revisionHistoryLimit` | Number of old ReplicaSets to retain | `10` | +| `imageRenderer.enabled` | Enable the image-renderer deployment & service | `false` | +| `imageRenderer.image.repository` | image-renderer Image repository | `grafana/grafana-image-renderer` | +| `imageRenderer.image.tag` | image-renderer Image tag | `latest` | +| `imageRenderer.image.sha` | image-renderer Image sha (optional) | `""` | +| `imageRenderer.image.pullPolicy` | image-renderer ImagePullPolicy | `Always` | +| `imageRenderer.env` | extra env-vars for image-renderer | `{}` | +| `imageRenderer.serviceAccountName` | image-renderer deployment serviceAccountName | `""` | +| `imageRenderer.securityContext` | image-renderer deployment securityContext | `{}` | +| `imageRenderer.hostAliases` | image-renderer deployment Host Aliases | `[]` | +| `imageRenderer.priorityClassName` | image-renderer deployment priority class | `''` | +| `imageRenderer.service.enabled` | Enable the image-renderer service | `true` | +| `imageRenderer.service.portName` | image-renderer service port name | `http` | +| `imageRenderer.service.port` | image-renderer port used by deployment | `8081` | +| `imageRenderer.service.targetPort` | image-renderer service port used by service | `8081` | +| `imageRenderer.appProtocol` | Adds the appProtocol field to the service | `` | +| `imageRenderer.grafanaSubPath` | Grafana sub path to use for image renderer callback url | `''` | +| `imageRenderer.podPortName` | name of the image-renderer port on the pod | `http` | +| `imageRenderer.revisionHistoryLimit` | number of image-renderer replica sets to keep | `10` | +| `imageRenderer.networkPolicy.limitIngress` | Enable a NetworkPolicy to limit inbound traffic from only the created grafana pods | `true` | +| `imageRenderer.networkPolicy.limitEgress` | Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods | `false` | +| `imageRenderer.resources` | Set resource limits for image-renderer pdos | `{}` | +| `imageRenderer.nodeSelector` | Node labels for pod assignment | `{}` | +| `imageRenderer.tolerations` | Toleration labels for pod assignment | `[]` | +| `imageRenderer.affinity` | Affinity settings for pod assignment | `{}` | +| `networkPolicy.enabled` | Enable creation of NetworkPolicy resources. | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | +| `networkPolicy.explicitNamespacesSelector` | A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed | `{}` | +| `networkPolicy.ingress` | Enable the creation of an ingress network policy | `true` | +| `networkPolicy.egress.enabled` | Enable the creation of an egress network policy | `false` | +| `networkPolicy.egress.ports` | An array of ports to allow for the egress | `[]` | +| `enableKubeBackwardCompatibility` | Enable backward compatibility of kubernetes where pod's defintion version below 1.13 doesn't have the enableServiceLinks option | `false` | + + + +### Example ingress with path + +With grafana 6.3 and above +```yaml +grafana.ini: + server: + domain: monitoring.example.com + root_url: "%(protocol)s://%(domain)s/grafana" + serve_from_sub_path: true +ingress: + enabled: true + hosts: + - "monitoring.example.com" + path: "/grafana" +``` + +### Example of extraVolumeMounts + +Volume can be type persistentVolumeClaim or hostPath but not both at same time. +If neither existingClaim or hostPath argument is given then type is emptyDir. + +```yaml +- extraVolumeMounts: + - name: plugins + mountPath: /var/lib/grafana/plugins + subPath: configs/grafana/plugins + existingClaim: existing-grafana-claim + readOnly: false + - name: dashboards + mountPath: /var/lib/grafana/dashboards + hostPath: /usr/shared/grafana/dashboards + readOnly: false +``` + +## Import dashboards + +There are a few methods to import dashboards to Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +dashboards: + default: + some-dashboard: + json: | + { + "annotations": + + ... + # Complete json file here + ... + + "title": "Some Dashboard", + "uid": "abcd1234", + "version": 1 + } + custom-dashboard: + # This is a path to a file inside the dashboards directory inside the chart directory + file: dashboards/custom-dashboard.json + prometheus-stats: + # Ref: https://grafana.com/dashboards/2 + gnetId: 2 + revision: 2 + datasource: Prometheus + local-dashboard: + url: https://raw.githubusercontent.com/user/repository/master/dashboards/dashboard.json +``` + +## BASE64 dashboards + +Dashboards could be stored on a server that does not return JSON directly and instead of it returns a Base64 encoded file (e.g. Gerrit) +A new parameter has been added to the url use case so if you specify a b64content value equals to true after the url entry a Base64 decoding is applied before save the file to disk. +If this entry is not set or is equals to false not decoding is applied to the file before saving it to disk. + +### Gerrit use case + +Gerrit API for download files has the following schema: where {project-name} and +{file-id} usually has '/' in their values and so they MUST be replaced by %2F so if project-name is user/repo, branch-id is master and file-id is equals to dir1/dir2/dashboard +the url value is + +## Sidecar for dashboards + +If the parameter `sidecar.dashboards.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster and filters out the ones with +a label as defined in `sidecar.dashboards.label`. The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported +dashboards are deleted/updated. + +A recommendation is to use one configmap per dashboard, as a reduction of multiple dashboards inside +one configmap is currently not properly mirrored in grafana. + +Example dashboard config: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-dashboard + labels: + grafana_dashboard: "1" +data: + k8s-dashboard.json: |- + [...] +``` + +## Sidecar for datasources + +If the parameter `sidecar.datasources.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.datasources.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the data sources in grafana can be imported. + +Secrets are recommended over configmaps for this usecase because datasources usually contain private +data like usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example values to add a datasource adapted from [Grafana](http://docs.grafana.org/administration/provisioning/#example-datasource-config-file): + +```yaml +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + # name of the datasource. Required + - name: Graphite + # datasource type. Required + type: graphite + # access mode. proxy or direct (Server or Browser in the UI). Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://localhost:8080 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: + # basic auth username + basicAuthUser: + # basic auth password + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: + # fields that will be converted to json and stored in json_data + jsonData: + graphiteVersion: "1.1" + tlsAuth: true + tlsAuthWithCACert: true + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: false +``` + +## Sidecar for notifiers + +If the parameter `sidecar.notifiers.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.notifiers.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the notification channels in grafana can be imported. The secrets must be created before +`helm install` so that the notifiers init container can list the secrets. + +Secrets are recommended over configmaps for this usecase because alert notification channels usually contain +private data like SMTP usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example datasource config adapted from [Grafana](https://grafana.com/docs/grafana/latest/administration/provisioning/#alert-notification-channels): + +```yaml +notifiers: + - name: notification-channel-1 + type: slack + uid: notifier1 + # either + org_id: 2 + # or + org_name: Main Org. + is_default: true + send_reminder: true + frequency: 1h + disable_resolve_message: false + # See `Supported Settings` section for settings supporter for each + # alert notification type. + settings: + recipient: 'XXX' + token: 'xoxb' + uploadImage: true + url: https://slack.com + +delete_notifiers: + - name: notification-channel-1 + uid: notifier1 + org_id: 2 + - name: notification-channel-2 + # default org_id: 1 +``` + +## How to serve Grafana with a path prefix (/grafana) + +In order to serve Grafana with a prefix (e.g., ), add the following to your values.yaml. + +```yaml +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/use-regex: "true" + + path: /grafana/?(.*) + hosts: + - k8s.example.dev + +grafana.ini: + server: + root_url: http://localhost:3000/grafana # this host can be localhost +``` + +## How to securely reference secrets in grafana.ini + +This example uses Grafana [file providers](https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider) for secret values and the `extraSecretMounts` configuration flag (Additional grafana server secret mounts) to mount the secrets. + +In grafana.ini: + +```yaml +grafana.ini: + [auth.generic_oauth] + enabled = true + client_id = $__file{/etc/secrets/auth_generic_oauth/client_id} + client_secret = $__file{/etc/secrets/auth_generic_oauth/client_secret} +``` + +Existing secret, or created along with helm: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: auth-generic-oauth-secret +type: Opaque +stringData: + client_id: + client_secret: +``` + +Include in the `extraSecretMounts` configuration flag: + +```yaml +- extraSecretMounts: + - name: auth-generic-oauth-secret-mount + secretName: auth-generic-oauth-secret + defaultMode: 0440 + mountPath: /etc/secrets/auth_generic_oauth + readOnly: true +``` + +### extraSecretMounts using a Container Storage Interface (CSI) provider + +This example uses a CSI driver e.g. retrieving secrets using [Azure Key Vault Provider](https://github.com/Azure/secrets-store-csi-driver-provider-azure) + +```yaml +- extraSecretMounts: + - name: secrets-store-inline + mountPath: /run/secrets + readOnly: true + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "my-provider" + nodePublishSecretRef: + name: akv-creds +``` + +## Image Renderer Plug-In + +This chart supports enabling [remote image rendering](https://github.com/grafana/grafana-image-renderer/blob/master/README.md#run-in-docker) + +```yaml +imageRenderer: + enabled: true +``` + +### Image Renderer NetworkPolicy + +By default the image-renderer pods will have a network policy which only allows ingress traffic from the created grafana instance + +### High Availability for unified alerting + +If you want to run Grafana in a high availability cluster you need to enable +the headless service by setting `headlessService: true` in your `values.yaml` +file. + +As next step you have to setup the `grafana.ini` in your `values.yaml` in a way +that it will make use of the headless service to obtain all the IPs of the +cluster. You should replace ``{{ Name }}`` with the name of your helm deployment. + +```yaml +grafana.ini: + ... + unified_alerting: + enabled: true + ha_peers: {{ Name }}-headless:9094 + alerting: + enabled: false +``` diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/dashboards/custom-dashboard.json b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/dashboards/custom-dashboard.json new file mode 100644 index 0000000000..9e26dfeeb6 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/dashboards/custom-dashboard.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/NOTES.txt b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/NOTES.txt new file mode 100644 index 0000000000..1fc8436d95 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/NOTES.txt @@ -0,0 +1,54 @@ +1. Get your '{{ .Values.adminUser }}' user password by running: + + kubectl get secret --namespace {{ template "grafana.namespace" . }} {{ template "grafana.fullname" . }} -o jsonpath="{.data.admin-password}" | base64 --decode ; echo + +2. The Grafana server can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster: + + {{ template "grafana.fullname" . }}.{{ template "grafana.namespace" . }}.svc.cluster.local +{{ if .Values.ingress.enabled }} + If you bind grafana to 80, please update values in values.yaml and reinstall: + ``` + securityContext: + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + + command: + - "setcap" + - "'cap_net_bind_service=+ep'" + - "/usr/sbin/grafana-server &&" + - "sh" + - "/run.sh" + ``` + Details refer to https://grafana.com/docs/installation/configuration/#http-port. + Or grafana would always crash. + + From outside the cluster, the server URL(s) are: +{{- range .Values.ingress.hosts }} + http://{{ . }} +{{- end }} +{{ else }} + Get the Grafana URL to visit by running these commands in the same shell: +{{ if contains "NodePort" .Values.service.type -}} + export NODE_PORT=$(kubectl get --namespace {{ template "grafana.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "grafana.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ template "grafana.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{ else if contains "LoadBalancer" .Values.service.type -}} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc --namespace {{ template "grafana.namespace" . }} -w {{ template "grafana.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ template "grafana.namespace" . }} {{ template "grafana.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + http://$SERVICE_IP:{{ .Values.service.port -}} +{{ else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ template "grafana.namespace" . }} -l "app.kubernetes.io/name={{ template "grafana.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ template "grafana.namespace" . }} port-forward $POD_NAME 3000 +{{- end }} +{{- end }} + +3. Login with the password from step 1 and the username: {{ .Values.adminUser }} + +{{- if not .Values.persistence.enabled }} +################################################################################# +###### WARNING: Persistence is disabled!!! You will lose your data when ##### +###### the Grafana pod is terminated. ##### +################################################################################# +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/_helpers.tpl b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/_helpers.tpl new file mode 100644 index 0000000000..e5e3b287dd --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/_helpers.tpl @@ -0,0 +1,214 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "grafana.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "grafana.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "grafana.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name of the service account +*/}} +{{- define "grafana.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "grafana.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{- define "grafana.serviceAccountNameTest" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (print (include "grafana.fullname" .) "-test") .Values.serviceAccount.nameTest }} +{{- else -}} + {{ default "default" .Values.serviceAccount.nameTest }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "grafana.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "grafana.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels }} +{{- end }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "grafana.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "grafana.imageRenderer.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.imageRenderer.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels ImageRenderer +*/}} +{{- define "grafana.imageRenderer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Looks if there's an existing secret and reuse its password. If not it generates +new password and use it. +*/}} +{{- define "grafana.password" -}} +{{- $secret := (lookup "v1" "Secret" (include "grafana.namespace" .) (include "grafana.fullname" .) ) -}} + {{- if $secret -}} + {{- index $secret "data" "admin-password" -}} + {{- else -}} + {{- (randAlphaNum 40) | b64enc | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for rbac. +*/}} +{{- define "grafana.rbac.apiVersion" -}} + {{- if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }} + {{- print "rbac.authorization.k8s.io/v1" -}} + {{- else -}} + {{- print "rbac.authorization.k8s.io/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for HorizontalPodAutoscaler. +*/}} +{{- define "grafana.hpa.apiVersion" -}} + {{- if .Capabilities.APIVersions.Has "autoscaling/v2" }} + {{- print "autoscaling/v2" -}} + {{- else if .Capabilities.APIVersions.Has "autoscaling/v1" }} + {{- print "autoscaling/v1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "grafana.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for podDisruptionBudget. +*/}} +{{- define "grafana.podDisruptionBudget.apiVersion" -}} + {{- if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" -}} + {{- print "policy/v1" -}} + {{- else -}} + {{- print "policy/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return if ingress is stable. +*/}} +{{- define "grafana.ingress.isStable" -}} + {{- eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} + +{{/* +Return if ingress supports ingressClassName. +*/}} +{{- define "grafana.ingress.supportsIngressClassName" -}} + {{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}} +{{- end -}} + +{{/* +Return if ingress supports pathType. +*/}} +{{- define "grafana.ingress.supportsPathType" -}} + {{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}} +{{- end -}} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/_pod.tpl b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/_pod.tpl new file mode 100644 index 0000000000..ae36c28a03 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/_pod.tpl @@ -0,0 +1,885 @@ +{{- define "grafana.pod" -}} +{{- if .Values.schedulerName }} +schedulerName: "{{ .Values.schedulerName }}" +{{- end }} +serviceAccountName: {{ template "grafana.serviceAccountName" . }} +automountServiceAccountToken: {{ .Values.serviceAccount.autoMount }} +{{- with .Values.securityContext }} +securityContext: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.hostAliases }} +hostAliases: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- if .Values.priorityClassName }} +priorityClassName: {{ .Values.priorityClassName }} +{{- end }} +{{- if ( or .Values.persistence.enabled .Values.dashboards .Values.sidecar.notifiers.enabled .Values.extraInitContainers (and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources)) }} +initContainers: +{{- end }} +{{- if ( and .Values.persistence.enabled .Values.initChownData.enabled ) }} + - name: init-chown-data + {{- if .Values.initChownData.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}@sha256:{{ .Values.initChownData.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.initChownData.image.pullPolicy }} + securityContext: + runAsNonRoot: false + runAsUser: 0 + command: ["chown", "-R", "{{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.runAsGroup }}", "/var/lib/grafana"] + {{- with .Values.initChownData.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: storage + mountPath: "/var/lib/grafana" +{{- if .Values.persistence.subPath }} + subPath: {{ tpl .Values.persistence.subPath . }} +{{- end }} +{{- end }} +{{- if .Values.dashboards }} + - name: download-dashboards + {{- if .Values.downloadDashboardsImage.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}@sha256:{{ .Values.downloadDashboardsImage.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.downloadDashboardsImage.pullPolicy }} + command: ["/bin/sh"] + args: [ "-c", "mkdir -p /var/lib/grafana/dashboards/default && /bin/sh -x /etc/grafana/download_dashboards.sh" ] + {{- with .Values.downloadDashboards.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + env: +{{- range $key, $value := .Values.downloadDashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" +{{- end }} + {{- with .Values.downloadDashboards.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- if .Values.downloadDashboards.envFromSecret }} + envFrom: + - secretRef: + name: {{ tpl .Values.downloadDashboards.envFromSecret . }} +{{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/download_dashboards.sh" + subPath: download_dashboards.sh + - name: storage + mountPath: "/var/lib/grafana" +{{- if .Values.persistence.subPath }} + subPath: {{ tpl .Values.persistence.subPath . }} +{{- end }} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources }} + - name: {{ template "grafana.name" . }}-init-sc-datasources + {{- if .Values.sidecar.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- if .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.datasources.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + - name: NAMESPACE + value: "{{ template "project-prometheus-stack.projectNamespaceList" . }}" + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end }} +{{- if .Values.sidecar.notifiers.enabled }} + - name: {{ template "grafana.name" . }}-sc-notifiers + {{- if .Values.sidecar.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: LIST + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- if .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.notifiers.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + - name: NAMESPACE + value: "{{ template "project-prometheus-stack.projectNamespaceList" . }}" + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- if .Values.extraInitContainers }} +{{ tpl (toYaml .Values.extraInitContainers) . | indent 2 }} +{{- end }} +{{- if .Values.image.pullSecrets }} +imagePullSecrets: +{{- $root := . }} +{{- range .Values.image.pullSecrets }} + - name: {{ tpl . $root }} +{{- end }} +{{- end }} +{{- if not .Values.enableKubeBackwardCompatibility }} +enableServiceLinks: {{ .Values.enableServiceLinks }} +{{- end }} +containers: +{{- if .Values.sidecar.dashboards.enabled }} + - name: {{ template "grafana.name" . }}-sc-dashboard + {{- if .Values.sidecar.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.dashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.dashboards.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.dashboards.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.dashboards.label }}" + {{- if .Values.sidecar.dashboards.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.dashboards.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + {{- end }} + - name: FOLDER + value: "{{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }}" + - name: RESOURCE + value: {{ quote .Values.sidecar.dashboards.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + - name: NAMESPACE + value: "{{ template "project-prometheus-stack.projectNamespaceList" . }}" + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if .Values.sidecar.dashboards.folderAnnotation }} + - name: FOLDER_ANNOTATION + value: "{{ .Values.sidecar.dashboards.folderAnnotation }}" + {{- end }} + {{- if .Values.sidecar.dashboards.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.dashboards.script }}" + {{- end }} + {{- if .Values.sidecar.dashboards.watchServerTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchServerTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.dashboards.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.dashboards.watchClientTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchClientTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.dashboards.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- if .Values.sidecar.dashboards.extraMounts }} + {{- toYaml .Values.sidecar.dashboards.extraMounts | trim | nindent 6}} + {{- end }} +{{- end}} +{{- if .Values.sidecar.datasources.enabled }} + - name: {{ template "grafana.name" . }}-sc-datasources + {{- if .Values.sidecar.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.datasources.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- if .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.datasources.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + - name: NAMESPACE + value: "{{ template "project-prometheus-stack.projectNamespaceList" . }}" + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if .Values.sidecar.datasources.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.datasources.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.datasources.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.datasources.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.datasources.watchServerTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchServerTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.datasources.watchClientTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchClientTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end}} +{{- if .Values.sidecar.plugins.enabled }} + - name: {{ template "grafana.name" . }}-sc-plugins + {{- if .Values.sidecar.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.plugins.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.plugins.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.plugins.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.plugins.label }}" + {{- if .Values.sidecar.plugins.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.plugins.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/plugins" + - name: RESOURCE + value: {{ quote .Values.sidecar.plugins.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + - name: NAMESPACE + value: "{{ template "project-prometheus-stack.projectNamespaceList" . }}" + {{- if .Values.sidecar.plugins.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.plugins.script }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.plugins.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.plugins.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.plugins.watchServerTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchServerTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.plugins.watchClientTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchClientTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" +{{- end}} + - name: {{ .Chart.Name }} + {{- if .Values.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.command }} + command: + {{- range .Values.command }} + - {{ . }} + {{- end }} + {{- end}} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/grafana.ini" + subPath: grafana.ini + {{- if .Values.ldap.enabled }} + - name: ldap + mountPath: "/etc/grafana/ldap.toml" + subPath: ldap.toml + {{- end }} + {{- $root := . }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + mountPath: {{ tpl .mountPath $root }} + subPath: {{ (tpl .subPath $root) | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + - name: storage + mountPath: "/var/lib/grafana" +{{- if .Values.persistence.subPath }} + subPath: {{ tpl .Values.persistence.subPath . }} +{{- end }} +{{- if .Values.dashboards }} +{{- range $provider, $dashboards := .Values.dashboards }} +{{- range $key, $value := $dashboards }} +{{- if (or (hasKey $value "json") (hasKey $value "file")) }} + - name: dashboards-{{ $provider }} + mountPath: "/var/lib/grafana/dashboards/{{ $provider }}/{{ $key }}.json" + subPath: "{{ $key }}.json" +{{- end }} +{{- end }} +{{- end }} +{{- end -}} +{{- if .Values.dashboardsConfigMaps }} +{{- range (keys .Values.dashboardsConfigMaps | sortAlpha) }} + - name: dashboards-{{ . }} + mountPath: "/var/lib/grafana/dashboards/{{ . }}" +{{- end }} +{{- end }} +{{- if .Values.datasources }} +{{- range (keys .Values.datasources | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} +{{- end }} +{{- end }} +{{- if .Values.notifiers }} +{{- range (keys .Values.notifiers | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} +{{- end }} +{{- end }} +{{- if .Values.alerting }} +{{- range (keys .Values.alerting | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} +{{- end }} +{{- end }} +{{- if .Values.dashboardProviders }} +{{- range (keys .Values.dashboardProviders | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/dashboards/{{ . }}" + subPath: {{ . | quote }} +{{- end }} +{{- end }} +{{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} +{{ if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + mountPath: "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml" + subPath: provider.yaml +{{- end}} +{{- end}} +{{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end}} +{{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" +{{- end}} +{{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + subPath: {{ .subPath | default "" }} + {{- end }} + {{- range .Values.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + ports: + - name: {{ .Values.podPortName }} + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + env: + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if .Values.plugins }} + - name: GF_INSTALL_PLUGINS + valueFrom: + configMapKeyRef: + name: {{ template "grafana.fullname" . }} + key: plugins + {{- end }} + {{- if .Values.smtp.existingSecret }} + - name: GF_SMTP_USER + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.userKey | default "user" }} + - name: GF_SMTP_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.passwordKey | default "password" }} + {{- end }} + {{- if .Values.imageRenderer.enabled }} + - name: GF_RENDERING_SERVER_URL + value: http://{{ template "grafana.fullname" . }}-image-renderer.{{ template "grafana.namespace" . }}:{{ .Values.imageRenderer.service.port }}/render + - name: GF_RENDERING_CALLBACK_URL + value: {{ .Values.imageRenderer.grafanaProtocol }}://{{ template "grafana.fullname" . }}.{{ template "grafana.namespace" . }}:{{ .Values.service.port }}/{{ .Values.imageRenderer.grafanaSubPath }} + {{- end }} + - name: GF_PATHS_DATA + value: {{ (get .Values "grafana.ini").paths.data }} + - name: GF_PATHS_LOGS + value: {{ (get .Values "grafana.ini").paths.logs }} + - name: GF_PATHS_PLUGINS + value: {{ (get .Values "grafana.ini").paths.plugins }} + - name: GF_PATHS_PROVISIONING + value: {{ (get .Values "grafana.ini").paths.provisioning }} + {{- range $key, $value := .Values.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: +{{ tpl (toYaml $value) $ | indent 10 }} + {{- end }} +{{- range $key, $value := .Values.env }} + - name: "{{ tpl $key $ }}" + value: "{{ tpl (print $value) $ }}" +{{- end }} + {{- if or .Values.envFromSecret (or .Values.envRenderSecret .Values.envFromSecrets) .Values.envFromConfigMaps }} + envFrom: + {{- if .Values.envFromSecret }} + - secretRef: + name: {{ tpl .Values.envFromSecret . }} + {{- end }} + {{- if .Values.envRenderSecret }} + - secretRef: + name: {{ template "grafana.fullname" . }}-env + {{- end }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- end }} + {{- range .Values.envFromConfigMaps }} + - configMapRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- end }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- if .Values.lifecycleHooks }} + lifecycle: {{ tpl (.Values.lifecycleHooks | toYaml) . | nindent 6 }} +{{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- with .Values.extraContainers }} +{{ tpl . $ | indent 2 }} +{{- end }} +nodeSelector: {{ include "linux-node-selector" . | nindent 2 }} +{{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 2 }} +{{- end }} +{{- $root := . }} +{{- with .Values.affinity }} +affinity: +{{ tpl (toYaml .) $root | indent 2 }} +{{- end }} +{{- with .Values.topologySpreadConstraints }} +topologySpreadConstraints: + {{- toYaml . | nindent 2 }} +{{- end }} +tolerations: {{ include "linux-node-tolerations" . | nindent 2 }} +{{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 2 }} +{{- end }} +volumes: + - name: config + configMap: + name: {{ template "grafana.fullname" . }} +{{- $root := . }} +{{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + configMap: + name: {{ tpl .configMap $root }} + {{- if .items }} + items: {{ toYaml .items | nindent 6 }} + {{- end }} +{{- end }} + {{- if .Values.dashboards }} + {{- range (keys .Values.dashboards | sortAlpha) }} + - name: dashboards-{{ . }} + configMap: + name: {{ template "grafana.fullname" $ }}-dashboards-{{ . }} + {{- end }} + {{- end }} + {{- if .Values.dashboardsConfigMaps }} + {{ $root := . }} + {{- range $provider, $name := .Values.dashboardsConfigMaps }} + - name: dashboards-{{ $provider }} + configMap: + name: {{ tpl $name $root }} + {{- end }} + {{- end }} + {{- if .Values.ldap.enabled }} + - name: ldap + secret: + {{- if .Values.ldap.existingSecret }} + secretName: {{ .Values.ldap.existingSecret }} + {{- else }} + secretName: {{ template "grafana.fullname" . }} + {{- end }} + items: + - key: ldap-toml + path: ldap.toml + {{- end }} +{{- if and .Values.persistence.enabled (eq .Values.persistence.type "pvc") }} + - name: storage + persistentVolumeClaim: + claimName: {{ tpl (.Values.persistence.existingClaim | default (include "grafana.fullname" .)) . }} +{{- else if and .Values.persistence.enabled (eq .Values.persistence.type "statefulset") }} +# nothing +{{- else }} + - name: storage +{{- if .Values.persistence.inMemory.enabled }} + emptyDir: + medium: Memory +{{- if .Values.persistence.inMemory.sizeLimit }} + sizeLimit: {{ .Values.persistence.inMemory.sizeLimit }} +{{- end -}} +{{- else }} + emptyDir: {} +{{- end -}} +{{- end -}} +{{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume +{{- if .Values.sidecar.dashboards.sizeLimit }} + emptyDir: + sizeLimit: {{ .Values.sidecar.dashboards.sizeLimit }} +{{- else }} + emptyDir: {} +{{- end -}} +{{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + configMap: + name: {{ template "grafana.fullname" . }}-config-dashboards +{{- end }} +{{- end }} +{{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume +{{- if .Values.sidecar.datasources.sizeLimit }} + emptyDir: + sizeLimit: {{ .Values.sidecar.datasources.sizeLimit }} +{{- else }} + emptyDir: {} +{{- end -}} +{{- end -}} +{{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume +{{- if .Values.sidecar.plugins.sizeLimit }} + emptyDir: + sizeLimit: {{ .Values.sidecar.plugins.sizeLimit }} +{{- else }} + emptyDir: {} +{{- end -}} +{{- end -}} +{{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume +{{- if .Values.sidecar.notifiers.sizeLimit }} + emptyDir: + sizeLimit: {{ .Values.sidecar.notifiers.sizeLimit }} +{{- else }} + emptyDir: {} +{{- end -}} +{{- end -}} +{{- range .Values.extraSecretMounts }} +{{- if .secretName }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + defaultMode: {{ .defaultMode }} + {{- if .items }} + items: {{ toYaml .items | nindent 6 }} + {{- end }} +{{- else if .projected }} + - name: {{ .name }} + projected: {{- toYaml .projected | nindent 6 }} +{{- else if .csi }} + - name: {{ .name }} + csi: {{- toYaml .csi | nindent 6 }} +{{- end }} +{{- end }} +{{- range .Values.extraVolumeMounts }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + path: {{ .hostPath }} + {{- else if .csi }} + csi: + data: + {{ toYaml .data | nindent 6 }} + {{- else }} + emptyDir: {} + {{- end }} +{{- end }} +{{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + emptyDir: {} +{{- end -}} +{{- if .Values.extraContainerVolumes }} +{{ tpl (toYaml .Values.extraContainerVolumes) . | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/clusterrole.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/clusterrole.yaml new file mode 100644 index 0000000000..154658b51c --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/clusterrole.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.rbac.create (not .Values.rbac.namespaced) (not .Values.rbac.useExistingRole) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} + name: {{ template "grafana.fullname" . }}-clusterrole +{{- if or .Values.sidecar.dashboards.enabled (or .Values.rbac.extraClusterRoleRules (or .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled)) }} +rules: +{{- if or .Values.sidecar.dashboards.enabled (or .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled) }} +- apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] +{{- end}} +{{- with .Values.rbac.extraClusterRoleRules }} +{{ toYaml . | indent 0 }} +{{- end}} +{{- else }} +rules: [] +{{- end}} +{{- end}} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/clusterrolebinding.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..4accbfac04 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.rbac.create (not .Values.rbac.namespaced) }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "grafana.fullname" . }}-clusterrolebinding + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +subjects: + - kind: ServiceAccount + name: {{ template "grafana.serviceAccountName" . }} + namespace: {{ template "grafana.namespace" . }} +roleRef: + kind: ClusterRole +{{- if (not .Values.rbac.useExistingRole) }} + name: {{ template "grafana.fullname" . }}-clusterrole +{{- else }} + name: {{ .Values.rbac.useExistingRole }} +{{- end }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/configmap-dashboard-provider.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/configmap-dashboard-provider.yaml new file mode 100644 index 0000000000..65d73858e0 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/configmap-dashboard-provider.yaml @@ -0,0 +1,29 @@ +{{- if .Values.sidecar.dashboards.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} + name: {{ template "grafana.fullname" . }}-config-dashboards + namespace: {{ template "grafana.namespace" . }} +data: + provider.yaml: |- + apiVersion: 1 + providers: + - name: '{{ .Values.sidecar.dashboards.provider.name }}' + orgId: {{ .Values.sidecar.dashboards.provider.orgid }} + {{- if not .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + folder: '{{ .Values.sidecar.dashboards.provider.folder }}' + {{- end}} + type: {{ .Values.sidecar.dashboards.provider.type }} + disableDeletion: {{ .Values.sidecar.dashboards.provider.disableDelete }} + allowUiUpdates: {{ .Values.sidecar.dashboards.provider.allowUiUpdates }} + updateIntervalSeconds: {{ .Values.sidecar.dashboards.provider.updateIntervalSeconds | default 30 }} + options: + foldersFromFilesStructure: {{ .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + path: {{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }} +{{- end}} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/configmap.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/configmap.yaml new file mode 100644 index 0000000000..2461d17504 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/configmap.yaml @@ -0,0 +1,117 @@ +{{- if .Values.createConfigmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +data: +{{- if .Values.plugins }} + plugins: {{ join "," .Values.plugins }} +{{- end }} + grafana.ini: | +{{- range $elem, $elemVal := index .Values "grafana.ini" }} + {{- if not (kindIs "map" $elemVal) }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} +{{- end }} +{{- range $key, $value := index .Values "grafana.ini" }} + {{- if kindIs "map" $value }} + [{{ $key }}] + {{- range $elem, $elemVal := $value }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} + +{{- if .Values.datasources }} +{{ $root := . }} + {{- range $key, $value := .Values.datasources }} + {{ $key }}: | +{{ tpl (toYaml $value | indent 4) $root }} + {{- end -}} +{{- end -}} + +{{- if .Values.notifiers }} + {{- range $key, $value := .Values.notifiers }} + {{ $key }}: | +{{ toYaml $value | indent 4 }} + {{- end -}} +{{- end -}} + +{{- if .Values.alerting }} +{{ $root := . }} + {{- range $key, $value := .Values.alerting }} + {{ $key }}: | +{{ tpl (toYaml $value | indent 4) $root }} + {{- end -}} +{{- end -}} + +{{- if .Values.dashboardProviders }} + {{- range $key, $value := .Values.dashboardProviders }} + {{ $key }}: | +{{ toYaml $value | indent 4 }} + {{- end -}} +{{- end -}} + +{{- if .Values.dashboards }} + download_dashboards.sh: | + #!/usr/bin/env sh + set -euf + {{- if .Values.dashboardProviders }} + {{- range $key, $value := .Values.dashboardProviders }} + {{- range $value.providers }} + mkdir -p {{ .options.path }} + {{- end }} + {{- end }} + {{- end }} + {{ $dashboardProviders := .Values.dashboardProviders }} + {{- range $provider, $dashboards := .Values.dashboards }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "gnetId") (hasKey $value "url")) }} + curl -skf \ + --connect-timeout 60 \ + --max-time 60 \ + {{- if not $value.b64content }} + -H "Accept: application/json" \ + {{- if $value.token }} + -H "Authorization: token {{ $value.token }}" \ + {{- end }} + {{- if $value.bearerToken }} + -H "Authorization: Bearer {{ $value.bearerToken }}" \ + {{- end }} + {{- if $value.gitlabToken }} + -H "PRIVATE-TOKEN: {{ $value.gitlabToken }}" \ + {{- end }} + -H "Content-Type: application/json;charset=UTF-8" \ + {{ end }} + {{- $dpPath := "" -}} + {{- range $kd := (index $dashboardProviders "dashboardproviders.yaml").providers }} + {{- if eq $kd.name $provider -}} + {{- $dpPath = $kd.options.path -}} + {{- end -}} + {{- end -}} + {{- if $value.url -}}"{{ $value.url }}"{{- else -}}"https://grafana.com/api/dashboards/{{ $value.gnetId }}/revisions/{{- if $value.revision -}}{{ $value.revision }}{{- else -}}1{{- end -}}/download"{{- end -}}{{ if $value.datasource }} | sed '/-- .* --/! s/"datasource":.*,/"datasource": "{{ $value.datasource }}",/g'{{ end }}{{- if $value.b64content -}} | base64 -d {{- end -}} \ + > "{{- if $dpPath -}}{{ $dpPath }}{{- else -}}/var/lib/grafana/dashboards/{{ $provider }}{{- end -}}/{{ $key }}.json" + {{- end }} + {{- end -}} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/dashboards-json-configmap.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/dashboards-json-configmap.yaml new file mode 100644 index 0000000000..59e0be6415 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/dashboards-json-configmap.yaml @@ -0,0 +1,35 @@ +{{- if .Values.dashboards }} +{{ $files := .Files }} +{{- range $provider, $dashboards := .Values.dashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "grafana.fullname" $ }}-dashboards-{{ $provider }} + namespace: {{ template "grafana.namespace" $ }} + labels: + {{- include "grafana.labels" $ | nindent 4 }} + dashboard-provider: {{ $provider }} +{{- if $dashboards }} +data: +{{- $dashboardFound := false }} +{{- range $key, $value := $dashboards }} +{{- if (or (hasKey $value "json") (hasKey $value "file")) }} +{{- $dashboardFound = true }} +{{ print $key | indent 2 }}.json: +{{- if hasKey $value "json" }} + |- +{{ $value.json | indent 6 }} +{{- end }} +{{- if hasKey $value "file" }} +{{ toYaml ( $files.Get $value.file ) | indent 4}} +{{- end }} +{{- end }} +{{- end }} +{{- if not $dashboardFound }} + {} +{{- end }} +{{- end }} +--- +{{- end }} + +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/deployment.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/deployment.yaml new file mode 100644 index 0000000000..fee9c335a5 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/deployment.yaml @@ -0,0 +1,50 @@ +{{ if (and (not .Values.useStatefulSet) (or (not .Values.persistence.enabled) (eq .Values.persistence.type "pvc"))) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.labels }} +{{ toYaml .Values.labels | indent 4 }} +{{- end }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + {{- if and (not .Values.autoscaling.enabled) (.Values.replicas) }} + replicas: {{ .Values.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} +{{- with .Values.deploymentStrategy }} + strategy: +{{ toYaml . | trim | indent 4 }} +{{- end }} + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} +{{- with .Values.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} +{{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} +{{- end }} +{{- if .Values.envRenderSecret }} + checksum/secret-env: {{ include (print $.Template.BasePath "/secret-env.yaml") . | sha256sum }} +{{- end }} +{{- with .Values.podAnnotations }} +{{ toYaml . | indent 8 }} +{{- end }} + spec: + {{- include "grafana.pod" . | indent 6 }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/headless-service.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/headless-service.yaml new file mode 100644 index 0000000000..b5faddcfce --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/headless-service.yaml @@ -0,0 +1,22 @@ +{{- if or .Values.headlessService (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "statefulset"))}} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "grafana.fullname" . }}-headless + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + clusterIP: None + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} + type: ClusterIP + ports: + - protocol: TCP + port: 3000 + targetPort: {{ .Values.service.targetPort }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/hpa.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/hpa.yaml new file mode 100644 index 0000000000..236a06d658 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/hpa.yaml @@ -0,0 +1,21 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: {{ template "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ template "grafana.name" . }} + helm.sh/chart: {{ template "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "grafana.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: +{{ toYaml .Values.autoscaling.metrics | indent 4 }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/image-renderer-deployment.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/image-renderer-deployment.yaml new file mode 100644 index 0000000000..97a8675b22 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/image-renderer-deployment.yaml @@ -0,0 +1,123 @@ +{{ if .Values.imageRenderer.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "grafana.fullname" . }}-image-renderer + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} +{{- if .Values.imageRenderer.labels }} +{{ toYaml .Values.imageRenderer.labels | indent 4 }} +{{- end }} +{{- with .Values.imageRenderer.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.imageRenderer.replicas }} + revisionHistoryLimit: {{ .Values.imageRenderer.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} +{{- with .Values.imageRenderer.deploymentStrategy }} + strategy: +{{ toYaml . | trim | indent 4 }} +{{- end }} + template: + metadata: + labels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 8 }} +{{- with .Values.imageRenderer.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} +{{- with .Values.imageRenderer.podAnnotations }} +{{ toYaml . | indent 8 }} +{{- end }} + spec: + + {{- if .Values.imageRenderer.schedulerName }} + schedulerName: "{{ .Values.imageRenderer.schedulerName }}" + {{- end }} + {{- if .Values.imageRenderer.serviceAccountName }} + serviceAccountName: "{{ .Values.imageRenderer.serviceAccountName }}" + {{- else }} + serviceAccountName: {{ template "grafana.serviceAccountName" . }} + {{- end }} + {{- if .Values.imageRenderer.securityContext }} + securityContext: + {{- toYaml .Values.imageRenderer.securityContext | nindent 8 }} + {{- end }} + {{- if .Values.imageRenderer.hostAliases }} + hostAliases: + {{- toYaml .Values.imageRenderer.hostAliases | nindent 8 }} + {{- end }} + {{- if .Values.imageRenderer.priorityClassName }} + priorityClassName: {{ .Values.imageRenderer.priorityClassName }} + {{- end }} + {{- if .Values.imageRenderer.image.pullSecrets }} + imagePullSecrets: + {{- $root := . }} + {{- range .Values.imageRenderer.image.pullSecrets }} + - name: {{ tpl . $root }} + {{- end}} + {{- end }} + containers: + - name: {{ .Chart.Name }}-image-renderer + {{- if .Values.imageRenderer.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}@sha256:{{ .Values.imageRenderer.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.imageRenderer.image.pullPolicy }} + {{- if .Values.imageRenderer.command }} + command: + {{- range .Values.imageRenderer.command }} + - {{ . }} + {{- end }} + {{- end}} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + containerPort: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: {{ .Values.imageRenderer.service.portName }} + env: + - name: HTTP_PORT + value: {{ .Values.imageRenderer.service.targetPort | quote }} + {{- range $key, $value := .Values.imageRenderer.env }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + securityContext: + capabilities: + drop: ['all'] + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /tmp + name: image-renderer-tmpfs + {{- with .Values.imageRenderer.resources }} + resources: +{{ toYaml . | indent 12 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.imageRenderer.nodeSelector }} +{{ toYaml . | indent 8 }} + {{- end }} + {{- $root := . }} + {{- with .Values.imageRenderer.affinity }} + affinity: +{{ tpl (toYaml .) $root | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.imageRenderer.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} + volumes: + - name: image-renderer-tmpfs + emptyDir: {} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/image-renderer-network-policy.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/image-renderer-network-policy.yaml new file mode 100644 index 0000000000..0d9bdfe4d0 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/image-renderer-network-policy.yaml @@ -0,0 +1,73 @@ +{{- if and (.Values.imageRenderer.enabled) (.Values.imageRenderer.networkPolicy.limitIngress) }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "grafana.fullname" . }}-image-renderer-ingress + namespace: {{ template "grafana.namespace" . }} + annotations: + comment: Limit image-renderer ingress traffic from grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- if .Values.imageRenderer.podLabels }} + {{ toYaml .Values.imageRenderer.podLabels | nindent 6 }} + {{- end }} + + policyTypes: + - Ingress + ingress: + - ports: + - port: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + from: + - namespaceSelector: + matchLabels: + name: {{ template "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- if .Values.podLabels }} + {{ toYaml .Values.podLabels | nindent 14 }} + {{- end }} +{{ end }} + +{{- if and (.Values.imageRenderer.enabled) (.Values.imageRenderer.networkPolicy.limitEgress) }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "grafana.fullname" . }}-image-renderer-egress + namespace: {{ template "grafana.namespace" . }} + annotations: + comment: Limit image-renderer egress traffic to grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- if .Values.imageRenderer.podLabels }} + {{ toYaml .Values.imageRenderer.podLabels | nindent 6 }} + {{- end }} + + policyTypes: + - Egress + egress: + # allow dns resolution + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # talk only to grafana + - ports: + - port: {{ .Values.service.port }} + protocol: TCP + to: + - podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- if .Values.podLabels }} + {{ toYaml .Values.podLabels | nindent 14 }} + {{- end }} +{{ end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/image-renderer-service.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/image-renderer-service.yaml new file mode 100644 index 0000000000..fcf707a3f7 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/image-renderer-service.yaml @@ -0,0 +1,33 @@ +{{ if .Values.imageRenderer.enabled }} +{{ if .Values.imageRenderer.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "grafana.fullname" . }}-image-renderer + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} +{{- if .Values.imageRenderer.service.labels }} +{{ toYaml .Values.imageRenderer.service.labels | indent 4 }} +{{- end }} +{{- with .Values.imageRenderer.service.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + type: ClusterIP + {{- if .Values.imageRenderer.service.clusterIP }} + clusterIP: {{ .Values.imageRenderer.service.clusterIP }} + {{end}} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + port: {{ .Values.imageRenderer.service.port }} + protocol: TCP + targetPort: {{ .Values.imageRenderer.service.targetPort }} + {{- if .Values.imageRenderer.appProtocol }} + appProtocol: {{ .Values.imageRenderer.appProtocol }} + {{- end }} + selector: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 4 }} +{{ end }} +{{ end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/ingress.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/ingress.yaml new file mode 100644 index 0000000000..7699cecaa8 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/ingress.yaml @@ -0,0 +1,78 @@ +{{- if .Values.ingress.enabled -}} +{{- $ingressApiIsStable := eq (include "grafana.ingress.isStable" .) "true" -}} +{{- $ingressSupportsIngressClassName := eq (include "grafana.ingress.supportsIngressClassName" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "grafana.ingress.supportsPathType" .) "true" -}} +{{- $fullName := include "grafana.fullname" . -}} +{{- $servicePort := .Values.service.port -}} +{{- $ingressPath := .Values.ingress.path -}} +{{- $ingressPathType := .Values.ingress.pathType -}} +{{- $extraPaths := .Values.ingress.extraPaths -}} +apiVersion: {{ include "grafana.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.ingress.labels }} +{{ toYaml .Values.ingress.labels | indent 4 }} +{{- end }} + {{- if .Values.ingress.annotations }} + annotations: + {{- range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ tpl $value $ | quote }} + {{- end }} + {{- end }} +spec: + {{- if and $ingressSupportsIngressClassName .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end -}} +{{- if .Values.ingress.tls }} + tls: +{{ tpl (toYaml .Values.ingress.tls) $ | indent 4 }} +{{- end }} + rules: + {{- if .Values.ingress.hosts }} + {{- range .Values.ingress.hosts }} + - host: {{ tpl . $}} + http: + paths: +{{- if $extraPaths }} +{{ toYaml $extraPaths | indent 10 }} +{{- end }} + - path: {{ $ingressPath }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end }} + {{- else }} + - http: + paths: + - backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- if $ingressPath }} + path: {{ $ingressPath }} + {{- end }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + {{- end -}} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/networkpolicy.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/networkpolicy.yaml new file mode 100644 index 0000000000..b751d9436f --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/networkpolicy.yaml @@ -0,0 +1,52 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{ toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + policyTypes: + {{- if .Values.networkPolicy.ingress }} + - Ingress + {{- end }} + {{- if .Values.networkPolicy.egress.enabled }} + - Egress + {{- end }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + + {{- if .Values.networkPolicy.egress.enabled }} + egress: + - ports: + {{ .Values.networkPolicy.egress.ports | toJson }} + {{- end }} + {{- if .Values.networkPolicy.ingress }} + ingress: + - ports: + - port: {{ .Values.service.targetPort }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ template "grafana.fullname" . }}-client: "true" + {{- with .Values.networkPolicy.explicitNamespacesSelector }} + - namespaceSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + - podSelector: + matchLabels: + {{- include "grafana.labels" . | nindent 14 }} + role: read + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/nginx-config.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/nginx-config.yaml new file mode 100644 index 0000000000..a7abf9ffd9 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/nginx-config.yaml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-nginx-proxy-config + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 8080; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $http_host; + + location /api/dashboards { + proxy_pass http://localhost:3000; + } + + location /api/search { + proxy_pass http://localhost:3000; + + sub_filter_types application/json; + sub_filter_once off; + } + + location /api/live/ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $http_host; + proxy_pass http://localhost:3000; + } + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:3000/; + + sub_filter_once off; +{{- if and (not (empty .Values.global.cattle.clusterId )) (not (eq .Values.global.cattle.clusterId "local")) -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; +{{- else }} + sub_filter '"appSubUrl":""' '"appSubUrl":"/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; +{{- end }} + + sub_filter '"url":"/' '"url":"./'; + sub_filter ':"/avatar/' ':"avatar/'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/poddisruptionbudget.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000000..70901b70c4 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/poddisruptionbudget.yaml @@ -0,0 +1,22 @@ +{{- if .Values.podDisruptionBudget }} +apiVersion: {{ include "grafana.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.labels }} +{{ toYaml .Values.labels | indent 4 }} +{{- end }} +spec: +{{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} +{{- end }} +{{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} +{{- end }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/podsecuritypolicy.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000..82d295ad13 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/podsecuritypolicy.yaml @@ -0,0 +1,45 @@ +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "grafana.fullname" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.rbac.pspAnnotations }} + annotations: {{ toYaml .Values.rbac.pspAnnotations | nindent 4 }} +{{- end }} +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + # Default set from Docker, with DAC_OVERRIDE and CHOWN + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'csi' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/pvc.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/pvc.yaml new file mode 100644 index 0000000000..8a3ee12226 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/pvc.yaml @@ -0,0 +1,35 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "pvc")}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.persistence.annotations }} + annotations: +{{ toYaml . | indent 4 }} + {{- end }} + {{- with .Values.persistence.finalizers }} + finalizers: +{{ toYaml . | indent 4 }} + {{- end }} +spec: + accessModes: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ required "Must provide size for persistent volumes used by Grafana" .Values.persistence.size | quote }} + {{- if .Values.persistence.storageClassName }} + storageClassName: {{ .Values.persistence.storageClassName }} + {{- end -}} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: +{{ toYaml . | indent 6 }} + {{- end }} +{{- end -}} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/role.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/role.yaml new file mode 100644 index 0000000000..80e2c596ae --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/role.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.rbac.create (not .Values.rbac.useExistingRole) -}} +apiVersion: {{ template "grafana.rbac.apiVersion" . }} +kind: Role +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +{{- if or .Values.global.cattle.psp.enabled (and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled (or .Values.sidecar.datasources.enabled (or .Values.sidecar.plugins.enabled .Values.rbac.extraRoleRules)))) }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ template "grafana.fullname" . }}] +{{- end }} +{{- if and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled (or .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled)) }} +- apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] +{{- end }} +{{- with .Values.rbac.extraRoleRules }} +{{ toYaml . | indent 0 }} +{{- end}} +{{- else }} +rules: [] +{{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/rolebinding.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/rolebinding.yaml new file mode 100644 index 0000000000..e0107255ee --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/rolebinding.yaml @@ -0,0 +1,25 @@ +{{- if .Values.rbac.create -}} +apiVersion: {{ template "grafana.rbac.apiVersion" . }} +kind: RoleBinding +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role +{{- if (not .Values.rbac.useExistingRole) }} + name: {{ template "grafana.fullname" . }} +{{- else }} + name: {{ .Values.rbac.useExistingRole }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "grafana.serviceAccountName" . }} + namespace: {{ template "grafana.namespace" . }} +{{- end -}} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/secret-env.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/secret-env.yaml new file mode 100644 index 0000000000..5c09313e66 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/secret-env.yaml @@ -0,0 +1,14 @@ +{{- if .Values.envRenderSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "grafana.fullname" . }}-env + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +type: Opaque +data: +{{- range $key, $val := .Values.envRenderSecret }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end -}} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/secret.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/secret.yaml new file mode 100644 index 0000000000..c8aa750acb --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/secret.yaml @@ -0,0 +1,26 @@ +{{- if or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +type: Opaque +data: + {{- if and (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }} + admin-user: {{ .Values.adminUser | b64enc | quote }} + {{- if .Values.adminPassword }} + admin-password: {{ .Values.adminPassword | b64enc | quote }} + {{- else }} + admin-password: {{ template "grafana.password" . }} + {{- end }} + {{- end }} + {{- if not .Values.ldap.existingSecret }} + ldap-toml: {{ tpl .Values.ldap.config $ | b64enc | quote }} + {{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/service.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/service.yaml new file mode 100644 index 0000000000..d0a1756c69 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/service.yaml @@ -0,0 +1,55 @@ +{{ if .Values.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.service.labels }} +{{ toYaml .Values.service.labels | indent 4 }} +{{- end }} +{{- $root := . }} +{{- with .Values.service.annotations }} + annotations: +{{ tpl (toYaml . | indent 4) $root }} +{{- end }} +spec: +{{- if (or (eq .Values.service.type "ClusterIP") (empty .Values.service.type)) }} + type: ClusterIP + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{end}} +{{- else if eq .Values.service.type "LoadBalancer" }} + type: {{ .Values.service.type }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml .Values.service.loadBalancerSourceRanges | indent 4 }} + {{- end -}} +{{- else }} + type: {{ .Values.service.type }} +{{- end }} +{{- if .Values.service.externalIPs }} + externalIPs: +{{ toYaml .Values.service.externalIPs | indent 4 }} +{{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP + targetPort: {{ .Values.service.targetPort }} + {{- if .Values.service.appProtocol }} + appProtocol: {{ .Values.service.appProtocol }} + {{- end }} + {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} + nodePort: {{.Values.service.nodePort}} + {{ end }} + {{- if .Values.extraExposePorts }} + {{- tpl (toYaml .Values.extraExposePorts) . | nindent 4 }} + {{- end }} + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} +{{ end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/serviceaccount.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/serviceaccount.yaml new file mode 100644 index 0000000000..4ccee15ed4 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- $root := . }} +{{- with .Values.serviceAccount.annotations }} + annotations: +{{ tpl (toYaml . | indent 4) $root }} +{{- end }} + name: {{ template "grafana.serviceAccountName" . }} + namespace: {{ template "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/servicemonitor.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/servicemonitor.yaml new file mode 100644 index 0000000000..31ab6b8894 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- if .Values.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "grafana.fullname" . }} + {{- if .Values.serviceMonitor.namespace }} + namespace: {{ tpl .Values.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ template "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- if .Values.serviceMonitor.labels }} + {{- toYaml .Values.serviceMonitor.labels | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.serviceMonitor.path }} + scheme: {{ .Values.serviceMonitor.scheme }} + {{- if .Values.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + metricRelabelings: + {{- if .Values.serviceMonitor.metricRelabelings }} + {{- toYaml .Values.serviceMonitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.serviceMonitor.relabelings }} + relabelings: + {{- toYaml .Values.serviceMonitor.relabelings | nindent 4 }} + {{- end }} + jobLabel: "{{ .Release.Name }}" + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + namespaceSelector: + matchNames: + - {{ template "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/statefulset.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/statefulset.yaml new file mode 100644 index 0000000000..aa6f305e22 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/statefulset.yaml @@ -0,0 +1,56 @@ +{{- if (or (.Values.useStatefulSet) (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "statefulset")))}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + serviceName: {{ template "grafana.fullname" . }}-headless + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} +{{- with .Values.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} +{{- end }} +{{- with .Values.podAnnotations }} +{{ toYaml . | indent 8 }} +{{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} + {{- if .Values.persistence.enabled}} + volumeClaimTemplates: + - metadata: + name: storage + spec: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + accessModes: {{ .Values.persistence.accessModes }} + storageClassName: {{ .Values.persistence.storageClassName }} + resources: + requests: + storage: {{ required "Must provide size for persistent volumes used by Grafana" .Values.persistence.size }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: +{{ toYaml . | indent 10 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-configmap.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-configmap.yaml new file mode 100644 index 0000000000..ff53aaf1b3 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-configmap.yaml @@ -0,0 +1,17 @@ +{{- if .Values.testFramework.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "grafana.fullname" . }}-test + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + run.sh: |- + @test "Test Health" { + url="http://{{ template "grafana.fullname" . }}/api/health" + + code=$(wget --server-response --spider --timeout 10 --tries 1 ${url} 2>&1 | awk '/^ HTTP/{print $2}') + [ "$code" == "200" ] + } +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml new file mode 100644 index 0000000000..5dd736efcb --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.testFramework.enabled .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "grafana.fullname" . }}-test + labels: + {{- include "grafana.labels" . | nindent 4 }} +spec: + allowPrivilegeEscalation: true + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + fsGroup: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + runAsUser: + rule: RunAsAny + volumes: + - configMap + - downwardAPI + - emptyDir + - projected + - csi + - secret +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-role.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-role.yaml new file mode 100644 index 0000000000..ea2f8c6b7c --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-role.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.testFramework.enabled .Values.global.cattle.psp.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "grafana.fullname" . }}-test + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ template "grafana.fullname" . }}-test] +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-rolebinding.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-rolebinding.yaml new file mode 100644 index 0000000000..7eda26512b --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.testFramework.enabled .Values.global.cattle.psp.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "grafana.fullname" . }}-test + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "grafana.fullname" . }}-test +subjects: +- kind: ServiceAccount + name: {{ template "grafana.serviceAccountNameTest" . }} + namespace: {{ template "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-serviceaccount.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-serviceaccount.yaml new file mode 100644 index 0000000000..5c33507337 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test-serviceaccount.yaml @@ -0,0 +1,9 @@ +{{- if and .Values.testFramework.enabled .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + name: {{ template "grafana.serviceAccountNameTest" . }} + namespace: {{ template "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test.yaml new file mode 100644 index 0000000000..3a84fbe000 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/templates/tests/test.yaml @@ -0,0 +1,51 @@ +{{- if .Values.testFramework.enabled }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ template "grafana.fullname" . }}-test + labels: + {{- include "grafana.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + namespace: {{ template "grafana.namespace" . }} +spec: + serviceAccountName: {{ template "grafana.serviceAccountNameTest" . }} + {{- if .Values.testFramework.securityContext }} + securityContext: {{ toYaml .Values.testFramework.securityContext | nindent 4 }} + {{- end }} + {{- $root := . }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ tpl . $root }} + {{- end}} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} + {{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 4 }} + {{- end }} + {{- $root := . }} + {{- with .Values.affinity }} + affinity: +{{ tpl (toYaml .) $root | indent 4 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 4 }} +{{- end }} + containers: + - name: {{ .Release.Name }}-test + image: "{{ template "system_default_registry" . }}{{ .Values.testFramework.image}}:{{ .Values.testFramework.tag }}" + imagePullPolicy: "{{ .Values.testFramework.imagePullPolicy}}" + command: ["/opt/bats/bin/bats", "-t", "/tests/run.sh"] + volumeMounts: + - mountPath: /tests + name: tests + readOnly: true + volumes: + - name: tests + configMap: + name: {{ template "grafana.fullname" . }}-test + restartPolicy: Never +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/values.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/values.yaml new file mode 100644 index 0000000000..196a07bfc9 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/charts/grafana/values.yaml @@ -0,0 +1,1062 @@ +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + projectNamespaces: [] + +rbac: + create: true + ## Use an existing ClusterRole/Role (depending on rbac.namespaced false/true) + # useExistingRole: name-of-some-(cluster)role + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default,runtime/default' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + namespaced: false + extraRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] + extraClusterRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] +serviceAccount: + create: true + name: + nameTest: +## Service account annotations. Can be templated. +# annotations: +# eks.amazonaws.com/role-arn: arn:aws:iam::123456789000:role/iam-role-name-here + autoMount: true + +replicas: 1 + +## Create a headless service for the deployment +headlessService: false + +## Create HorizontalPodAutoscaler object for deployment type +# +autoscaling: + enabled: false +# minReplicas: 1 +# maxReplicas: 10 +# metrics: +# - type: Resource +# resource: +# name: cpu +# targetAverageUtilization: 60 +# - type: Resource +# resource: +# name: memory +# targetAverageUtilization: 60 + +## See `kubectl explain poddisruptionbudget.spec` for more +## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} +# minAvailable: 1 +# maxUnavailable: 1 + +## See `kubectl explain deployment.spec.strategy` for more +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +deploymentStrategy: + type: RollingUpdate + +readinessProbe: + httpGet: + path: /api/health + port: 3000 + +livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + failureThreshold: 10 + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: "default-scheduler" + +image: + repository: rancher/mirrored-grafana-grafana + # Overrides the Grafana image tag whose default is the chart appVersion + tag: 9.1.5 + sha: "" + pullPolicy: IfNotPresent + + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Can be templated. + ## + # pullSecrets: + # - myRegistrKeySecretName + +testFramework: + enabled: false + image: "rancher/mirrored-bats-bats" + tag: "v1.4.1" + imagePullPolicy: IfNotPresent + securityContext: + runAsNonRoot: true + runAsUser: 1000 + +securityContext: + runAsNonRoot: true + runAsUser: 472 + runAsGroup: 472 + fsGroup: 472 + +containerSecurityContext: + {} + +# Enable creating the grafana configmap +createConfigmap: true + +# Extra configmaps to mount in grafana pods +# Values are templated. +extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # subPath: certificates.crt # (optional) + # configMap: certs-configmap + # readOnly: true + + +extraEmptyDirMounts: [] + # - name: provisioning-notifiers + # mountPath: /etc/grafana/provisioning/notifiers + + +# Apply extra labels to common labels. +extraLabels: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: + +downloadDashboardsImage: + repository: rancher/mirrored-curlimages-curl + tag: 7.85.0 + sha: "" + pullPolicy: IfNotPresent + +downloadDashboards: + env: {} + envFromSecret: "" + resources: {} + securityContext: {} + +## Pod Annotations +# podAnnotations: {} + +## Pod Labels +# podLabels: {} + +podPortName: grafana + +## Deployment annotations +# annotations: {} + +## Expose the grafana service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## +service: + enabled: true + type: ClusterIP + port: 80 + targetPort: 3000 + # targetPort: 4181 To be used with a proxy extraContainer + ## Service annotations. Can be templated. + annotations: {} + labels: {} + portName: service + # Adds the appProtocol field to the service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + +serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 1m + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + +extraExposePorts: [] + # - name: keycloak + # port: 8080 + # targetPort: 8080 + # type: ClusterIP + +# overrides pod.spec.hostAliases in the grafana deployment's pods +hostAliases: [] + # - ip: "1.2.3.4" + # hostnames: + # - "my.host.com" + +ingress: + enabled: false + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + # Values can be templated + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + labels: {} + path: / + + # pathType is only for k8s >= 1.18 + pathType: Prefix + + hosts: + - chart-example.local + ## Extra paths to prepend to every host configuration. This is useful when working with annotation based services. + extraPaths: [] + # - path: /* + # backend: + # serviceName: ssl-redirect + # servicePort: use-annotation + ## Or for k8s > 1.19 + # - path: /* + # pathType: Prefix + # backend: + # service: + # name: ssl-redirect + # port: + # name: use-annotation + + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +## Node labels for pod assignment +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +# +nodeSelector: {} + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Topology Spread Constraints +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +## +topologySpreadConstraints: [] + +## Additional init containers (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +## +extraInitContainers: [] + +## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod +extraContainers: "" +# extraContainers: | +# - name: proxy +# image: quay.io/gambol99/keycloak-proxy:latest +# args: +# - -provider=github +# - -client-id= +# - -client-secret= +# - -github-org= +# - -email-domain=* +# - -cookie-secret= +# - -http-address=http://0.0.0.0:4181 +# - -upstream-url=http://127.0.0.1:3000 +# ports: +# - name: proxy-web +# containerPort: 4181 + +## Volumes that can be used in init containers that will not be mounted to deployment pods +extraContainerVolumes: [] +# - name: volume-from-secret +# secret: +# secretName: secret-to-mount +# - name: empty-dir-volume +# emptyDir: {} + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + type: pvc + enabled: false + # storageClassName: default + accessModes: + - ReadWriteOnce + size: 10Gi + # annotations: {} + finalizers: + - kubernetes.io/pvc-protection + # selectorLabels: {} + ## Sub-directory of the PV to mount. Can be templated. + # subPath: "" + ## Name of an existing PVC. Can be templated. + # existingClaim: + + ## If persistence is not enabled, this allows to mount the + ## local storage in-memory to improve performance + ## + inMemory: + enabled: false + ## The maximum usage on memory medium EmptyDir would be + ## the minimum value between the SizeLimit specified + ## here and the sum of memory limits of all containers in a pod + ## + # sizeLimit: 300Mi + +initChownData: + ## If false, data ownership will not be reset at startup + ## This allows the prometheus-server to be run with an arbitrary user + ## + enabled: true + + ## initChownData container image + ## + image: + repository: rancher/mirrored-library-busybox + tag: "1.31.1" + sha: "" + pullPolicy: IfNotPresent + + ## initChownData resource requests and limits + ## Ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + +# Administrator credentials when not using an existing secret (see below) +adminUser: admin +# adminPassword: strongpassword + +# Use an existing secret for the admin user. +admin: + ## Name of the secret. Can be templated. + existingSecret: "" + userKey: admin-user + passwordKey: admin-password + +## Define command to be executed at startup by grafana container +## Needed if using `vault-env` to manage secrets (ref: https://banzaicloud.com/blog/inject-secrets-into-pods-vault/) +## Default is "run.sh" as defined in grafana's Dockerfile +# command: +# - "sh" +# - "/run.sh" + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: + +## Extra environment variables that will be pass onto deployment pods +## +## to provide grafana with access to CloudWatch on AWS EKS: +## 1. create an iam role of type "Web identity" with provider oidc.eks.* (note the provider for later) +## 2. edit the "Trust relationships" of the role, add a line inside the StringEquals clause using the +## same oidc eks provider as noted before (same as the existing line) +## also, replace NAMESPACE and prometheus-operator-grafana with the service account namespace and name +## +## "oidc.eks.us-east-1.amazonaws.com/id/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:sub": "system:serviceaccount:NAMESPACE:prometheus-operator-grafana", +## +## 3. attach a policy to the role, you can use a built in policy called CloudWatchReadOnlyAccess +## 4. use the following env: (replace 123456789000 and iam-role-name-here with your aws account number and role name) +## +## env: +## AWS_ROLE_ARN: arn:aws:iam::123456789000:role/iam-role-name-here +## AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token +## AWS_REGION: us-east-1 +## +## 5. uncomment the EKS section in extraSecretMounts: below +## 6. uncomment the annotation section in the serviceAccount: above +## make sure to replace arn:aws:iam::123456789000:role/iam-role-name-here with your role arn + +env: {} + +## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core +## Renders in container spec as: +## env: +## ... +## - name: +## valueFrom: +## +envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## The name of a secret in the same kubernetes namespace which contain values to be added to the environment +## This can be useful for auth tokens, etc. Value is templated. +envFromSecret: "" + +## Sensible environment variables that will be rendered as new secret object +## This can be useful for auth tokens, etc +envRenderSecret: {} + +## The names of secrets in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the secret must be defined with an optional key. +## Name is templated. +envFromSecrets: [] +## - name: secret-name +## optional: true + +## The names of conifgmaps in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the configmap must be defined with an optional key. +## Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#configmapenvsource-v1-core +envFromConfigMaps: [] +## - name: configmap-name +## optional: true + +# Inject Kubernetes services as environment variables. +# See https://kubernetes.io/docs/concepts/services-networking/connect-applications-service/#environment-variables +enableServiceLinks: true + +## Additional grafana server secret mounts +# Defines additional mounts with secrets. Secrets must be manually created in the namespace. +extraSecretMounts: [] + # - name: secret-files + # mountPath: /etc/secrets + # secretName: grafana-secret-files + # readOnly: true + # subPath: "" + # + # for AWS EKS (cloudwatch) use the following (see also instruction in env: above) + # - name: aws-iam-token + # mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount + # readOnly: true + # projected: + # defaultMode: 420 + # sources: + # - serviceAccountToken: + # audience: sts.amazonaws.com + # expirationSeconds: 86400 + # path: token + # + # for CSI e.g. Azure Key Vault use the following + # - name: secrets-store-inline + # mountPath: /run/secrets + # readOnly: true + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "akv-grafana-spc" + # nodePublishSecretRef: # Only required when using service principal mode + # name: grafana-akv-creds # Only required when using service principal mode + +## Additional grafana server volume mounts +# Defines additional volume mounts. +extraVolumeMounts: [] + # - name: extra-volume-0 + # mountPath: /mnt/volume0 + # readOnly: true + # existingClaim: volume-claim + # - name: extra-volume-1 + # mountPath: /mnt/volume1 + # readOnly: true + # hostPath: /usr/shared/ + # - name: grafana-secrets + # csi: true + # data: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "grafana-env-spc" + +## Container Lifecycle Hooks. Execute a specific bash command or make an HTTP request +lifecycleHooks: {} + # postStart: + # exec: + # command: [] + +## Pass the plugins you want installed as a list. +## +plugins: [] + # - digrich-bubblechart-panel + # - grafana-clock-panel + +## Configure grafana datasources +## ref: http://docs.grafana.org/administration/provisioning/#datasources +## +datasources: {} +# datasources.yaml: +# apiVersion: 1 +# datasources: +# - name: Prometheus +# type: prometheus +# url: http://prometheus-prometheus-server +# access: proxy +# isDefault: true +# - name: CloudWatch +# type: cloudwatch +# access: proxy +# uid: cloudwatch +# editable: false +# jsonData: +# authType: default +# defaultRegion: us-east-1 + +## Configure grafana alerting (can be templated) +## ref: http://docs.grafana.org/administration/provisioning/#alerting +## +alerting: {} + # rules.yaml: + # apiVersion: 1 + # groups: + # - orgId: 1 + # name: '{{ .Chart.Name }}_my_rule_group' + # folder: my_first_folder + # interval: 60s + # rules: + # - uid: my_id_1 + # title: my_first_rule + # condition: A + # data: + # - refId: A + # datasourceUid: '-100' + # model: + # conditions: + # - evaluator: + # params: + # - 3 + # type: gt + # operator: + # type: and + # query: + # params: + # - A + # reducer: + # type: last + # type: query + # datasource: + # type: __expr__ + # uid: '-100' + # expression: 1==0 + # intervalMs: 1000 + # maxDataPoints: 43200 + # refId: A + # type: math + # dashboardUid: my_dashboard + # panelId: 123 + # noDataState: Alerting + # for: 60s + # annotations: + # some_key: some_value + # labels: + # team: sre_team_1 + # contactpoints.yaml: + # apiVersion: 1 + # contactPoints: + # - orgId: 1 + # name: cp_1 + # receivers: + # - uid: first_uid + # type: pagerduty + # settings: + # integrationKey: XXX + # severity: critical + # class: ping failure + # component: Grafana + # group: app-stack + # summary: | + # {{ `{{ template "default.message" . }}` }} + +## Configure notifiers +## ref: http://docs.grafana.org/administration/provisioning/#alert-notification-channels +## +notifiers: {} +# notifiers.yaml: +# notifiers: +# - name: email-notifier +# type: email +# uid: email1 +# # either: +# org_id: 1 +# # or +# org_name: Main Org. +# is_default: true +# settings: +# addresses: an_email_address@example.com +# delete_notifiers: + +## Configure grafana dashboard providers +## ref: http://docs.grafana.org/administration/provisioning/#dashboards +## +## `path` must be /var/lib/grafana/dashboards/ +## +dashboardProviders: {} +# dashboardproviders.yaml: +# apiVersion: 1 +# providers: +# - name: 'default' +# orgId: 1 +# folder: '' +# type: file +# disableDeletion: false +# editable: true +# options: +# path: /var/lib/grafana/dashboards/default + +## Configure grafana dashboard to import +## NOTE: To use dashboards you must also enable/configure dashboardProviders +## ref: https://grafana.com/dashboards +## +## dashboards per provider, use provider name as key. +## +dashboards: {} + # default: + # some-dashboard: + # json: | + # $RAW_JSON + # custom-dashboard: + # file: dashboards/custom-dashboard.json + # prometheus-stats: + # gnetId: 2 + # revision: 2 + # datasource: Prometheus + # local-dashboard: + # url: https://example.com/repository/test.json + # token: '' + # local-dashboard-base64: + # url: https://example.com/repository/test-b64.json + # token: '' + # b64content: true + # local-dashboard-gitlab: + # url: https://example.com/repository/test-gitlab.json + # gitlabToken: '' + # local-dashboard-bitbucket: + # url: https://example.com/repository/test-bitbucket.json + # bearerToken: '' + +## Reference to external ConfigMap per provider. Use provider name as key and ConfigMap name as value. +## A provider dashboards must be defined either by external ConfigMaps or in values.yaml, not in both. +## ConfigMap data example: +## +## data: +## example-dashboard.json: | +## RAW_JSON +## +dashboardsConfigMaps: {} +# default: "" + +## Grafana's primary configuration +## NOTE: values in map will be converted to ini format +## ref: http://docs.grafana.org/installation/configuration/ +## +grafana.ini: + paths: + data: /var/lib/grafana/ + logs: /var/log/grafana + plugins: /var/lib/grafana/plugins + provisioning: /etc/grafana/provisioning + analytics: + check_for_updates: true + log: + mode: console + grafana_net: + url: https://grafana.net + server: + domain: "{{ if (and .Values.ingress.enabled .Values.ingress.hosts) }}{{ .Values.ingress.hosts | first }}{{ end }}" +## grafana Authentication can be enabled with the following values on grafana.ini + # server: + # The full public facing url you use in browser, used for redirects and emails + # root_url: + # https://grafana.com/docs/grafana/latest/auth/github/#enable-github-in-grafana + # auth.github: + # enabled: false + # allow_sign_up: false + # scopes: user:email,read:org + # auth_url: https://github.com/login/oauth/authorize + # token_url: https://github.com/login/oauth/access_token + # api_url: https://api.github.com/user + # team_ids: + # allowed_organizations: + # client_id: + # client_secret: +## LDAP Authentication can be enabled with the following values on grafana.ini +## NOTE: Grafana will fail to start if the value for ldap.toml is invalid + # auth.ldap: + # enabled: true + # allow_sign_up: true + # config_file: /etc/grafana/ldap.toml + +## Grafana's LDAP configuration +## Templated by the template in _helpers.tpl +## NOTE: To enable the grafana.ini must be configured with auth.ldap.enabled +## ref: http://docs.grafana.org/installation/configuration/#auth-ldap +## ref: http://docs.grafana.org/installation/ldap/#configuration +ldap: + enabled: false + # `existingSecret` is a reference to an existing secret containing the ldap configuration + # for Grafana in a key `ldap-toml`. + existingSecret: "" + # `config` is the content of `ldap.toml` that will be stored in the created secret + config: "" + # config: |- + # verbose_logging = true + + # [[servers]] + # host = "my-ldap-server" + # port = 636 + # use_ssl = true + # start_tls = false + # ssl_skip_verify = false + # bind_dn = "uid=%s,ou=users,dc=myorg,dc=com" + +## Grafana's SMTP configuration +## NOTE: To enable, grafana.ini must be configured with smtp.enabled +## ref: http://docs.grafana.org/installation/configuration/#smtp +smtp: + # `existingSecret` is a reference to an existing secret containing the smtp configuration + # for Grafana. + existingSecret: "" + userKey: "user" + passwordKey: "password" + +## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders +## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards +sidecar: + image: + repository: rancher/mirrored-kiwigrid-k8s-sidecar + tag: 1.19.2 + sha: "" + imagePullPolicy: IfNotPresent + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + securityContext: {} + # skipTlsVerify Set to true to skip tls verification for kube api calls + # skipTlsVerify: true + enableUniqueFilenames: false + readinessProbe: {} + livenessProbe: {} + # Log level default for all sidecars. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. Defaults to INFO + # logLevel: INFO + dashboards: + enabled: false + # Additional environment variables for the dashboards sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + SCProvider: true + # label that the configmaps with dashboards are marked with + label: grafana_dashboard + # value of label that the configmaps with dashboards are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # folder in the pod that should hold the collected dashboards (unless `defaultFolderName` is set) + folder: /tmp/dashboards + # The default folder name, it will create a subfolder under the `folder` and put dashboards in there instead + defaultFolderName: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # If specified, the sidecar will look for annotation with this name to create folder and put graph here. + # You can use this parameter together with `provider.foldersFromFilesStructure`to annotate configmaps and create folder structure. + folderAnnotation: null + # Absolute path to shell script to execute after a configmap got reloaded + script: null + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # provider configuration that lets grafana manage the dashboards + provider: + # name of the provider, should be unique + name: sidecarProvider + # orgid as configured in grafana + orgid: 1 + # folder in which the dashboards should be imported in grafana + folder: '' + # type of the provider + type: file + # disableDelete to activate a import-only behaviour + disableDelete: false + # allow updating provisioned dashboards from the UI + allowUiUpdates: false + # allow Grafana to replicate dashboard structure from filesystem + foldersFromFilesStructure: false + # Additional dashboard sidecar volume mounts + extraMounts: [] + # Sets the size limit of the dashboard sidecar emptyDir volume + sizeLimit: {} + datasources: + enabled: false + # Additional environment variables for the datasourcessidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with datasources are marked with + label: grafana_datasource + # value of label that the configmaps with datasources are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload datasources + reloadURL: "http://localhost:3000/api/admin/provisioning/datasources/reload" + # Absolute path to shell script to execute after a datasource got reloaded + script: null + skipReload: true + # Deploy the datasource sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any datasources defined at startup time. + initDatasources: true + # Sets the size limit of the datasource sidecar emptyDir volume + sizeLimit: {} + plugins: + enabled: false + # Additional environment variables for the plugins sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with plugins are marked with + label: grafana_plugin + # value of label that the configmaps with plugins are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload plugins + reloadURL: "http://localhost:3000/api/admin/provisioning/plugins/reload" + # Absolute path to shell script to execute after a plugin got reloaded + script: null + skipReload: false + # Deploy the datasource sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any plugins defined at startup time. + initPlugins: false + # Sets the size limit of the plugin sidecar emptyDir volume + sizeLimit: {} + notifiers: + enabled: false + # Additional environment variables for the notifierssidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with notifiers are marked with + label: grafana_notifier + # value of label that the configmaps with notifiers are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # search in configmap, secret or both + resource: both + # Sets the size limit of the notifier sidecar emptyDir volume + sizeLimit: {} + +## Override the deployment namespace +## +namespaceOverride: "" + +## Number of old ReplicaSets to retain +## +revisionHistoryLimit: 10 + +## Add a seperate remote image renderer deployment/service +imageRenderer: + # Enable the image-renderer deployment & service + enabled: false + replicas: 1 + image: + # image-renderer Image repository + repository: rancher/mirrored-grafana-grafana-image-renderer + # image-renderer Image tag + tag: 3.0.1 + # image-renderer Image sha (optional) + sha: "" + # image-renderer ImagePullPolicy + pullPolicy: Always + # extra environment variables + env: + HTTP_HOST: "0.0.0.0" + # RENDERING_ARGS: --no-sandbox,--disable-gpu,--window-size=1280x758 + # RENDERING_MODE: clustered + # IGNORE_HTTPS_ERRORS: true + # image-renderer deployment serviceAccount + serviceAccountName: "" + # image-renderer deployment securityContext + securityContext: {} + # image-renderer deployment Host Aliases + hostAliases: [] + # image-renderer deployment priority class + priorityClassName: '' + service: + # Enable the image-renderer service + enabled: true + # image-renderer service port name + portName: 'http' + # image-renderer service port used by both service and deployment + port: 8081 + targetPort: 8081 + # Adds the appProtocol field to the image-renderer service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + # If https is enabled in Grafana, this needs to be set as 'https' to correctly configure the callback used in Grafana + grafanaProtocol: http + # In case a sub_path is used this needs to be added to the image renderer callback + grafanaSubPath: "" + # name of the image-renderer port on the pod + podPortName: http + # number of image-renderer replica sets to keep + revisionHistoryLimit: 10 + networkPolicy: + # Enable a NetworkPolicy to limit inbound traffic to only the created grafana pods + limitIngress: true + # Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods + limitEgress: false + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + ## Node labels for pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + # + nodeSelector: {} + + ## Tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## Affinity for pod assignment (evaluated as template) + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + +networkPolicy: + ## @param networkPolicy.enabled Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now. + ## + enabled: false + ## @param networkPolicy.allowExternal Don't require client label for connections + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to grafana port defined. + ## When true, grafana will accept connections from any source + ## (with the correct destination port). + ## + ingress: true + ## @param networkPolicy.ingress When true enables the creation + ## an ingress network policy + ## + allowExternal: true + ## @param networkPolicy.explicitNamespacesSelector A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed + ## If explicitNamespacesSelector is missing or set to {}, only client Pods that are in the networkPolicy's namespace + ## and that match other criteria, the ones that have the good label, can reach the grafana. + ## But sometimes, we want the grafana to be accessible to clients from other namespaces, in this case, we can use this + ## LabelSelector to select these namespaces, note that the networkPolicy's namespace should also be explicitly added. + ## + ## Example: + ## explicitNamespacesSelector: + ## matchLabels: + ## role: frontend + ## matchExpressions: + ## - {key: role, operator: In, values: [frontend]} + ## + explicitNamespacesSelector: {} + ## + ## + ## + ## + ## + ## + egress: + ## @param networkPolicy.egress.enabled When enabled, an egress network policy will be + ## created allowing grafana to connect to external data sources from kubernetes cluster. + enabled: false + ## + ## @param networkPolicy.egress.ports Add individual ports to be allowed by the egress + ports: [] + ## Add ports to the egress by specifying - port: + ## E.X. + ## ports: + ## - port: 80 + ## - port: 443 + ## + ## + ## + ## + ## + ## + +# Enable backward compatibility of kubernetes where version below 1.13 doesn't have the enableServiceLinks option +enableKubeBackwardCompatibility: false +useStatefulSet: false diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/files/federate/federate-scrape-config.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/files/federate/federate-scrape-config.yaml new file mode 100644 index 0000000000..b5ceb698f9 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/files/federate/federate-scrape-config.yaml @@ -0,0 +1,13 @@ +- job_name: 'federate' + scrape_interval: {{ .Values.federate.interval }} + honor_labels: true + metrics_path: '/federate' + + params: + 'match[]': + - '{namespace=~"{{ include "project-prometheus-stack.projectNamespaceList" . | replace "," "|" }}", job=~"kube-state-metrics|kubelet|k3s-server"}' + - '{namespace=~"{{ include "project-prometheus-stack.projectNamespaceList" . | replace "," "|" }}", job=""}' + + static_configs: + - targets: {{ .Values.federate.targets | toYaml | nindent 6 }} + diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/files/rancher/pods/rancher-pod-containers.json b/charts/rancher-project-monitoring/0.3.0+up0.3.3/files/rancher/pods/rancher-pod-containers.json new file mode 100644 index 0000000000..9e53081a73 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/files/rancher/pods/rancher-pod-containers.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "CFS throttled ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "System ({{container}})", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Total ({{container}})", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "User ({{container}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}) by (container)", + "interval": "", + "legendFormat": "({{container}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Dropped ({{container}})", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Errors ({{container}})", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Errors ({{container}})", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Dropped ({{container}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Write ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Read ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod (Containers)", + "uid": "rancher-pod-containers-1", + "version": 8 +} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/files/rancher/pods/rancher-pod.json b/charts/rancher-project-monitoring/0.3.0+up0.3.3/files/rancher/pods/rancher-pod.json new file mode 100644 index 0000000000..65c6bf18ed --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/files/rancher/pods/rancher-pod.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values({__name__=~\"container_.*|windows_container_.*\", namespace!=\"\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values({__name__=~\"container_.*|windows_container_.*\", namespace=\"$namespace\", pod!=\"\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod", + "uid": "rancher-pod-1", + "version": 8 +} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/files/rancher/workloads/rancher-workload-pods.json b/charts/rancher-project-monitoring/0.3.0+up0.3.3/files/rancher/workloads/rancher-workload-pods.json new file mode 100644 index 0000000000..f6b5078afb --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/files/rancher/workloads/rancher-workload-pods.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "CFS throttled ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "System ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Total ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "User ({{pod}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(container_memory_working_set_bytes{namespace=~\"$namespace\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "({{pod}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Dropped ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Errors ({{pod}})", + "refId": "D" + }, + { + "expr": "(sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Errors ({{pod}})", + "refId": "E" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Dropped ({{pod}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Write ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Read ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload (Pods)", + "uid": "rancher-workload-pods-1", + "version": 8 +} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/files/rancher/workloads/rancher-workload.json b/charts/rancher-project-monitoring/0.3.0+up0.3.3/files/rancher/workloads/rancher-workload.json new file mode 100644 index 0000000000..9f5317c2f0 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/files/rancher/workloads/rancher-workload.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum((sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum((sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(container_memory_working_set_bytes{namespace=~\"$namespace\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum((sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum((sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum((sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload", + "uid": "rancher-workload-1", + "version": 8 +} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/questions.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/questions.yaml new file mode 100644 index 0000000000..c15f36e7ec --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/questions.yaml @@ -0,0 +1,128 @@ +questions: +- variable: alertmanager.enabled + label: Enable Alertmanager + default: true + type: boolean + group: Alertmanager +- variable: grafana.enabled + label: Enable Grafana + default: true + type: boolean + group: Grafana + show_subquestion_if: true + subquestions: + - variable: grafana.adminUser + label: Grafana Admin User + type: string + default: admin + group: Grafana + - variable: grafana.adminPassword + label: Grafana Admin Password + type: string + default: prom-operator + group: Grafana + - variable: grafana.sidecar.dashboards.label + label: Default Grafana Dashboard Label + default: grafana_dashboard + description: All ConfigMaps with this label are parsed as Grafana Dashboards + type: string + group: Grafana +- variable: grafana.persistence.enabled + type: boolean + required: true + label: Enable Persistent Volume + show_subquestion_if: true + group: Grafana + subquestions: + - variable: grafana.persistence.size + type: string + default: 1Gi + label: Size + - variable: grafana.persistence.storageClass + type: storageclass + label: Storage Class Name + - variable: grafana.persistence.accessModes + type: enum + default: + - ReadWriteOnce + options: + - [ReadWriteOnce] + - [ReadWriteMany] + - [ReadOnlyMany] + label: Access Mode +- variable: prometheus.prometheusSpec.scrapeInterval + type: string + required: true + label: Scrape Interval + default: 30s + group: Prometheus +- variable: prometheus.prometheusSpec.evaluationInterval + type: string + required: true + label: Evaluation Interval + default: 1m + group: Prometheus +- variable: prometheus.prometheusSpec.retention + type: string + required: true + label: Retention + default: 10d + group: Prometheus +- variable: prometheus.prometheusSpec.retentionSize + type: string + required: false + label: Retention Size + default: 50GB + group: Prometheus +- variable: prometheus.prometheusSpec.resources.requests.cpu + type: string + required: true + label: Requested CPU + default: 750m + group: Prometheus +- variable: prometheus.prometheusSpec.resources.requests.memory + type: string + required: true + label: Requested Memory + default: 750Mi + group: Prometheus +- variable: prometheus.prometheusSpec.resources.limits.cpu + type: string + required: true + label: CPU Limit + default: 1000m + group: Prometheus +- variable: prometheus.prometheusSpec.resources.limits.memory + type: string + required: true + label: Memory Limit + default: 3000Mi + group: Prometheus +- variable: prometheus.prometheusSpec.storage.enabled + type: boolean + required: true + label: Enable Persistent Volume + show_subquestion_if: true + group: Prometheus + subquestions: + - variable: prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage + type: string + default: 50Gi + label: Size + - variable: prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.storageClassName + type: storageclass + label: Storage Class Name + - variable: prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.accessModes + type: enum + default: + - ReadWriteOnce + options: + - [ReadWriteOnce] + - [ReadWriteMany] + - [ReadOnlyMany] + label: Access Mode +- variable: federate.enabled + label: Enable Federated Metrics From Cluster Prometheus + default: true + type: boolean + group: Federation diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/NOTES.txt b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/NOTES.txt new file mode 100644 index 0000000000..9e4d6d87ca --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/NOTES.txt @@ -0,0 +1,4 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "project-prometheus-stack.namespace" . }} get pods -l "release={{ $.Release.Name }}" + +Visit https://github.com/prometheus-operator/kube-prometheus for instructions on how to create & configure Alertmanager and Prometheus instances using the Operator. diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/_helpers.tpl b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/_helpers.tpl new file mode 100644 index 0000000000..5333d28be7 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/_helpers.tpl @@ -0,0 +1,252 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{/* +https://github.com/helm/helm/issues/4535#issuecomment-477778391 +Usage: {{ include "call-nested" (list . "SUBCHART_NAME" "TEMPLATE") }} +e.g. {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +*/}} +{{- define "call-nested" }} +{{- $dot := index . 0 }} +{{- $subchart := index . 1 | splitList "." }} +{{- $template := index . 2 }} +{{- $values := $dot.Values }} +{{- range $subchart }} +{{- $values = index $values . }} +{{- end }} +{{- include $template (dict "Chart" (dict "Name" (last $subchart)) "Values" $values "Release" $dot.Release "Capabilities" $dot.Capabilities) }} +{{- end }} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Prometheus Operator + +{{/* Comma-delimited list of namespaces that need to be watched to configure Project Prometheus Stack components */}} +{{- define "project-prometheus-stack.projectNamespaceList" -}} +{{ append .Values.global.cattle.projectNamespaces .Release.Namespace | uniq | join "," }} +{{- end }} + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "project-prometheus-stack.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "project-prometheus-stack.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 26 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Prometheus custom resource instance name */}} +{{- define "project-prometheus-stack.prometheus.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "project-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "project-prometheus-stack.fullname" .) "-prometheus" }} +{{- end }} +{{- end }} + +{{/* Alertmanager custom resource instance name */}} +{{- define "project-prometheus-stack.alertmanager.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "project-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "project-prometheus-stack.fullname" .) "-alertmanager" -}} +{{- end }} +{{- end }} + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "project-prometheus-stack.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "project-prometheus-stack.labels" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "project-prometheus-stack.name" . }} +chart: {{ template "project-prometheus-stack.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end }} + +{{/* Create the name of prometheus service account to use */}} +{{- define "project-prometheus-stack.prometheus.serviceAccountName" -}} +{{- if .Values.prometheus.serviceAccount.create -}} + {{ default (print (include "project-prometheus-stack.fullname" .) "-prometheus") .Values.prometheus.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheus.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of alertmanager service account to use */}} +{{- define "project-prometheus-stack.alertmanager.serviceAccountName" -}} +{{- if .Values.alertmanager.serviceAccount.create -}} + {{ default (print (include "project-prometheus-stack.fullname" .) "-alertmanager") .Values.alertmanager.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.alertmanager.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "project-prometheus-stack.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the grafana namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "project-prometheus-stack-grafana.namespace" -}} + {{- if .Values.grafana.namespaceOverride -}} + {{- .Values.grafana.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* Allow KubeVersion to be overridden. */}} +{{- define "project-prometheus-stack.kubeVersion" -}} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} +{{- end -}} + +{{/* Get Ingress API Version */}} +{{- define "project-prometheus-stack.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" (include "project-prometheus-stack.kubeVersion" .)) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* Check Ingress stability */}} +{{- define "project-prometheus-stack.ingress.isStable" -}} + {{- eq (include "project-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} + +{{/* Check Ingress supports pathType */}} +{{/* pathType was added to networking.k8s.io/v1beta1 in Kubernetes 1.18 */}} +{{- define "project-prometheus-stack.ingress.supportsPathType" -}} + {{- or (eq (include "project-prometheus-stack.ingress.isStable" .) "true") (and (eq (include "project-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" (include "project-prometheus-stack.kubeVersion" .))) -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "project-prometheus-stack.pdb.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" (include "project-prometheus-stack.kubeVersion" .)) -}} + {{- print "policy/v1" -}} + {{- else -}} + {{- print "policy/v1beta1" -}} + {{- end -}} + {{- end -}} + +{{/* Get value based on current Kubernetes version */}} +{{- define "project-prometheus-stack.kubeVersionDefaultValue" -}} + {{- $values := index . 0 }} + {{- $kubeVersion := index . 1 }} + {{- $old := index . 2 }} + {{- $new := index . 3 }} + {{- $default := index . 4 }} + {{- if kindIs "invalid" $default -}} + {{- if semverCompare $kubeVersion (include "project-prometheus-stack.kubeVersion" $values) -}} + {{- print $new -}} + {{- else -}} + {{- print $old -}} + {{- end -}} + {{- else -}} + {{- print $default }} + {{- end -}} +{{- end -}} + +{{/* +To help compatibility with other charts which use global.imagePullSecrets. +Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). +global: + imagePullSecrets: + - name: pullSecret1 + - name: pullSecret2 + +or + +global: + imagePullSecrets: + - pullSecret1 + - pullSecret2 +*/}} +{{- define "project-prometheus-stack.imagePullSecrets" -}} +{{- range .Values.global.imagePullSecrets }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + + +{{/* Define ingress for all hardened namespaces */}} +{{- define "project-prometheus-stack.hardened.networkPolicy.ingress" -}} +{{- $root := index . 0 }} +{{- $ns := index . 1 }} +{{- if $root.Values.global.networkPolicy.ingress -}} +{{ toYaml $root.Values.global.networkPolicy.ingress }} +{{- end }} +{{- if $root.Values.global.networkPolicy.limitIngressToProject }} +- from: +{{- if $root.Values.global.cattle.projectNamespaceSelector }} + - namespaceSelector: {{- $root.Values.global.cattle.projectNamespaceSelector | toYaml | nindent 6 }} +{{- end }} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ $ns }} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/alertmanager.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/alertmanager.yaml new file mode 100644 index 0000000000..f8b3db0c4c --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/alertmanager.yaml @@ -0,0 +1,165 @@ +{{- if .Values.alertmanager.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: Alertmanager +metadata: + name: {{ template "project-prometheus-stack.alertmanager.crname" . }} + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-alertmanager +{{ include "project-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.annotations }} + annotations: +{{ toYaml .Values.alertmanager.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.alertmanagerSpec.image }} + {{- if and .Values.alertmanager.alertmanagerSpec.image.tag .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.alertmanager.alertmanagerSpec.image.repository }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.tag }} + image: "{{ template "system_default_registry" . }}{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.alertmanager.alertmanagerSpec.image.repository }}" + {{- end }} + version: {{ .Values.alertmanager.alertmanagerSpec.image.tag }} + {{- if .Values.alertmanager.alertmanagerSpec.image.sha }} + sha: {{ .Values.alertmanager.alertmanagerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.alertmanager.alertmanagerSpec.replicas }} + listenLocal: {{ .Values.alertmanager.alertmanagerSpec.listenLocal }} + serviceAccountName: {{ template "project-prometheus-stack.alertmanager.serviceAccountName" . }} +{{- if .Values.alertmanager.alertmanagerSpec.externalUrl }} + externalUrl: "{{ tpl .Values.alertmanager.alertmanagerSpec.externalUrl . }}" +{{- else if and .Values.alertmanager.ingress.enabled .Values.alertmanager.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.alertmanager.ingress.hosts 0) . }}{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- else if not (or (empty .Values.global.cattle.url) (empty .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "project-prometheus-stack.namespace" . }}/services/http:{{ template "project-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "project-prometheus-stack.fullname" . }}-alertmanager.{{ template "project-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.nodeSelector }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.alertmanager.alertmanagerSpec.paused }} + logFormat: {{ .Values.alertmanager.alertmanagerSpec.logFormat | quote }} + logLevel: {{ .Values.alertmanager.alertmanagerSpec.logLevel | quote }} + retention: {{ .Values.alertmanager.alertmanagerSpec.retention | quote }} +{{- if .Values.alertmanager.alertmanagerSpec.secrets }} + secrets: +{{ toYaml .Values.alertmanager.alertmanagerSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configSecret }} + configSecret: {{ .Values.alertmanager.alertmanagerSpec.configSecret }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configMaps }} + configMaps: +{{ toYaml .Values.alertmanager.alertmanagerSpec.configMaps | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector }} + alertmanagerConfigSelector: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector | indent 4}} +{{ else }} + alertmanagerConfigSelector: {} +{{- end }} + alertmanagerConfigNamespaceSelector: {{ .Values.global.cattle.projectNamespaceSelector | toYaml | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.web }} + web: +{{ toYaml .Values.alertmanager.alertmanagerSpec.web | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration }} + alertmanagerConfiguration: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.resources }} + resources: +{{ toYaml .Values.alertmanager.alertmanagerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + routePrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.securityContext }} + securityContext: +{{ toYaml .Values.alertmanager.alertmanagerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.storage }} + storage: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.storage | indent 4) . }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.alertmanager.alertmanagerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.alertmanager.alertmanagerSpec.podAntiAffinity .Values.alertmanager.alertmanagerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.affinity }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "project-prometheus-stack.alertmanager.crname" . }}]} +{{- else if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "project-prometheus-stack.alertmanager.crname" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.tolerations }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "project-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.containers }} + containers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.initContainers }} + initContainers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.priorityClassName }} + priorityClassName: {{.Values.alertmanager.alertmanagerSpec.priorityClassName }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.additionalPeers }} + additionalPeers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.additionalPeers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumes }} + volumes: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.alertmanager.alertmanagerSpec.portName }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} + clusterAdvertiseAddress: {{ .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} + forceEnableClusterMode: {{ .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.minReadySeconds }} + minReadySeconds: {{ .Values.alertmanager.alertmanagerSpec.minReadySeconds }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/extrasecret.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/extrasecret.yaml new file mode 100644 index 0000000000..af469be753 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.alertmanager.extraSecret.data -}} +{{- $secretName := printf "alertmanager-%s-extra" (include "project-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.alertmanager.extraSecret.name }} + namespace: {{ template "project-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.extraSecret.annotations }} + annotations: +{{ toYaml .Values.alertmanager.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "project-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.alertmanager.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/ingress.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/ingress.yaml new file mode 100644 index 0000000000..022922b067 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.ingress.enabled }} +{{- $pathType := .Values.alertmanager.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := printf "%s-%s" (include "project-prometheus-stack.fullname" .) "alertmanager" }} +{{- $servicePort := .Values.alertmanager.ingress.servicePort | default .Values.alertmanager.service.port -}} +{{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }} +{{- $paths := .Values.alertmanager.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "project-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "project-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "project-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "project-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.ingress.annotations }} + annotations: +{{ toYaml .Values.alertmanager.ingress.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-alertmanager +{{- if .Values.alertmanager.ingress.labels }} +{{ toYaml .Values.alertmanager.ingress.labels | indent 4 }} +{{- end }} +{{ include "project-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.alertmanager.ingress.ingressClassName }} + ingressClassName: {{ .Values.alertmanager.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.alertmanager.ingress.hosts }} + {{- range $host := .Values.alertmanager.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.alertmanager.ingress.tls }} + tls: +{{ tpl (toYaml .Values.alertmanager.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/podDisruptionBudget.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/podDisruptionBudget.yaml new file mode 100644 index 0000000000..a8d7feaf56 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.podDisruptionBudget.enabled }} +apiVersion: {{ include "project-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-alertmanager +{{ include "project-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.alertmanager.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.alertmanager.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "project-prometheus-stack.alertmanager.crname" . }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/psp-role.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/psp-role.yaml new file mode 100644 index 0000000000..b959bf0074 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/psp-role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.global.rbac.create .Values.global.cattle.psp.enabled }} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-alertmanager +{{ include "project-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "project-prometheus-stack.fullname" . }}-alertmanager +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/psp-rolebinding.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/psp-rolebinding.yaml new file mode 100644 index 0000000000..d027f8e561 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/psp-rolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.alertmanager.enabled .Values.global.rbac.create .Values.global.cattle.psp.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-alertmanager +{{ include "project-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "project-prometheus-stack.fullname" . }}-alertmanager +subjects: + - kind: ServiceAccount + name: {{ template "project-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "project-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/psp.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/psp.yaml new file mode 100644 index 0000000000..138f1180e2 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/psp.yaml @@ -0,0 +1,45 @@ +{{- if and .Values.alertmanager.enabled .Values.global.rbac.create .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-alertmanager + labels: + app: {{ template "project-prometheus-stack.name" . }}-alertmanager +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "project-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/secret.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/secret.yaml new file mode 100644 index 0000000000..848d085f2c --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/secret.yaml @@ -0,0 +1,33 @@ +{{- if .Values.alertmanager.enabled }} +{{/* This file is applied when the operation is helm install and the target secret does not exist. */}} +{{- $secretName := (printf "%s-alertmanager-secret" (include "project-prometheus-stack.fullname" .)) }} +{{- if (not (lookup "v1" "Secret" (include "project-prometheus-stack.namespace" .) $secretName)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + namespace: {{ template "project-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-weight": "3" + "helm.sh/resource-policy": keep +{{- if .Values.alertmanager.secret.annotations }} +{{ toYaml .Values.alertmanager.secret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-alertmanager +{{ include "project-prometheus-stack.labels" . | indent 4 }} +data: +{{- if .Values.alertmanager.tplConfig }} +{{- if eq (typeOf .Values.alertmanager.config) "string" }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.config) . | b64enc | quote }} +{{- else }} + alertmanager.yaml: {{ tpl (toYaml .Values.alertmanager.config) . | b64enc | quote }} +{{- end }} +{{- else }} + alertmanager.yaml: {{ toYaml .Values.alertmanager.config | b64enc | quote }} +{{- end}} +{{- range $key, $val := .Values.alertmanager.templateFiles }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/service.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/service.yaml new file mode 100644 index 0000000000..3fb5cb3c6e --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/service.yaml @@ -0,0 +1,53 @@ +{{- if .Values.alertmanager.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-alertmanager + self-monitor: {{ .Values.alertmanager.serviceMonitor.selfMonitor | quote }} +{{ include "project-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.service.labels }} +{{ toYaml .Values.alertmanager.service.labels | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.annotations }} + annotations: +{{ toYaml .Values.alertmanager.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.service.clusterIP }} + clusterIP: {{ .Values.alertmanager.service.clusterIP }} +{{- end }} +{{- if .Values.alertmanager.service.externalIPs }} + externalIPs: +{{ toYaml .Values.alertmanager.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.alertmanager.service.loadBalancerIP }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.alertmanager.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.alertmanager.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.alertmanager.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq .Values.alertmanager.service.type "NodePort" }} + nodePort: {{ .Values.alertmanager.service.nodePort }} + {{- end }} + port: {{ .Values.alertmanager.service.port }} + targetPort: {{ .Values.alertmanager.service.targetPort }} + protocol: TCP +{{- if .Values.alertmanager.service.additionalPorts }} +{{ toYaml .Values.alertmanager.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "project-prometheus-stack.alertmanager.crname" . }} + type: "{{ .Values.alertmanager.service.type }}" +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/serviceaccount.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/serviceaccount.yaml new file mode 100644 index 0000000000..00b4b17751 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/serviceaccount.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "project-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/name: {{ template "project-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "project-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.alertmanager.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "project-prometheus-stack.imagePullSecrets" . | trim | indent 2}} +{{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/servicemonitor.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/servicemonitor.yaml new file mode 100644 index 0000000000..1cc9c6f790 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/alertmanager/servicemonitor.yaml @@ -0,0 +1,55 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-alertmanager +{{ include "project-prometheus-stack.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app: {{ template "project-prometheus-stack.name" . }}-alertmanager + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "project-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if .Values.alertmanager.serviceMonitor.interval }} + interval: {{ .Values.alertmanager.serviceMonitor.interval }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.alertmanager.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.scheme }} + scheme: {{ .Values.alertmanager.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.alertmanager.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.tlsConfig }} + tlsConfig: {{ toYaml .Values.alertmanager.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.alertmanager.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.alertmanager.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.alertmanager.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.alertmanager.serviceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/dashboard-roles.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/dashboard-roles.yaml new file mode 100644 index 0000000000..535bc94e1f --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/dashboard-roles.yaml @@ -0,0 +1,179 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-admin + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: {{ include "project-prometheus-stack.labels" . | nindent 4 }} + helm.cattle.io/project-helm-chart-role: {{ .Release.Name }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + helm.cattle.io/project-helm-chart-role-aggregate-from: admin + {{- end }} +rules: +- apiGroups: + - "" + resources: + - services/proxy + resourceNames: + - "http:{{ template "project-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "https:{{ template "project-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" +{{- if .Values.alertmanager.enabled }} + - "http:{{ template "project-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" + - "https:{{ template "project-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" +{{- end }} +{{- if .Values.grafana.enabled }} + - "http:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" + - "https:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" +{{- end }} + verbs: + - 'get' + - 'create' + - 'update' + - 'patch' + - 'delete' +- apiGroups: + - "" + resources: + - endpoints + resourceNames: + - {{ template "project-prometheus-stack.fullname" . }}-prometheus +{{- if .Values.alertmanager.enabled }} + - {{ template "project-prometheus-stack.fullname" . }}-alertmanager +{{- end }} +{{- if .Values.grafana.enabled }} + - {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +{{- end }} + verbs: + - list +{{- if .Values.alertmanager.enabled }} +- apiGroups: + - "" + resources: + - "secrets" + resourceNames: + - {{ printf "%s-alertmanager-secret" (include "project-prometheus-stack.fullname" .) }} + verbs: + - get + - list + - watch + - update + - patch + - delete +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-edit + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: {{ include "project-prometheus-stack.labels" . | nindent 4 }} + helm.cattle.io/project-helm-chart-role: {{ .Release.Name }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + helm.cattle.io/project-helm-chart-role-aggregate-from: edit + {{- end }} +rules: +- apiGroups: + - "" + resources: + - services/proxy + resourceNames: + - "http:{{ template "project-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "https:{{ template "project-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" +{{- if .Values.alertmanager.enabled }} + - "http:{{ template "project-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" + - "https:{{ template "project-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" +{{- end }} +{{- if .Values.grafana.enabled }} + - "http:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" + - "https:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" +{{- end }} + verbs: + - 'get' + - 'create' + - 'update' + - 'patch' + - 'delete' +- apiGroups: + - "" + resources: + - endpoints + resourceNames: + - {{ template "project-prometheus-stack.fullname" . }}-prometheus +{{- if .Values.alertmanager.enabled }} + - {{ template "project-prometheus-stack.fullname" . }}-alertmanager +{{- end }} +{{- if .Values.grafana.enabled }} + - {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +{{- end }} + verbs: + - list +{{- if .Values.alertmanager.enabled }} +- apiGroups: + - "" + resources: + - "secrets" + resourceNames: + - {{ printf "%s-alertmanager-secret" (include "project-prometheus-stack.fullname" .) }} + verbs: + - get + - list + - watch + - update + - patch +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-view + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: {{ include "project-prometheus-stack.labels" . | nindent 4 }} + helm.cattle.io/project-helm-chart-role: {{ .Release.Name }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + helm.cattle.io/project-helm-chart-role-aggregate-from: view + {{- end }} +rules: +- apiGroups: + - "" + resources: + - services/proxy + resourceNames: + - "http:{{ template "project-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "https:{{ template "project-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" +{{- if .Values.alertmanager.enabled }} + - "http:{{ template "project-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" + - "https:{{ template "project-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" +{{- end }} +{{- if .Values.grafana.enabled }} + - "http:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" + - "https:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" +{{- end }} + verbs: + - 'get' +- apiGroups: + - "" + resources: + - endpoints + resourceNames: + - {{ template "project-prometheus-stack.fullname" . }}-prometheus +{{- if .Values.alertmanager.enabled }} + - {{ template "project-prometheus-stack.fullname" . }}-alertmanager +{{- end }} +{{- if .Values.grafana.enabled }} + - {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +{{- end }} + verbs: + - list +{{- if .Values.alertmanager.enabled }} +- apiGroups: + - "" + resources: + - "secrets" + resourceNames: + - {{ printf "%s-alertmanager-secret" (include "project-prometheus-stack.fullname" .) }} + verbs: + - get + - list + - watch +{{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/dashboard-values-configmap.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/dashboard-values-configmap.yaml new file mode 100644 index 0000000000..a51f50d2e5 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/dashboard-values-configmap.yaml @@ -0,0 +1,61 @@ +{{- $rancherDashboards := dict }} +{{- range $glob := tuple "files/rancher/workloads/*" "files/rancher/pods/*" -}} +{{- range $dashboard, $_ := ($.Files.Glob $glob) }} +{{- $dashboardMap := ($.Files.Get $dashboard | fromJson) }} +{{- $_ := set $rancherDashboards (get $dashboardMap "uid") (get $dashboardMap "title") -}} +{{- end }} +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-dashboard-values + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: {{ include "project-prometheus-stack.labels" . | nindent 4 }} + helm.cattle.io/dashboard-values-configmap: {{ .Release.Name }} +data: + values.json: |- + { +{{- if not .Values.prometheus.enabled }} + "prometheusURL": "", +{{- else if .Values.prometheus.prometheusSpec.externalUrl }} + "prometheusURL": "{{ tpl .Values.prometheus.prometheusSpec.externalUrl . }}", +{{- else if and .Values.prometheus.ingress.enabled .Values.prometheus.ingress.hosts }} + "prometheusURL": "http://{{ tpl (index .Values.prometheus.ingress.hosts 0) . }}{{ .Values.prometheus.prometheusSpec.routePrefix }}", +{{- else if not (or (empty .Values.global.cattle.url) (empty .Values.global.cattle.clusterId)) }} + "prometheusURL": "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "project-prometheus-stack.namespace" . }}/services/http:{{ template "project-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}/proxy", +{{- else }} + "prometheusURL": "http://{{ template "project-prometheus-stack.fullname" . }}-prometheus.{{ template "project-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }}", +{{- end }} + +{{- if not .Values.grafana.enabled }} + "grafanaURL": "", +{{- else if and .Values.grafana.ingress.enabled .Values.grafana.ingress.hosts }} + "grafanaURL": "http://{{ tpl (index .Values.grafana.ingress.hosts 0) . }}{{ .Values.grafana.grafanaSpec.ingress.path }}", +{{- else if not (or (empty .Values.global.cattle.url) (empty .Values.global.cattle.clusterId)) }} + {{- if eq .Values.global.cattle.clusterId "local" -}} + "grafanaURL": "{{ .Values.global.cattle.url }}/api/v1/namespaces/{{ template "project-prometheus-stack.namespace" . }}/services/http:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}/proxy", + {{- else }} + "grafanaURL": "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "project-prometheus-stack.namespace" . }}/services/http:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}/proxy", + {{- end }} +{{- else }} + "grafanaURL": "http://{{ include "call-nested" (list . "grafana" "grafana.fullname") }}.{{ template "project-prometheus-stack.namespace" . }}:{{ .Values.grafana.service.port }}", +{{- end }} + +{{- if not .Values.alertmanager.enabled }} + "alertmanagerURL": "", +{{- else if .Values.alertmanager.alertmanagerSpec.externalUrl }} + "alertmanagerURL": "{{ tpl .Values.alertmanager.alertmanagerSpec.externalUrl . }}", +{{- else if and .Values.alertmanager.ingress.enabled .Values.alertmanager.ingress.hosts }} + "alertmanagerURL": "http://{{ tpl (index .Values.alertmanager.ingress.hosts 0) . }}{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}", +{{- else if not (or (empty .Values.global.cattle.url) (empty .Values.global.cattle.clusterId)) }} + "alertmanagerURL": "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "project-prometheus-stack.namespace" . }}/services/http:{{ template "project-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}/proxy", +{{- else }} + "alertmanagerURL": "http://{{ template "project-prometheus-stack.fullname" . }}-alertmanager.{{ template "project-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }}", +{{- end }} + +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} + "rancherDashboards": {{- $rancherDashboards | toPrettyJson | nindent 8 }} +{{- else }} + "rancherDashboards": [] +{{- end }} + } \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/dashboards/rancher/pods-dashboards.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/dashboards/rancher/pods-dashboards.yaml new file mode 100644 index 0000000000..6214dc5c7e --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/dashboards/rancher/pods-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: rancher-default-dashboards-pods + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/pods/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/dashboards/rancher/workload-dashboards.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/dashboards/rancher/workload-dashboards.yaml new file mode 100644 index 0000000000..7934818f18 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/dashboards/rancher/workload-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: rancher-default-dashboards-workloads + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/workloads/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/configmap-dashboards.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/configmap-dashboards.yaml new file mode 100644 index 0000000000..c276357383 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/configmap-dashboards.yaml @@ -0,0 +1,24 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled) .Values.grafana.forceDeployDashboards }} +{{- $files := .Files.Glob "dashboards-1.14/*.json" }} +{{- if $files }} +apiVersion: v1 +kind: ConfigMapList +items: +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +- apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 6 }} + data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/configmaps-datasources.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/configmaps-datasources.yaml new file mode 100644 index 0000000000..7f2c5b558e --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/configmaps-datasources.yaml @@ -0,0 +1,46 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled) .Values.grafana.forceDeployDatasources }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-grafana-datasource + namespace: {{ include "project-prometheus-stack.namespace" . }} +{{- if .Values.grafana.sidecar.datasources.annotations }} + annotations: +{{ toYaml .Values.grafana.sidecar.datasources.annotations | indent 4 }} +{{- end }} + labels: + {{ $.Values.grafana.sidecar.datasources.label }}: "1" + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + datasource.yaml: |- + apiVersion: 1 +{{- if .Values.grafana.deleteDatasources }} + deleteDatasources: +{{ tpl (toYaml .Values.grafana.deleteDatasources | indent 6) . }} +{{- end }} + datasources: +{{- $scrapeInterval := .Values.grafana.sidecar.datasources.defaultDatasourceScrapeInterval | default .Values.prometheus.prometheusSpec.scrapeInterval | default "30s" }} +{{- if .Values.grafana.sidecar.datasources.defaultDatasourceEnabled }} + - name: Prometheus + type: prometheus + uid: {{ .Values.grafana.sidecar.datasources.uid }} + {{- if .Values.grafana.sidecar.datasources.url }} + url: {{ .Values.grafana.sidecar.datasources.url }} + {{- else }} + url: http://{{ template "project-prometheus-stack.fullname" . }}-prometheus.{{ template "project-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }}/{{ trimPrefix "/" .Values.prometheus.prometheusSpec.routePrefix }} + {{- end }} + access: proxy + isDefault: true + jsonData: + timeInterval: {{ $scrapeInterval }} +{{- if .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- end }} +{{- if .Values.grafana.additionalDataSources }} +{{ tpl (toYaml .Values.grafana.additionalDataSources | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml new file mode 100644 index 0000000000..8f8e6ffeed --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml @@ -0,0 +1,616 @@ +{{- /* +Generated from 'alertmanager-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) "alertmanager-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + alertmanager-overview.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "current set of alerts stored in the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(alertmanager_alerts{namespace=~\"$namespace\",service=~\"$service\"}) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid alerts received by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_alerts_received_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Received", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_alerts_invalid_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Invalid", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts receive rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Alerts", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_notifications_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_notifications_failed_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Failed", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notifications Send Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "latency of notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} 99th Percentile", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Median", + "refId": "B" + }, + { + "expr": "sum(rate(alertmanager_notification_latency_seconds_sum{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n/\nsum(rate(alertmanager_notification_latency_seconds_count{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notification Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Notifications", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "alertmanager-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "service", + "multi": false, + "name": "service", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, service)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "integration", + "options": [ + + ], + "query": "label_values(alertmanager_notifications_total{integration=~\".*\"}, integration)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Alertmanager / Overview", + "uid": "alertmanager-overview", + "version": 0 + } +{{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/cluster-total.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/cluster-total.yaml new file mode 100644 index 0000000000..7076406931 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/cluster-total.yaml @@ -0,0 +1,1646 @@ +{{- /* +Generated from 'cluster-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) "cluster-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + cluster-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "namespace", + "value": "namespace" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/8b7a8b326d7a6f1f04244066368c67af/kubernetes-networking-namespace-pods?orgId=1&refresh=30s&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth History", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Project", + "uid": "ff635a025bcfea7bc3dd4f508990a3e9", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/grafana-overview.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/grafana-overview.yaml new file mode 100644 index 0000000000..ef663df776 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/grafana-overview.yaml @@ -0,0 +1,635 @@ +{{- /* +Generated from 'grafana-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) "grafana-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + grafana-overview.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + + ], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3085, + "iteration": 1631554945276, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_alerting_result_total{job=~\"$job\", instance=~\"$instance\", state=\"alerting\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Firing Alerts", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "sum(grafana_stat_totals_dashboard{job=~\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Dashboards", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "displayMode": "auto" + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 10, + "options": { + "showHeader": true + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_build_info{job=~\"$job\", instance=~\"$instance\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Build Info", + "transformations": [ + { + "id": "labelsToFields", + "options": { + + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "branch": true, + "container": true, + "goversion": true, + "namespace": true, + "pod": true, + "revision": true + }, + "indexByName": { + "Time": 7, + "Value": 11, + "branch": 4, + "container": 8, + "edition": 2, + "goversion": 6, + "instance": 1, + "job": 0, + "namespace": 9, + "pod": 10, + "revision": 5, + "version": 3 + }, + "renameByName": { + + } + } + } + ], + "type": "table" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status_code) (irate(grafana_http_request_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[1m])) ", + "interval": "", + "legendFormat": "{{`{{`}}status_code{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "RPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:157", + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:158", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "99th Percentile", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.50, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "50th Percentile", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(irate(grafana_http_request_duration_seconds_sum{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) * 1 / sum(irate(grafana_http_request_duration_seconds_count{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:210", + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:211", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 30, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": [ + "default/grafana" + ], + "value": [ + "default/grafana" + ] + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, job)", + "refId": "Billing Admin-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, instance)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "instance", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, instance)", + "refId": "Billing Admin-instance-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Grafana Overview", + "uid": "6be0s85Mk", + "version": 2 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml new file mode 100644 index 0000000000..c87b41888d --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml @@ -0,0 +1,2770 @@ +{{- /* +Generated from 'k8s-resources-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) "k8s-resources-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", namespace=\"$namespace\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\",namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\",namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\",namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\",namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\",namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\",namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\",namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\",namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Pods)", + "uid": "85a562078cdf77779eaa1add43ccec1e", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml new file mode 100644 index 0000000000..e2e8d2aa27 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml @@ -0,0 +1,963 @@ +{{- /* +Generated from 'k8s-resources-node' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) "k8s-resources-node" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-node.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{node=~\"$node\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{node=~\"$node\", container!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_rss{node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_cache{node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_swap{node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "node", + "options": [ + + ], + "query": "label_values(kube_pod_info, node)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Node (Pods)", + "uid": "200ac8fdbfbb74b39aff88118e4d1c2c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml new file mode 100644 index 0000000000..c8d57cfd4d --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml @@ -0,0 +1,2442 @@ +{{- /* +Generated from 'k8s-resources-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) "k8s-resources-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-pod.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval])) by (container) /sum(increase(container_cpu_cfs_periods_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}[$__rate_interval])) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.25, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Throttling", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Throttling", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (WSS)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage (WSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Pod - Read & Writes)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(container) (rate(container_fs_reads_total{container!=\"\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{container!=\"\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Containers)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_bytes_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+)\", container!=\"\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Pod", + "uid": "6581e46e4e5c7ba40a07646395ef7b23", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-project.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-project.yaml new file mode 100644 index 0000000000..56d9c77ff4 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-project.yaml @@ -0,0 +1,2480 @@ +{{- /* +Generated from 'k8s-resources-cluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) "k8s-resources-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-cluster.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "/d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_requests:sum{}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_limits:sum{}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{container!=\"\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "/d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{container!=\"\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests by Namespace", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Requests", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 11, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Received", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Transmitted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Namespace", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{container!=\"\"}[5m]) + rate(container_fs_writes_total{container!=\"\"}[5m])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{container!=\"\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{container!=\"\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_total{container!=\"\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{container!=\"\"}[5m]) + rate(container_fs_writes_total{container!=\"\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{container!=\"\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{container!=\"\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{container!=\"\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Project", + "uid": "efa86fd1d0c121a26444b636a3f509a8", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml new file mode 100644 index 0000000000..4e72d70ad8 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml @@ -0,0 +1,1997 @@ +{{- /* +Generated from 'k8s-resources-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) "k8s-resources-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workload.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Pod", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\"}, workload)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\"$workload\"}, workload_type)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Workload", + "uid": "a164a7f0339f99e89cea5cb47e9be617", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml new file mode 100644 index 0000000000..c6548c439e --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml @@ -0,0 +1,2162 @@ +{{- /* +Generated from 'k8s-resources-workloads-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) "k8s-resources-workloads-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workloads-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$type", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Workload", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Workloads)", + "uid": "a87fb0d919ec0ea5f6543124e16c42a5", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml new file mode 100644 index 0000000000..0d4fdc0f43 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml @@ -0,0 +1,1438 @@ +{{- /* +Generated from 'namespace-by-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) "namespace-by-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-pod.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "pod", + "value": "pod" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/7a18067ce943a40ae25454675c19ff5c/kubernetes-networking-pod?orgId=1&refresh=30s&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Pods)", + "uid": "8b7a8b326d7a6f1f04244066368c67af", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml new file mode 100644 index 0000000000..aa2a3573c1 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml @@ -0,0 +1,1710 @@ +{{- /* +Generated from 'namespace-by-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) "namespace-by-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-workload.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "workload", + "value": "workload" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/728bf77cc1166d2f3133bf25846876cc/kubernetes-networking-workload?orgId=1&refresh=30s&var-namespace=$namespace&var-type=$type&var-workload=$__cell", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total, namespace)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Workload)", + "uid": "bbb2a765a623ae38130206c7d94a160f", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml new file mode 100644 index 0000000000..cd599c62d4 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml @@ -0,0 +1,561 @@ +{{- /* +Generated from 'persistentvolumesusage' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) "persistentvolumesusage" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + persistentvolumesusage.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Space", + "refId": "A" + }, + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\n(\n topk(1, kubelet_volume_stats_capacity_bytes{metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n topk(1, kubelet_volume_stats_available_bytes{metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n/\ntopk(1, kubelet_volume_stats_capacity_bytes{metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume Space Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used inodes", + "refId": "A" + }, + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes{metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " Free inodes", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume inodes Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\ntopk(1, kubelet_volume_stats_inodes_used{metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n/\ntopk(1, kubelet_volume_stats_inodes{metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{metrics_path=\"/metrics\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "PersistentVolumeClaim", + "multi": false, + "name": "volume", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{metrics_path=\"/metrics\", namespace=\"$namespace\"}, persistentvolumeclaim)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Persistent Volumes", + "uid": "919b92a8e8041bd567af9edab12c840c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/pod-total.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/pod-total.yaml new file mode 100644 index 0000000000..48b0db9b43 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/pod-total.yaml @@ -0,0 +1,1202 @@ +{{- /* +Generated from 'pod-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) "pod-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + pod-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 8, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{namespace=~\"$namespace\"}, pod)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{namespace=~\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Pod", + "uid": "7a18067ce943a40ae25454675c19ff5c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml new file mode 100644 index 0000000000..2f0a6bfaea --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml @@ -0,0 +1,1674 @@ +{{- /* +Generated from 'prometheus-remote-write' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.prometheus.prometheusSpec.remoteWriteDashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) "prometheus-remote-write" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus-remote-write.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} != 0)\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Highest Timestamp In vs. Highest Timestamp Sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "clamp_min(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n, 0)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate[5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Timestamps", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n- \n (rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate, in vs. succeeded or dropped [5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Samples", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Max Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Min Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Desired Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shards", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Shard Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\"} or prometheus_remote_storage_samples_pending{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pending Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shard Details", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_wal_segment_current{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "TSDB Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}consumer{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Remote Write Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Segments", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Dropped Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Failed Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Retried Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Enqueue Retries", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Misc. Rates", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": true, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_container_info{image=~\".*prometheus.*\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{cluster=~\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "url", + "options": [ + + ], + "query": "label_values(prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}, url)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Remote Write", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/prometheus.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/prometheus.yaml new file mode 100644 index 0000000000..c94ce25235 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/prometheus.yaml @@ -0,0 +1,1235 @@ +{{- /* +Generated from 'prometheus' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) "prometheus" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Count", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Uptime", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Instance", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "instance", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Job", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "job", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Version", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "version", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count by (job, instance, version) (prometheus_build_info{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "max by (job, instance) (time() - process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Prometheus Stats", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Prometheus Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m])) by (scrape_job) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}scrape_job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Target Sync", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_sd_discovered_targets{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Targets", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Targets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Discovery", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}interval{{`}}`}} configured", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Scrape Interval Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded body size limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded sample limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "duplicate timestamp: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of bounds: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of order: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scrape failures", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Appended Samples", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Retrieval", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head series", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Series", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head chunks", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Query Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",job=~\"$job\",instance=~\"$instance\"}) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}slice{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Stage Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=\"prometheus-k8s\",namespace=\"monitoring\"}, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=~\"$job\"}, instance)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Overview", + "uid": "", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/workload-total.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/workload-total.yaml new file mode 100644 index 0000000000..0dfd208c2a --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/grafana/dashboards-1.14/workload-total.yaml @@ -0,0 +1,1412 @@ +{{- /* +Generated from 'workload-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "project-prometheus-stack.namespace" . }} + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) "workload-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "project-prometheus-stack.name" $ }}-grafana +{{ include "project-prometheus-stack.labels" $ | indent 4 }} +data: + workload-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 8, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 14, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\"}, workload)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\"}, workload)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Workload", + "uid": "728bf77cc1166d2f3133bf25846876cc", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/_rules.tpl b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/_rules.tpl new file mode 100644 index 0000000000..d82cc2d354 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/_rules.tpl @@ -0,0 +1,8 @@ +{{- define "rules.names" }} +rules: + - "alertmanager.rules" + - "general.rules" + - "kubernetes-storage" + - "prometheus" + - "kubernetes-apps" +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/clusterrole.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/clusterrole.yaml new file mode 100644 index 0000000000..b5d97aecd1 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/clusterrole.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "project-prometheus-stack.name" . }}-prometheus +{{ include "project-prometheus-stack.labels" . | indent 4 }} +rules: +# This permission are not in the kube-prometheus repo +# they're grabbed from https://github.com/prometheus/prometheus/blob/master/documentation/examples/rbac-setup.yml +- apiGroups: [""] + resources: + - nodes + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - "networking.k8s.io" + resources: + - ingresses + verbs: ["get", "list", "watch"] +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/clusterrolebinding.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/clusterrolebinding.yaml new file mode 100644 index 0000000000..18249a0c76 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "project-prometheus-stack.name" . }}-prometheus +{{ include "project-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "project-prometheus-stack.fullname" . }}-prometheus +subjects: + - kind: ServiceAccount + name: {{ template "project-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "project-prometheus-stack.namespace" . }} +{{- end }} + diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/extrasecret.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/extrasecret.yaml new file mode 100644 index 0000000000..61664d0197 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.prometheus.extraSecret.data -}} +{{- $secretName := printf "prometheus-%s-extra" (include "project-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.prometheus.extraSecret.name }} + namespace: {{ template "project-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.extraSecret.annotations }} + annotations: +{{ toYaml .Values.prometheus.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "project-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.prometheus.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/federate.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/federate.yaml new file mode 100644 index 0000000000..29d4ba2078 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/federate.yaml @@ -0,0 +1,10 @@ +{{- if and .Values.prometheus.enabled .Values.federate.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-federate + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: {{ include "project-prometheus-stack.labels" . | nindent 4 }} +data: + federate-scrape-config.yaml: {{ tpl (.Files.Get "files/federate/federate-scrape-config.yaml" ) $ | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/ingress.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/ingress.yaml new file mode 100644 index 0000000000..387e2b5dea --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled -}} + {{- $pathType := .Values.prometheus.ingress.pathType | default "ImplementationSpecific" -}} + {{- $serviceName := printf "%s-%s" (include "project-prometheus-stack.fullname" .) "prometheus" -}} + {{- $servicePort := .Values.prometheus.ingress.servicePort | default .Values.prometheus.service.port -}} + {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix -}} + {{- $paths := .Values.prometheus.ingress.paths | default $routePrefix -}} + {{- $apiIsStable := eq (include "project-prometheus-stack.ingress.isStable" .) "true" -}} + {{- $ingressSupportsPathType := eq (include "project-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "project-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.ingress.annotations }} + annotations: +{{ toYaml .Values.prometheus.ingress.annotations | indent 4 }} +{{- end }} + name: {{ $serviceName }} + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-prometheus +{{ include "project-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.ingress.labels }} +{{ toYaml .Values.prometheus.ingress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.ingress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.ingress.hosts }} + {{- range $host := .Values.prometheus.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.ingress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/nginx-config.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/nginx-config.yaml new file mode 100644 index 0000000000..c28edc0707 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/nginx-config.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-nginx-proxy-config + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-prometheus +{{ include "project-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + server { + listen 8081; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:9090/; + + sub_filter_once off; + sub_filter 'var PATH_PREFIX = "";' 'var PATH_PREFIX = ".";'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/podDisruptionBudget.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/podDisruptionBudget.yaml new file mode 100644 index 0000000000..24e4c6a776 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.podDisruptionBudget.enabled }} +apiVersion: {{ include "project-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-prometheus +{{ include "project-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.prometheus.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.prometheus.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.prometheus.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.prometheus.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: prometheus + prometheus: {{ template "project-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/prometheus.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/prometheus.yaml new file mode 100644 index 0000000000..2e4cfe526c --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/prometheus.yaml @@ -0,0 +1,319 @@ +{{- if .Values.prometheus.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: {{ template "project-prometheus-stack.prometheus.crname" . }} + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-prometheus +{{ include "project-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +spec: + alerting: + alertmanagers: +{{- if .Values.prometheus.prometheusSpec.alertingEndpoints }} +{{ toYaml .Values.prometheus.prometheusSpec.alertingEndpoints | indent 6 }} +{{- else if .Values.alertmanager.enabled }} + - namespace: {{ template "project-prometheus-stack.namespace" . }} + name: {{ template "project-prometheus-stack.fullname" . }}-alertmanager + port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + pathPrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" + {{- end }} + apiVersion: {{ .Values.alertmanager.apiVersion }} +{{- else }} + [] +{{- end }} +{{- if .Values.prometheus.prometheusSpec.apiserverConfig }} + apiserverConfig: +{{ toYaml .Values.prometheus.prometheusSpec.apiserverConfig | indent 4}} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.image }} + {{- if and .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.image.repository }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.tag }} + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.image.repository }}" + {{- end }} + version: {{ .Values.prometheus.prometheusSpec.image.tag }} + {{- if .Values.prometheus.prometheusSpec.image.sha }} + sha: {{ .Values.prometheus.prometheusSpec.image.sha }} + {{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.externalLabels }} + externalLabels: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.externalLabels | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusExternalLabelNameClear }} + prometheusExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.prometheusExternalLabelName }} + prometheusExternalLabelName: "{{ .Values.prometheus.prometheusSpec.prometheusExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} + enableRemoteWriteReceiver: {{ .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.externalUrl }} + externalUrl: "{{ tpl .Values.prometheus.prometheusSpec.externalUrl . }}" +{{- else if and .Values.prometheus.ingress.enabled .Values.prometheus.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.prometheus.ingress.hosts 0) . }}{{ .Values.prometheus.prometheusSpec.routePrefix }}" +{{- else if not (or (empty .Values.global.cattle.url) (empty .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "project-prometheus-stack.namespace" . }}/services/http:{{ template "project-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "project-prometheus-stack.fullname" . }}-prometheus.{{ template "project-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.nodeSelector }} +{{ toYaml .Values.prometheus.prometheusSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.prometheus.prometheusSpec.paused }} + replicas: {{ .Values.prometheus.prometheusSpec.replicas }} + shards: {{ .Values.prometheus.prometheusSpec.shards }} + logLevel: {{ .Values.prometheus.prometheusSpec.logLevel }} + logFormat: {{ .Values.prometheus.prometheusSpec.logFormat }} + listenLocal: {{ .Values.prometheus.prometheusSpec.listenLocal }} + enableAdminAPI: {{ .Values.prometheus.prometheusSpec.enableAdminAPI }} +{{- if .Values.prometheus.prometheusSpec.web }} + web: +{{ toYaml .Values.prometheus.prometheusSpec.web | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.exemplars }} + exemplars: + {{ toYaml .Values.prometheus.prometheusSpec.exemplars | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableFeatures }} + enableFeatures: +{{- range $enableFeatures := .Values.prometheus.prometheusSpec.enableFeatures }} + - {{ tpl $enableFeatures $ }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeInterval }} + scrapeInterval: {{ .Values.prometheus.prometheusSpec.scrapeInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.prometheusSpec.scrapeTimeout }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.evaluationInterval }} + evaluationInterval: {{ .Values.prometheus.prometheusSpec.evaluationInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.resources }} + resources: +{{ toYaml .Values.prometheus.prometheusSpec.resources | indent 4 }} +{{- end }} + retention: {{ .Values.prometheus.prometheusSpec.retention | quote }} +{{- if .Values.prometheus.prometheusSpec.retentionSize }} + retentionSize: {{ .Values.prometheus.prometheusSpec.retentionSize | quote }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.walCompression false }} + walCompression: false +{{ else }} + walCompression: true +{{- end }} +{{- if .Values.prometheus.prometheusSpec.routePrefix }} + routePrefix: {{ .Values.prometheus.prometheusSpec.routePrefix | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.secrets }} + secrets: +{{ toYaml .Values.prometheus.prometheusSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.configMaps }} + configMaps: +{{ toYaml .Values.prometheus.prometheusSpec.configMaps | indent 4 }} +{{- end }} + serviceAccountName: {{ template "project-prometheus-stack.prometheus.serviceAccountName" . }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorSelector }} + serviceMonitorSelector: +{{ toYaml .Values.prometheus.prometheusSpec.serviceMonitorSelector | indent 4 }} +{{ else if .Values.prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues }} + serviceMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + serviceMonitorSelector: {} +{{- end }} + serviceMonitorNamespaceSelector: {{ .Values.global.cattle.projectNamespaceSelector | toYaml | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.podMonitorSelector }} + podMonitorSelector: +{{ toYaml .Values.prometheus.prometheusSpec.podMonitorSelector | indent 4 }} +{{ else if .Values.prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues }} + podMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + podMonitorSelector: {} +{{- end }} + podMonitorNamespaceSelector: {{ .Values.global.cattle.projectNamespaceSelector | toYaml | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.probeSelector }} + probeSelector: +{{ toYaml .Values.prometheus.prometheusSpec.probeSelector | indent 4 }} +{{ else if .Values.prometheus.prometheusSpec.probeSelectorNilUsesHelmValues }} + probeSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + probeSelector: {} +{{- end }} +{{- if (or .Values.prometheus.prometheusSpec.remoteWrite .Values.prometheus.prometheusSpec.additionalRemoteWrite) }} + remoteWrite: +{{- if .Values.prometheus.prometheusSpec.remoteWrite }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteWrite | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteWrite }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteWrite | indent 4 }} +{{- end }} +{{- end }} + probeNamespaceSelector: {{ .Values.global.cattle.projectNamespaceSelector | toYaml | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.securityContext }} + securityContext: +{{ toYaml .Values.prometheus.prometheusSpec.securityContext | indent 4 }} +{{- end }} + ruleNamespaceSelector: {{ .Values.global.cattle.projectNamespaceSelector | toYaml | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.ruleSelector }} + ruleSelector: +{{ toYaml .Values.prometheus.prometheusSpec.ruleSelector | indent 4}} +{{- else if .Values.prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.storage.enabled }} + storage: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.storageSpec | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMetadata }} + podMetadata: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMetadata | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.query }} + query: +{{ toYaml .Values.prometheus.prometheusSpec.query | indent 4}} +{{- end }} +{{- if or .Values.prometheus.prometheusSpec.podAntiAffinity .Values.prometheus.prometheusSpec.affinity }} + affinity: +{{- if .Values.prometheus.prometheusSpec.affinity }} +{{ toYaml .Values.prometheus.prometheusSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "project-prometheus-stack.fullname" . }}-prometheus]} +{{- else if eq .Values.prometheus.prometheusSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "project-prometheus-stack.fullname" . }}-prometheus]} +{{- end }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.tolerations }} +{{ toYaml .Values.prometheus.prometheusSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.prometheus.prometheusSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "project-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.federate.enabled }} + additionalScrapeConfigs: + name: {{ template "project-prometheus-stack.fullname" . }}-federate + key: federate-scrape-config.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.containers }} + containers: +{{ tpl .Values.prometheus.prometheusSpec.containers $ | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.initContainers }} + initContainers: +{{ toYaml .Values.prometheus.prometheusSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.priorityClassName }} + priorityClassName: {{ .Values.prometheus.prometheusSpec.priorityClassName }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.disableCompaction }} + disableCompaction: {{ .Values.prometheus.prometheusSpec.disableCompaction }} +{{- end }} + portName: {{ .Values.prometheus.prometheusSpec.portName }} +{{- if .Values.prometheus.prometheusSpec.volumes }} + volumes: +{{ toYaml .Values.prometheus.prometheusSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.prometheus.prometheusSpec.volumeMounts | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs }} + arbitraryFSAccessThroughSMs: +{{ toYaml .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorLabels }} + overrideHonorLabels: {{ .Values.prometheus.prometheusSpec.overrideHonorLabels }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} + overrideHonorTimestamps: {{ .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} +{{- end }} + ignoreNamespaceSelectors: true # always hard-coded to true for security reasons +{{- if .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} + enforcedNamespaceLabel: {{ .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} +{{- $prometheusDefaultRulesExcludedFromEnforce := (include "rules.names" .) | fromYaml }} + prometheusRulesExcludedFromEnforce: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - ruleNamespace: "{{ template "project-prometheus-stack.namespace" $ }}" + ruleName: "{{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce }} +{{ toYaml .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce | indent 4 }} +{{- end }} + excludedFromEnforcement: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - resource: prometheusrules + namespace: "{{ template "project-prometheus-stack.namespace" $ }}" + name: "{{ printf "%s-%s" (include "project-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.excludedFromEnforcement }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.excludedFromEnforcement | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.queryLogFile }} + queryLogFile: {{ .Values.prometheus.prometheusSpec.queryLogFile }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedSampleLimit }} + enforcedSampleLimit: {{ .Values.prometheus.prometheusSpec.enforcedSampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedTargetLimit }} + enforcedTargetLimit: {{ .Values.prometheus.prometheusSpec.enforcedTargetLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelLimit }} + enforcedLabelLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} + enforcedLabelNameLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit}} + enforcedLabelValueLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} + allowOverlappingBlocks: {{ .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.minReadySeconds }} + minReadySeconds: {{ .Values.prometheus.prometheusSpec.minReadySeconds }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/psp-clusterrole.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/psp-clusterrole.yaml new file mode 100644 index 0000000000..8a3cbea5c4 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/psp-clusterrole.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create .Values.global.cattle.psp.enabled }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "project-prometheus-stack.name" . }}-prometheus +{{ include "project-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "project-prometheus-stack.fullname" . }}-prometheus +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/psp-clusterrolebinding.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..736525c936 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/psp-clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create .Values.global.cattle.psp.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "project-prometheus-stack.name" . }}-prometheus +{{ include "project-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "project-prometheus-stack.fullname" . }}-prometheus-psp +subjects: + - kind: ServiceAccount + name: {{ template "project-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "project-prometheus-stack.namespace" . }} +{{- end }} + diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/psp.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/psp.yaml new file mode 100644 index 0000000000..b0130af75b --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/psp.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "project-prometheus-stack.name" . }}-prometheus +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "project-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' +{{- if .Values.prometheus.podSecurityPolicy.volumes }} +{{ toYaml .Values.prometheus.podSecurityPolicy.volumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- if .Values.prometheus.podSecurityPolicy.allowedCapabilities }} + allowedCapabilities: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedCapabilities | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml new file mode 100644 index 0000000000..0cca1fb3e5 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml @@ -0,0 +1,217 @@ +{{- /* +Generated from 'alertmanager.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/alertmanager-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.alertmanager }} +{{- $alertmanagerJob := printf "%s-%s" (include "project-prometheus-stack.fullname" .) "alertmanager" }} +{{- $namespace := printf "%s" (include "project-prometheus-stack.namespace" .) }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" .) "alertmanager.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }} +{{ include "project-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: alertmanager.rules + rules: +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedReload | default false) }} + - alert: AlertmanagerFailedReload + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Configuration has failed to load for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedreload + summary: Reloading an Alertmanager configuration has failed. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_config_last_reload_successful{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: 10m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerMembersInconsistent | default false) }} + - alert: AlertmanagerMembersInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} has only found {{`{{`}} $value {{`}}`}} members of the {{`{{`}}$labels.job{{`}}`}} cluster. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagermembersinconsistent + summary: A member of an Alertmanager cluster has not found all other cluster members. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + < on (namespace,service) group_left + count by (namespace,service) (max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m])) + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedToSendAlerts | default false) }} + - alert: AlertmanagerFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} failed to send {{`{{`}} $value | humanizePercentage {{`}}`}} of notifications to {{`{{`}} $labels.integration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedtosendalerts + summary: An Alertmanager instance failed to send notifications. + expr: |- + ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + / + rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 0.01 + for: 5m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a critical integration. + expr: |- + min by (namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + / + rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + ) + > 0.01 + for: 5m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a non-critical integration. + expr: |- + min by (namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + / + rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + ) + > 0.01 + for: 5m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerConfigInconsistent | default false) }} + - alert: AlertmanagerConfigInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have different configurations. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerconfiginconsistent + summary: Alertmanager instances within the same cluster have different configurations. + expr: |- + count by (namespace,service) ( + count_values by (namespace,service) ("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) + ) + != 1 + for: 20m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterDown | default false) }} + - alert: AlertmanagerClusterDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have been up for less than half of the last 5m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterdown + summary: Half or more of the Alertmanager instances within the same cluster are down. + expr: |- + ( + count by (namespace,service) ( + avg_over_time(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) < 0.5 + ) + / + count by (namespace,service) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: 5m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterCrashlooping | default false) }} + - alert: AlertmanagerClusterCrashlooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have restarted at least 5 times in the last 10m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclustercrashlooping + summary: Half or more of the Alertmanager instances within the same cluster are crashlooping. + expr: |- + ( + count by (namespace,service) ( + changes(process_start_time_seconds{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[10m]) > 4 + ) + / + count by (namespace,service) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: 5m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/general.rules.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/general.rules.yaml new file mode 100644 index 0000000000..13c158c08e --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/general.rules.yaml @@ -0,0 +1,98 @@ +{{- /* +Generated from 'general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.general }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" .) "general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }} +{{ include "project-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: general.rules + rules: +{{- if not (.Values.defaultRules.disabled.TargetDown | default false) }} + - alert: TargetDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.4g" $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.service {{`}}`}} targets in {{`{{`}} $labels.namespace {{`}}`}} namespace are down.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/targetdown + summary: One or more targets are unreachable. + expr: 100 * (count(up == 0) BY (job, namespace, service) / count(up) BY (job, namespace, service)) > 10 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.Watchdog | default false) }} + - alert: Watchdog + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'This is an alert meant to ensure that the entire alerting pipeline is functional. + + This alert is always firing, therefore it should always be firing in Alertmanager + + and always fire against a receiver. There are integrations with various notification + + mechanisms that send a notification when this alert is not firing. For example the + + "DeadMansSnitch" integration in PagerDuty. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/watchdog + summary: An alert that should always be firing to certify that Alertmanager is working properly. + expr: vector(1) + labels: + severity: none +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.InfoInhibitor | default false) }} + - alert: InfoInhibitor + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'This is an alert that is used to inhibit info alerts. + + By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with + + other alerts. + + This alert fires whenever there''s a severity="info" alert, and stops firing when another alert with a + + severity of ''warning'' or ''critical'' starts firing on the same namespace. + + This alert should be routed to a null receiver and configured to inhibit alerts with severity="info". + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/infoinhibitor + summary: Info-level alert inhibition. + expr: ALERTS{severity = "info"} == 1 unless on(namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 + labels: + severity: none +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml new file mode 100644 index 0000000000..c956157eb2 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -0,0 +1,375 @@ +{{- /* +Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesApps }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" .) "kubernetes-apps" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }} +{{ include "project-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-apps + rules: +{{- if not (.Values.defaultRules.disabled.KubePodCrashLooping | default false) }} + - alert: KubePodCrashLooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: 'Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is in waiting state (reason: "CrashLoopBackOff").' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodcrashlooping + summary: Pod is crash looping. + expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[5m]) >= 1 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePodNotReady | default false) }} + - alert: KubePodNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodnotready + summary: Pod has been in a non-ready state for more than 15 minutes. + expr: |- + sum by (namespace, pod, cluster) ( + max by(namespace, pod, cluster) ( + kube_pod_status_phase{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}", phase=~"Pending|Unknown"} + ) * on(namespace, pod, cluster) group_left(owner_kind) topk by(namespace, pod, cluster) ( + 1, max by(namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}) + ) + ) > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentGenerationMismatch | default false) }} + - alert: KubeDeploymentGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Deployment generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} does not match, this indicates that the Deployment has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentgenerationmismatch + summary: Deployment generation mismatch due to possible roll-back + expr: |- + kube_deployment_status_observed_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_deployment_metadata_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentReplicasMismatch | default false) }} + - alert: KubeDeploymentReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentreplicasmismatch + summary: Deployment has not matched the expected number of replicas. + expr: |- + ( + kube_deployment_spec_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + > + kube_deployment_status_replicas_available{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_deployment_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetReplicasMismatch | default false) }} + - alert: KubeStatefulSetReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetreplicasmismatch + summary: Deployment has not matched the expected number of replicas. + expr: |- + ( + kube_statefulset_status_replicas_ready{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetGenerationMismatch | default false) }} + - alert: KubeStatefulSetGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: StatefulSet generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetgenerationmismatch + summary: StatefulSet generation mismatch due to possible roll-back + expr: |- + kube_statefulset_status_observed_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_metadata_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetUpdateNotRolledOut | default false) }} + - alert: KubeStatefulSetUpdateNotRolledOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} update has not been rolled out. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetupdatenotrolledout + summary: StatefulSet update has not been rolled out. + expr: |- + ( + max without (revision) ( + kube_statefulset_status_current_revision{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + unless + kube_statefulset_status_update_revision{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) + * + ( + kube_statefulset_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetRolloutStuck | default false) }} + - alert: KubeDaemonSetRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} has not finished or progressed for at least 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetrolloutstuck + summary: DaemonSet rollout is stuck. + expr: |- + ( + ( + kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + 0 + ) or ( + kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_available{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeContainerWaiting | default false) }} + - alert: KubeContainerWaiting + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: pod/{{`{{`}} $labels.pod {{`}}`}} in namespace {{`{{`}} $labels.namespace {{`}}`}} on container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontainerwaiting + summary: Pod container waiting longer than 1 hour + expr: sum by (namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) > 0 + for: 1h + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetNotScheduled | default false) }} + - alert: KubeDaemonSetNotScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are not scheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetnotscheduled + summary: DaemonSet pods are not scheduled. + expr: |- + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + - + kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetMisScheduled | default false) }} + - alert: KubeDaemonSetMisScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are running where they are not supposed to run.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetmisscheduled + summary: DaemonSet pods are misscheduled. + expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobNotCompleted | default false) }} + - alert: KubeJobNotCompleted + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than {{`{{`}} "43200" | humanizeDuration {{`}}`}} to complete. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobnotcompleted + summary: Job did not complete in time + expr: |- + time() - max by(namespace, job_name, cluster) (kube_job_status_start_time{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + and + kube_job_status_active{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0) > 43200 + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobFailed | default false) }} + - alert: KubeJobFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} failed to complete. Removing failed job after investigation should clear this alert. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobfailed + summary: Job failed to complete. + expr: kube_job_failed{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaReplicasMismatch | default false) }} + - alert: KubeHpaReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has not matched the desired number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpareplicasmismatch + summary: HPA has not matched desired number of replicas. + expr: |- + (kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + > + kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + < + kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) + and + changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[15m]) == 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaMaxedOut | default false) }} + - alert: KubeHpaMaxedOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has been running at max replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpamaxedout + summary: HPA is running at max replicas + expr: |- + kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + == + kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml new file mode 100644 index 0000000000..b6fe43fb7f --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml @@ -0,0 +1,160 @@ +{{- /* +Generated from 'kubernetes-storage' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesStorage }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" .) "kubernetes-storage" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }} +{{ include "project-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-storage + rules: +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is only {{`{{`}} $value | humanizePercentage {{`}}`}} free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + kubelet_volume_stats_available_bytes{namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + < 0.03 + and + kubelet_volume_stats_used_bytes{namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: 1m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} is available. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + ( + kubelet_volume_stats_available_bytes{namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_used_bytes{namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_available_bytes{namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: 1h + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} only has {{`{{`}} $value | humanizePercentage {{`}}`}} free inodes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.03 + and + kubelet_volume_stats_inodes_used{namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: 1m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is expected to run out of inodes within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} of its inodes are free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_inodes_used{namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_inodes_free{namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: 1h + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeErrors | default false) }} + - alert: KubePersistentVolumeErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: The persistent volume {{`{{`}} $labels.persistentvolume {{`}}`}} has status {{`{{`}} $labels.phase {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeerrors + summary: PersistentVolume is having issues with provisioning. + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0 + for: 5m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/prometheus.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/prometheus.yaml new file mode 100644 index 0000000000..d3aa167ddd --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/rules-1.14/prometheus.yaml @@ -0,0 +1,448 @@ +{{- /* +Generated from 'prometheus' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/prometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheus }} +{{- $prometheusJob := printf "%s-%s" (include "project-prometheus-stack.fullname" .) "prometheus" }} +{{- $namespace := printf "%s" (include "project-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "project-prometheus-stack.fullname" .) "prometheus" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }} +{{ include "project-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusBadConfig | default false) }} + - alert: PrometheusBadConfig + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to reload its configuration. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusbadconfig + summary: Failed Prometheus configuration reload. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_config_last_reload_successful{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: 10m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotificationQueueRunningFull | default false) }} + - alert: PrometheusNotificationQueueRunningFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Alert notification queue of Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is running full. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotificationqueuerunningfull + summary: Prometheus alert notification queue predicted to run full in less than 30m. + expr: |- + # Without min_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + predict_linear(prometheus_notifications_queue_length{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m], 60 * 30) + > + min_over_time(prometheus_notifications_queue_capacity{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToSomeAlertmanagers | default false) }} + - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to Alertmanager {{`{{`}}$labels.alertmanager{{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstosomealertmanagers + summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager. + expr: |- + ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + * 100 + > 1 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotConnectedToAlertmanagers | default false) }} + - alert: PrometheusNotConnectedToAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not connected to any Alertmanagers. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotconnectedtoalertmanagers + summary: Prometheus is not connected to any Alertmanagers. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_notifications_alertmanagers_discovered{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) < 1 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBReloadsFailing | default false) }} + - alert: PrometheusTSDBReloadsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} reload failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbreloadsfailing + summary: Prometheus has issues reloading blocks from disk. + expr: increase(prometheus_tsdb_reloads_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: 4h + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBCompactionsFailing | default false) }} + - alert: PrometheusTSDBCompactionsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} compaction failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbcompactionsfailing + summary: Prometheus has issues compacting blocks. + expr: increase(prometheus_tsdb_compactions_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: 4h + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotIngestingSamples | default false) }} + - alert: PrometheusNotIngestingSamples + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not ingesting samples. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotingestingsamples + summary: Prometheus is not ingesting samples. + expr: |- + ( + rate(prometheus_tsdb_head_samples_appended_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) <= 0 + and + ( + sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + or + sum without(rule_group) (prometheus_rule_group_rules{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + ) + ) + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusDuplicateTimestamps | default false) }} + - alert: PrometheusDuplicateTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with different values but duplicated timestamp. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusduplicatetimestamps + summary: Prometheus is dropping samples with duplicate timestamps. + expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOutOfOrderTimestamps | default false) }} + - alert: PrometheusOutOfOrderTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with timestamps arriving out of order. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusoutofordertimestamps + summary: Prometheus drops samples with out-of-order timestamps. + expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 10m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteStorageFailures | default false) }} + - alert: PrometheusRemoteStorageFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} failed to send {{`{{`}} printf "%.1f" $value {{`}}`}}% of the samples to {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotestoragefailures + summary: Prometheus fails to send samples to remote storage. + expr: |- + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + / + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + + + (rate(prometheus_remote_storage_succeeded_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + ) + ) + * 100 + > 1 + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteBehind | default false) }} + - alert: PrometheusRemoteWriteBehind + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write is {{`{{`}} printf "%.1f" $value {{`}}`}}s behind for {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritebehind + summary: Prometheus remote write is behind. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + - ignoring(remote_name, url) group_right + max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 120 + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteDesiredShards | default false) }} + - alert: PrometheusRemoteWriteDesiredShards + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write desired shards calculation wants to run {{`{{`}} $value {{`}}`}} shards for queue {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}, which is more than the max of {{`{{`}} printf `prometheus_remote_storage_shards_max{instance="%s",job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}` $labels.instance | query | first | value {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritedesiredshards + summary: Prometheus remote write desired shards calculation wants to run more than configured max shards. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_shards_desired{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + > + max_over_time(prometheus_remote_storage_shards_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRuleFailures | default false) }} + - alert: PrometheusRuleFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to evaluate {{`{{`}} printf "%.0f" $value {{`}}`}} rules in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusrulefailures + summary: Prometheus is failing rule evaluations. + expr: increase(prometheus_rule_evaluation_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusMissingRuleEvaluations | default false) }} + - alert: PrometheusMissingRuleEvaluations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has missed {{`{{`}} printf "%.0f" $value {{`}}`}} rule group evaluations in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusmissingruleevaluations + summary: Prometheus is missing rule evaluations due to slow rule group evaluation. + expr: increase(prometheus_rule_group_iterations_missed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetLimitHit | default false) }} + - alert: PrometheusTargetLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because the number of targets exceeded the configured target_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetlimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit. + expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusLabelLimitHit | default false) }} + - alert: PrometheusLabelLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuslabellimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the labels limit. + expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeBodySizeLimitHit | default false) }} + - alert: PrometheusScrapeBodySizeLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured body_size_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapebodysizelimithit + summary: Prometheus has dropped some targets that exceeded body size limit. + expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeSampleLimitHit | default false) }} + - alert: PrometheusScrapeSampleLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured sample_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapesamplelimithit + summary: Prometheus has failed scrapes that have exceeded the configured sample limit. + expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetSyncFailure | default false) }} + - alert: PrometheusTargetSyncFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.0f" $value {{`}}`}} targets in Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} have failed to sync because invalid configuration was supplied.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetsyncfailure + summary: Prometheus has failed to sync targets. + expr: increase(prometheus_target_sync_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[30m]) > 0 + for: 5m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusHighQueryLoad | default false) }} + - alert: PrometheusHighQueryLoad + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} query API has less than 20% available capacity in its query engine for the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheushighqueryload + summary: Prometheus is reaching its maximum capacity serving concurrent requests. + expr: avg_over_time(prometheus_engine_queries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0.8 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToAnyAlertmanager | default false) }} + - alert: PrometheusErrorSendingAlertsToAnyAlertmanager + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% minimum errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to any Alertmanager.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstoanyalertmanager + summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager. + expr: |- + min without (alertmanager) ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + ) + * 100 + > 3 + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/service.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/service.yaml new file mode 100644 index 0000000000..411ec5fae0 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/service.yaml @@ -0,0 +1,56 @@ +{{- if .Values.prometheus.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-prometheus + self-monitor: {{ .Values.prometheus.serviceMonitor.selfMonitor | quote }} +{{ include "project-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.service.labels }} +{{ toYaml .Values.prometheus.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.annotations }} + annotations: +{{ toYaml .Values.prometheus.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheus.service.clusterIP }} + clusterIP: {{ .Values.prometheus.service.clusterIP }} +{{- end }} +{{- if .Values.prometheus.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheus.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.service.nodePort }} + {{- end }} + port: {{ .Values.prometheus.service.port }} + targetPort: {{ .Values.prometheus.service.targetPort }} +{{- if .Values.prometheus.service.additionalPorts }} +{{ toYaml .Values.prometheus.service.additionalPorts | indent 2 }} +{{- end }} + publishNotReadyAddresses: {{ .Values.prometheus.service.publishNotReadyAddresses }} + selector: + app.kubernetes.io/name: prometheus + prometheus: {{ template "project-prometheus-stack.prometheus.crname" . }} +{{- if .Values.prometheus.service.sessionAffinity }} + sessionAffinity: {{ .Values.prometheus.service.sessionAffinity }} +{{- end }} + type: "{{ .Values.prometheus.service.type }}" +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/serviceaccount.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/serviceaccount.yaml new file mode 100644 index 0000000000..b4d4416403 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/serviceaccount.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "project-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/name: {{ template "project-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "project-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.prometheus.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "project-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/servicemonitor.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/servicemonitor.yaml new file mode 100644 index 0000000000..2d995511e7 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/prometheus/servicemonitor.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "project-prometheus-stack.namespace" . }} + labels: + app: {{ template "project-prometheus-stack.name" . }}-prometheus +{{ include "project-prometheus-stack.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app: {{ template "project-prometheus-stack.name" . }}-prometheus + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "project-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{ toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.serviceMonitor.bearerTokenFile }} + {{- end }} + path: "{{ trimSuffix "/" .Values.prometheus.prometheusSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheus.serviceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/rancher-monitoring/hardened.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/rancher-monitoring/hardened.yaml new file mode 100644 index 0000000000..42f13a0f27 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/rancher-monitoring/hardened.yaml @@ -0,0 +1,129 @@ +{{- $namespaces := dict "_0" .Release.Namespace -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "project-prometheus-stack.fullname" . }}-patch-sa + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-patch-sa + labels: + app: {{ template "project-prometheus-stack.fullname" . }}-patch-sa + spec: + serviceAccountName: {{ template "project-prometheus-stack.fullname" . }}-patch-sa +{{- if .Values.global.kubectl.securityContext }} + securityContext: {{ toYaml .Values.global.kubectl.securityContext | nindent 8 }} +{{- end }} + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + {{- range $_, $ns := $namespaces }} + - name: patch-sa-{{ $ns }} + image: {{ template "system_default_registry" $ }}{{ $.Values.global.kubectl.repository }}:{{ $.Values.global.kubectl.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: ["kubectl", "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] + args: ["-n", "{{ $ns }}"] +{{- if $.Values.global.kubectl.resources }} + resources: {{ toYaml $.Values.global.kubectl.resources | nindent 10 }} +{{- end }} +{{- if $.Values.global.kubectl.containerSecurityContext }} + securityContext: {{ toYaml $.Values.global.kubectl.containerSecurityContext | nindent 10 }} +{{- end }} + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-patch-sa + labels: + app: {{ template "project-prometheus-stack.fullname" . }}-patch-sa +rules: +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: ['get', 'patch'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "project-prometheus-stack.fullname" . }}-patch-sa +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-patch-sa + labels: + app: {{ template "project-prometheus-stack.fullname" . }}-patch-sa +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "project-prometheus-stack.fullname" . }}-patch-sa +subjects: +- kind: ServiceAccount + name: {{ template "project-prometheus-stack.fullname" . }}-patch-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "project-prometheus-stack.fullname" . }}-patch-sa +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "project-prometheus-stack.fullname" . }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "project-prometheus-stack.fullname" . }}-patch-sa +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }} +{{- range $_, $ns := $namespaces }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: project-monitoring-policy + namespace: {{ $ns }} +spec: + podSelector: {} + ingress: {{- default (list dict | toYaml) (include "project-prometheus-stack.hardened.networkPolicy.ingress" (list $ $ns)) | nindent 4 }} + egress: {{- $.Values.global.networkPolicy.egress | toYaml | nindent 4 }} + policyTypes: + - Ingress + - Egress +{{- end }} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/validate-install-crd.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..b0dcf188bb --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/validate-install-crd.yaml @@ -0,0 +1,21 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/AlertmanagerConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/Alertmanager" false -}} +# {{- set $found "monitoring.coreos.com/v1/PodMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/Probe" false -}} +# {{- set $found "monitoring.coreos.com/v1/Prometheus" false -}} +# {{- set $found "monitoring.coreos.com/v1/PrometheusRule" false -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/ThanosRuler" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required Prometheus Operator CRDs are missing. Please install Prometheus Operator (e.g. rancher-monitoring) before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} \ No newline at end of file diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/validate-psp-install.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-project-monitoring/0.3.0+up0.3.3/values.yaml b/charts/rancher-project-monitoring/0.3.0+up0.3.3/values.yaml new file mode 100644 index 0000000000..1f0f165437 --- /dev/null +++ b/charts/rancher-project-monitoring/0.3.0+up0.3.3/values.yaml @@ -0,0 +1,1551 @@ +# Default values for project-prometheus-stack. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Rancher Project Monitoring Configuration + +## Provide a name in place of project-prometheus-stack for `app:` labels +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +nameOverride: "rancher-project-monitoring" + +## Override the deployment namespace +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +namespaceOverride: "" + +## Provide a k8s version to auto dashboard import script example: kubeTargetVersionOverride: 1.16.6 +## +kubeTargetVersionOverride: "" + +## Allow kubeVersion to be overridden while creating the ingress +## +kubeVersionOverride: "" + +## Provide a name to substitute for the full names of resources +## +fullnameOverride: "" + +## Labels to apply to all resources +## +commonLabels: {} +# scmhash: abc123 +# myLabel: aakkmd + +## Create default rules for monitoring the cluster +## +defaultRules: + create: true + rules: + general: true + prometheus: true + alertmanager: true + kubernetesApps: true + kubernetesStorage: true + + ## Reduce app namespace alert scope + appNamespacesTarget: ".*" + + ## Labels for default rules + labels: {} + ## Annotations for default rules + annotations: {} + + ## Additional labels for PrometheusRule alerts + additionalRuleLabels: {} + + ## Additional annotations for PrometheusRule alerts + additionalRuleAnnotations: {} + + ## Prefix for runbook URLs. Use this to override the first part of the runbookURLs that is common to all rules. + runbookUrl: "https://runbooks.prometheus-operator.dev/runbooks" + + ## Disabled PrometheusRule alerts + disabled: {} + +## +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + projectNamespaceSelector: {} + projectNamespaces: [] + kubectl: + repository: rancher/kubectl + tag: v1.20.2 + pullPolicy: IfNotPresent + securityContext: + runAsNonRoot: true + runAsUser: 1000 + networkPolicy: + # If activated, creates ingress network policies to only allow ingress traffic from workloads within the project. + # This only works correctly, if Project Network Isolation is activated for the cluster in Rancher. Otherwise, + # Ingress traffic from the nodes and thus from the Kubernetes API will be blocked, which breaks accessing the UIs + # through the Rancher/Kubernetes API Proxy in the Rancher UI. + limitIngressToProject: false + # Custom ingress restrictions. If null and limitIngressToProject=false, all ingress traffic will be allowed. + ingress: null + # By default, all egress traffic is allowed. + egress: + - {} + rbac: + ## Create RBAC resources for ServiceAccounts and users + ## + create: true + + userRoles: + ## Create default user Roles that the Helm Project Operator will automatically create RoleBindings for + ## + ## How does this work? + ## + ## The operator will watch for all subjects bound to each Kubernetes default ClusterRole in the project registration namespace + ## where the ProjectHelmChart that deployed this chart belongs to; if it observes a subject bound to a particular role in + ## the project registration namespace (e.g. edit) and if a Role exists that is deployed by this chart with the label + ## 'helm.cattle.io/project-helm-chart-role-aggregate-from': '', it will automaticaly create a RoleBinding + ## in the release namespace binding all such subjects to that Role. + ## + ## Note: while the default behavior is to use the Kubernetes default ClusterRole, the operator deployment (prometheus-federator) + ## can be configured to use a different set of ClusterRoles as the source of truth for admin, edit, and view permissions. + ## + create: true + ## Add labels to Roles + aggregateToDefaultRoles: true + + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + # or + # - "image-pull-secret" + +federate: + ## enabled indicates whether to add federation to any Project Prometheus Stacks by default + ## If not enabled, no federation will be turned on + enabled: true + + # Change this to point at all Prometheuses you want all your Project Prometheus Stacks to federate from + # By default, this matches the default deployment of Rancher Monitoring + targets: + - rancher-monitoring-prometheus.cattle-monitoring-system.svc:9090 + + ## Scrape interval + interval: "15s" + +## Configuration for alertmanager +## ref: https://prometheus.io/docs/alerting/alertmanager/ +## +alertmanager: + + ## Deploy alertmanager + ## + enabled: true + + ## Annotations for Alertmanager + ## + annotations: {} + + ## Api that prometheus will use to communicate with alertmanager. Possible values are v1, v2 + ## + apiVersion: v2 + + ## Service account for Alertmanager to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + + ## Configure pod disruption budgets for Alertmanager + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + ## Alertmanager configuration directives + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + config: + global: + resolve_timeout: 5m + inhibit_rules: + - source_matchers: + - 'severity = critical' + target_matchers: + - 'severity =~ warning|info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'severity = warning' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'alertname = InfoInhibitor' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + route: + group_by: ['namespace'] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: 'null' + routes: + - receiver: 'null' + matchers: + - alertname =~ "InfoInhibitor|Watchdog" + receivers: + - name: 'null' + templates: + - '/etc/alertmanager/config/*.tmpl' + + ## Pass the Alertmanager configuration directives through Helm's templating + ## engine. If the Alertmanager configuration contains Alertmanager templates, + ## they'll need to be properly escaped so that they are not interpreted by + ## Helm + ## ref: https://helm.sh/docs/developing_charts/#using-the-tpl-function + ## https://prometheus.io/docs/alerting/configuration/#tmpl_string + ## https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + tplConfig: false + + ## Alertmanager template files to format alerts + ## By default, templateFiles are placed in /etc/alertmanager/config/ and if + ## they have a .tmpl file suffix will be loaded. See config.templates above + ## to change, add other suffixes. If adding other suffixes, be sure to update + ## config.templates above to include those suffixes. + ## ref: https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + ## + + templateFiles: + rancher_defaults.tmpl: |- + {{- define "slack.rancher.text" -}} + {{ template "rancher.text_multiple" . }} + {{- end -}} + + {{- define "rancher.text_multiple" -}} + *[GROUP - Details]* + One or more alarms in this group have triggered a notification. + + {{- if gt (len .GroupLabels.Values) 0 }} + *Group Labels:* + {{- range .GroupLabels.SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- if .ExternalURL }} + *Link to AlertManager:* {{ .ExternalURL }} + {{- end }} + + {{- range .Alerts }} + {{ template "rancher.text_single" . }} + {{- end }} + {{- end -}} + + {{- define "rancher.text_single" -}} + {{- if .Labels.alertname }} + *[ALERT - {{ .Labels.alertname }}]* + {{- else }} + *[ALERT]* + {{- end }} + {{- if .Labels.severity }} + *Severity:* `{{ .Labels.severity }}` + {{- end }} + {{- if .Labels.cluster }} + *Cluster:* {{ .Labels.cluster }} + {{- end }} + {{- if .Annotations.summary }} + *Summary:* {{ .Annotations.summary }} + {{- end }} + {{- if .Annotations.message }} + *Message:* {{ .Annotations.message }} + {{- end }} + {{- if .Annotations.description }} + *Description:* {{ .Annotations.description }} + {{- end }} + {{- if .Annotations.runbook_url }} + *Runbook URL:* <{{ .Annotations.runbook_url }}|:spiral_note_pad:> + {{- end }} + {{- with .Labels }} + {{- with .Remove (stringSlice "alertname" "severity" "cluster") }} + {{- if gt (len .) 0 }} + *Additional Labels:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Annotations }} + {{- with .Remove (stringSlice "summary" "message" "description" "runbook_url") }} + {{- if gt (len .) 0 }} + *Additional Annotations:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end -}} + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + + labels: {} + + ## Redirect ingress to an additional defined port on the service + # servicePort: 8081 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - alertmanager.domain.com + + ## Paths to use for ingress rules - one path should match the alertmanagerSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Alertmanager Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: alertmanager-general-tls + # hosts: + # - alertmanager.example.com + + ## Configuration for Alertmanager secret + ## + secret: + annotations: {} + + ## Configuration for Alertmanager service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Alertmanager Service to listen on + ## + port: 9093 + ## To be used with a proxy extraContainer port + ## + targetPort: 9093 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30903 + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + + ## Additional ports to open for Alertmanager service + additionalPorts: [] + # additionalPorts: + # - name: authenticated + # port: 8081 + # targetPort: 8081 + + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## If true, create a serviceMonitor for alertmanager + ## + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + selfMonitor: true + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Settings affecting alertmanagerSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerspec + ## + alertmanagerSpec: + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the Alertmanager pods. + ## + podMetadata: {} + + ## Image of Alertmanager + ## + image: + repository: rancher/mirrored-prometheus-alertmanager + tag: v0.24.0 + sha: "" + + ## Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the + ## Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/. + ## + secrets: [] + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. + ## The ConfigMaps are mounted into /etc/alertmanager/configmaps/. + ## + configMaps: [] + + ## ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for + ## this Alertmanager instance. Defaults to 'alertmanager-' The secret is mounted into /etc/alertmanager/config. + ## + # configSecret: + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerwebspec + web: {} + + ## AlertmanagerConfigs to be selected to merge and configure Alertmanager with. + ## + alertmanagerConfigSelector: + # default ignores resources created by Rancher Monitoring + matchExpressions: + - key: release + operator: NotIn + values: + - rancher-monitoring + + ## AlermanagerConfig to be used as top level configuration + ## + alertmanagerConfiguration: {} + ## Example with select a global alertmanagerconfig + # alertmanagerConfiguration: + # name: global-alertmanager-Configuration + + ## Define Log Format + # Use logfmt (default) or json logging + logFormat: logfmt + + ## Log level for Alertmanager to be configured with. + ## + logLevel: info + + ## Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the + ## running cluster equal to the expected size. + replicas: 1 + + ## Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression + ## [0-9]+(ms|s|m|h) (milliseconds seconds minutes hours). + ## + retention: 120h + + ## Storage is the definition of how storage will be used by the Alertmanager instances. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storage: {} + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + + ## The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name. string false + ## + externalUrl: + + ## The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, + ## but the server serves requests under a different route prefix. For example for use with kubectl proxy. + ## + routePrefix: / + + ## If set to true all actions on the underlying managed objects are not going to be performed, except for delete actions. + ## + paused: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Define resources requests and limits for single Pods. + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + limits: + memory: 500Mi + cpu: 1000m + requests: + memory: 100Mi + cpu: 100m + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + ## + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the alertmanager instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## If specified, the pod's tolerations. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: alertmanager + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + + ## ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. + ## Note this is only for the Alertmanager UI, not the gossip communication. + ## + listenLocal: false + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod. + ## + containers: [] + # containers: + # - name: oauth-proxy + # image: quay.io/oauth2-proxy/oauth2-proxy:v7.3.0 + # args: + # - --upstream=http://127.0.0.1:9093 + # - --http-address=0.0.0.0:8081 + # - ... + # ports: + # - containerPort: 8081 + # name: oauth-proxy + # protocol: TCP + # resources: {} + + # Additional volumes on the output StatefulSet definition. + volumes: [] + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster. + ## + additionalPeers: [] + + ## PortName to use for Alert Manager. + ## + portName: "http-web" + + ## ClusterAdvertiseAddress is the explicit address to advertise in cluster. Needs to be provided for non RFC1918 [1] (public) addresses. [1] RFC1918: https://tools.ietf.org/html/rfc1918 + ## + clusterAdvertiseAddress: false + + ## ForceEnableClusterMode ensures Alertmanager does not deactivate the cluster mode when running with a single replica. + ## Use case is e.g. spanning an Alertmanager cluster across Kubernetes clusters with a single replica in each. + forceEnableClusterMode: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + +## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml +## +grafana: + enabled: true + namespaceOverride: "" + + ## Grafana's primary configuration + ## NOTE: values in map will be converted to ini format + ## ref: http://docs.grafana.org/installation/configuration/ + ## + grafana.ini: + users: + auto_assign_org_role: Viewer + auth: + disable_login_form: false + auth.anonymous: + enabled: true + org_role: Viewer + auth.basic: + enabled: false + security: + # Required to embed dashboards in Rancher Cluster Overview Dashboard on Cluster Explorer + allow_embedding: true + + deploymentStrategy: + type: Recreate + + ## ForceDeployDatasources Create datasource configmap even if grafana deployment has been disabled + ## + forceDeployDatasources: false + + ## ForceDeployDashboard Create dashboard configmap even if grafana deployment has been disabled + ## + forceDeployDashboards: false + + ## Deploy default dashboards + ## + defaultDashboardsEnabled: true + + ## Timezone for the default dashboards + ## Other options are: browser or a specific timezone, i.e. Europe/Luxembourg + ## + defaultDashboardsTimezone: utc + + adminPassword: prom-operator + + ingress: + ## If true, Grafana Ingress will be created + ## + enabled: false + + ## IngressClassName for Grafana Ingress. + ## Should be provided if Ingress is enable. + ## + # ingressClassName: nginx + + ## Annotations for Grafana Ingress + ## + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + + ## Labels to be added to the Ingress + ## + labels: {} + + ## Hostnames. + ## Must be provided if Ingress is enable. + ## + # hosts: + # - grafana.domain.com + hosts: [] + + ## Path for grafana ingress + path: / + + ## TLS configuration for grafana Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: grafana-general-tls + # hosts: + # - grafana.example.com + + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + labelValue: "1" + + ## Annotations for Grafana dashboard configmaps + ## + annotations: {} + multicluster: + global: + enabled: false + provider: + allowUiUpdates: false + datasources: + enabled: true + defaultDatasourceEnabled: true + + uid: prometheus + + ## URL of prometheus datasource + ## + # url: http://prometheus-stack-prometheus:9090/ + + # If not defined, will use prometheus.prometheusSpec.scrapeInterval or its default + # defaultDatasourceScrapeInterval: 15s + + ## Annotations for Grafana datasource configmaps + ## + annotations: {} + + label: grafana_datasource + labelValue: "1" + + ## Field with internal link pointing to existing data source in Grafana. + ## Can be provisioned via additionalDataSources + exemplarTraceIdDestinations: {} + # datasourceUid: Jaeger + # traceIdLabelName: trace_id + + extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # configMap: certs-configmap + # readOnly: true + + deleteDatasources: [] + # - name: example-datasource + # orgId: 1 + + ## Configure additional grafana datasources (passed through tpl) + ## ref: http://docs.grafana.org/administration/provisioning/#datasources + additionalDataSources: [] + # - name: prometheus-sample + # access: proxy + # basicAuth: true + # basicAuthPassword: pass + # basicAuthUser: daco + # editable: false + # jsonData: + # tlsSkipVerify: true + # orgId: 1 + # type: prometheus + # url: https://{{ printf "%s-prometheus.svc" .Release.Name }}:9090 + # version: 1 + + ## Passed to grafana subchart and used by servicemonitor below + ## + service: + portName: nginx-http + ## Port for Grafana Service to listen on + ## + port: 80 + ## To be used with a proxy extraContainer port + ## + targetPort: 8080 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30950 + ## Service type + ## + type: ClusterIP + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod + extraContainers: | + - name: grafana-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }}" + ports: + - containerPort: 8080 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: grafana-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## Volumes that can be used in containers + extraContainerVolumes: + - name: nginx-home + emptyDir: {} + - name: grafana-nginx + configMap: + name: grafana-nginx-proxy-config + items: + - key: nginx.conf + mode: 438 + path: nginx.conf + + ## If true, create a serviceMonitor for grafana + ## + serviceMonitor: + # If true, a ServiceMonitor CRD is created for a prometheus operator + # https://github.com/coreos/prometheus-operator + # + enabled: true + + # Path to use for scraping metrics. Might be different if server.root_url is set + # in grafana.ini + path: "/metrics" + + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + + # labels for the ServiceMonitor + labels: {} + + # Scrape interval. If not set, the Prometheus default scrape interval is used. + # + interval: "" + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + resources: + limits: + memory: 200Mi + cpu: 200m + requests: + memory: 100Mi + cpu: 100m + + testFramework: + enabled: false + +## Deploy a Prometheus instance +## +prometheus: + + enabled: true + + ## Annotations for Prometheus + ## + annotations: {} + + ## Service account for Prometheuses to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + + ## Configuration for Prometheus service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Prometheus Service to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 8081 + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30090 + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Additional port to define in the Service + additionalPorts: [] + # additionalPorts: + # - name: authenticated + # port: 8081 + # targetPort: 8081 + + ## Consider that all endpoints are considered "ready" even if the Pods themselves are not + ## Ref: https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/#ServiceSpec + publishNotReadyAddresses: false + + sessionAffinity: "" + + ## Configure pod disruption budgets for Prometheus + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Redirect ingress to an additional defined port on the service + # servicePort: 8081 + + ## Hostnames. + ## Must be provided if Ingress is enabled. + ## + # hosts: + # - prometheus.domain.com + hosts: [] + + ## Paths to use for ingress rules - one path should match the prometheusSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Prometheus Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: prometheus-general-tls + # hosts: + # - prometheus.example.com + + ## Configure additional options for default pod security policy for Prometheus + ## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + allowedCapabilities: [] + volumes: [] + + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + selfMonitor: true + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Settings affecting prometheusSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheusspec + ## + prometheusSpec: + ## If true, pass --storage.tsdb.max-block-duration=2h to prometheus. This is already done if using Thanos + ## + disableCompaction: false + ## APIServerConfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#apiserverconfig + ## + apiserverConfig: {} + + ## Interval between consecutive scrapes. + ## Defaults to 30s. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/release-0.44/pkg/prometheus/promcfg.go#L180-L183 + ## + scrapeInterval: "30s" + + ## Number of seconds to wait for target to respond before erroring + ## + scrapeTimeout: "" + + ## Interval between consecutive evaluations. + ## + evaluationInterval: "1m" + + ## ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP. + ## + listenLocal: false + + ## EnableAdminAPI enables Prometheus the administrative HTTP API which includes functionality such as deleting time series. + ## This is disabled by default. + ## ref: https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis + ## + enableAdminAPI: false + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#webtlsconfig + web: {} + + ## Exemplars related settings that are runtime reloadable. + ## It requires to enable the exemplar storage feature to be effective. + exemplars: "" + ## Maximum number of exemplars stored in memory for all series. + ## If not set, Prometheus uses its default value. + ## A value of zero or less than zero disables the storage. + # maxSize: 100000 + + # EnableFeatures API enables access to Prometheus disabled features. + # ref: https://prometheus.io/docs/prometheus/latest/disabled_features/ + enableFeatures: [] + # - exemplar-storage + + ## Image of Prometheus. + ## + image: + repository: rancher/mirrored-prometheus-prometheus + tag: v2.38.0 + sha: "" + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: prometheus + + ## Alertmanagers to which alerts will be sent + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerendpoints + ## + ## Default configuration will connect to the alertmanager deployed as part of this release + ## + alertingEndpoints: [] + # - name: "" + # namespace: "" + # port: http + # scheme: http + # pathPrefix: "" + # tlsConfig: {} + # bearerTokenFile: "" + # apiVersion: v2 + + ## External labels to add to any time series or alerts when communicating with external systems + ## + externalLabels: {} + + ## enable --web.enable-remote-write-receiver flag on prometheus-server + ## + enableRemoteWriteReceiver: false + + ## Name of the external label used to denote Prometheus instance name + ## + prometheusExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote Prometheus instance name + ## + prometheusExternalLabelNameClear: false + + ## External URL at which Prometheus will be reachable. + ## + externalUrl: "" + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The Secrets are mounted into /etc/prometheus/secrets/. Secrets changes after initial creation of a Prometheus object are not + ## reflected in the running Pods. To change the secrets mounted into the Prometheus Pods, the object must be deleted and recreated + ## with the new list of secrets. + ## + secrets: [] + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The ConfigMaps are mounted into /etc/prometheus/configmaps/. + ## + configMaps: [] + + ## QuerySpec defines the query command line flags when starting Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#queryspec + ## + query: {} + + ## If true, a nil or {} value for prometheus.prometheusSpec.ruleSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the PrometheusRule resources created + ## + ruleSelectorNilUsesHelmValues: false + + ## PrometheusRules to be selected for target discovery. + ## If {}, select all PrometheusRules + ## + ruleSelector: + # default ignores resources created by Rancher Monitoring + matchExpressions: + - key: release + operator: NotIn + values: + - rancher-monitoring + + ## If true, a nil or {} value for prometheus.prometheusSpec.serviceMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the servicemonitors created + ## + serviceMonitorSelectorNilUsesHelmValues: false + + ## ServiceMonitors to be selected for target discovery. + ## If {}, select all ServiceMonitors + ## + serviceMonitorSelector: + # default ignores resources created by Rancher Monitoring + matchExpressions: + - key: release + operator: NotIn + values: + - rancher-monitoring + + ## If true, a nil or {} value for prometheus.prometheusSpec.podMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the podmonitors created + ## + podMonitorSelectorNilUsesHelmValues: false + + ## PodMonitors to be selected for target discovery. + ## If {}, select all PodMonitors + ## + podMonitorSelector: + # default ignores resources created by Rancher Monitoring + matchExpressions: + - key: release + operator: NotIn + values: + - rancher-monitoring + + ## If true, a nil or {} value for prometheus.prometheusSpec.probeSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the probes created + ## + probeSelectorNilUsesHelmValues: true + + ## Probes to be selected for target discovery. + ## If {}, select all Probes + ## + probeSelector: + # default ignores resources created by Rancher Monitoring + matchExpressions: + - key: release + operator: NotIn + values: + - rancher-monitoring + + ## How long to retain metrics + ## + retention: 10d + + ## Maximum size of metrics + ## + retentionSize: "50GiB" + + ## Enable compression of the write-ahead log using Snappy. + ## + walCompression: true + + ## If true, the Operator won't process any Prometheus configuration changes + ## + paused: false + + ## Number of replicas of each shard to deploy for a Prometheus deployment. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## + replicas: 1 + + ## EXPERIMENTAL: Number of shards to distribute targets onto. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved. + ## Increasing shards will not reshard data either but it will continue to be available from the same instances. + ## To query globally use Thanos sidecar and Thanos querier or remote write data to a central location. + ## Sharding is done on the content of the `__address__` target meta-label. + ## + shards: 1 + + ## Log level for Prometheus be configured in + ## + logLevel: info + + ## Log format for Prometheus be configured in + ## + logFormat: logfmt + + ## Prefix used to register routes, overriding externalUrl route. + ## Useful for proxies that rewrite URLs. + ## + routePrefix: / + + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the prometheus pods. + ## + podMetadata: {} + # labels: + # app: prometheus + # k8s-app: prometheus + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the prometheus instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## The remote_write spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotewritespec + remoteWrite: [] + # - url: http://remote1/push + ## additionalRemoteWrite is appended to remoteWrite + additionalRemoteWrite: [] + + ## Enable/Disable Grafana dashboards provisioning for prometheus remote write feature + remoteWriteDashboards: false + + ## Resource limits & requests + ## + resources: + limits: + memory: 3000Mi + cpu: 1000m + requests: + memory: 750Mi + cpu: 750m + + storage: + enabled: false + ## Prometheus StorageSpec for persistent data + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storageSpec: {} + ## Using PersistentVolumeClaim + ## + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + ## Using tmpfs volume + ## + # emptyDir: + # medium: Memory + + # Additional volumes on the output StatefulSet definition. + volumes: + - name: nginx-home + emptyDir: {} + - name: prometheus-nginx + configMap: + name: prometheus-nginx-proxy-config + defaultMode: 438 + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. + ## https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to a Prometheus pod. + ## if using proxy extraContainer update targetPort with proxy container port + containers: | + - name: prometheus-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.proxy.image.repository }}:{{ .Values.prometheus.prometheusSpec.proxy.image.tag }}" + ports: + - containerPort: 8081 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: prometheus-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## PortName to use for Prometheus. + ## + portName: "http-web" + + ## ArbitraryFSAccessThroughSMs configures whether configuration based on a service monitor can access arbitrary files + ## on the file system of the Prometheus container e.g. bearer token files. + arbitraryFSAccessThroughSMs: false + + ## OverrideHonorLabels if set to true overrides all user configured honor_labels. If HonorLabels is set in ServiceMonitor + ## or PodMonitor to true, this overrides honor_labels to false. + overrideHonorLabels: false + + ## OverrideHonorTimestamps allows to globally enforce honoring timestamps in all scrape configs. + overrideHonorTimestamps: false + + ## EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. + ## The label value will always be the namespace of the object that is being created. + ## Disabled by default + enforcedNamespaceLabel: "" + + ## PrometheusRulesExcludedFromEnforce - list of prometheus rules to be excluded from enforcing of adding namespace labels. + ## Works only if enforcedNamespaceLabel set to true. Make sure both ruleNamespace and ruleName are set for each pair + ## Deprecated, use `excludedFromEnforcement` instead + prometheusRulesExcludedFromEnforce: [] + + ## ExcludedFromEnforcement - list of object references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects + ## to be excluded from enforcing a namespace label of origin. + ## Works only if enforcedNamespaceLabel set to true. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#objectreference + excludedFromEnforcement: [] + + ## QueryLogFile specifies the file to which PromQL queries are logged. Note that this location must be writable, + ## and can be persisted using an attached volume. Alternatively, the location can be set to a stdout location such + ## as /dev/stdout to log querie information to the default Prometheus log stream. This is only available in versions + ## of Prometheus >= 2.16.0. For more details, see the Prometheus docs (https://prometheus.io/docs/guides/query-log/) + queryLogFile: false + + ## EnforcedSampleLimit defines global limit on number of scraped samples that will be accepted. This overrides any SampleLimit + ## set per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the SampleLimit to keep overall + ## number of samples/series under the desired limit. Note that if SampleLimit is lower that value will be taken instead. + enforcedSampleLimit: false + + ## EnforcedTargetLimit defines a global limit on the number of scraped targets. This overrides any TargetLimit set + ## per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the TargetLimit to keep the overall + ## number of targets under the desired limit. Note that if TargetLimit is lower, that value will be taken instead, except + ## if either value is zero, in which case the non-zero value will be used. If both values are zero, no limit is enforced. + enforcedTargetLimit: false + + + ## Per-scrape limit on number of labels that will be accepted for a sample. If more than this number of labels are present + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelLimit: false + + ## Per-scrape limit on length of labels name that will be accepted for a sample. If a label name is longer than this number + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelNameLengthLimit: false + + ## Per-scrape limit on length of labels value that will be accepted for a sample. If a label value is longer than this + ## number post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus + ## versions 2.27.0 and newer. + enforcedLabelValueLengthLimit: false + + ## AllowOverlappingBlocks enables vertical compaction and vertical query merge in Prometheus. This is still experimental + ## in Prometheus so it may change in any upcoming release. + allowOverlappingBlocks: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + +## Setting to true produces cleaner resource names, but requires a data migration because the name of the persistent volume changes. Therefore this should only be set once on initial installation. +## +cleanPrometheusOperatorObjectNames: false \ No newline at end of file diff --git a/index.yaml b/index.yaml index d0db2fe910..6636d49f27 100755 --- a/index.yaml +++ b/index.yaml @@ -14801,6 +14801,42 @@ entries: urls: - assets/rancher-project-monitoring/rancher-project-monitoring-1.0.0+up0.1.0.tgz version: 1.0.0+up0.1.0 + - annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Project Monitoring + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.16.0-0 < 1.25.0-0' + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/rancher-version: '>= 2.6.5-0 < 2.7.0-0' + catalog.cattle.io/release-name: rancher-project-monitoring + apiVersion: v2 + appVersion: 0.59.1 + created: "2023-09-21T15:19:46.907035-07:00" + dependencies: + - condition: grafana.enabled + name: grafana + repository: file://./charts/grafana + description: Collects several related Helm charts, Grafana dashboards, and Prometheus + rules combined with documentation and scripts to provide easy to operate end-to-end + Kubernetes cluster monitoring with Prometheus. Depends on the existence of a + Cluster Prometheus deployed via Prometheus Operator + digest: d211f2643a2b1e0072c01f582d2c7459dd6e05e2f67d7defa25e70185515e5de + home: https://github.com/prometheus-operator/kube-prometheus + icon: https://raw.githubusercontent.com/prometheus/prometheus.github.io/master/assets/prometheus_logo-cb55bb5c346.png + keywords: + - prometheus + - monitoring + maintainers: + - email: arvind.iyengar@suse.com + name: Arvind + - email: amangeet.samra@suse.com + name: Geet + url: https://github.com/geethub97 + name: rancher-project-monitoring + type: application + urls: + - assets/rancher-project-monitoring/rancher-project-monitoring-0.3.0+up0.3.3.tgz + version: 0.3.0+up0.3.3 - annotations: catalog.cattle.io/certified: rancher catalog.cattle.io/display-name: Project Monitoring diff --git a/release.yaml b/release.yaml index 9d6e1cb0d1..d09be761e0 100644 --- a/release.yaml +++ b/release.yaml @@ -4,3 +4,5 @@ longhorn-crd: - 103.1.1+up1.4.4 prometheus-federator: - 0.3.0+up0.3.3 +rancher-project-monitoring: + - 0.3.0+up0.3.3